diff --git a/package.json b/package.json index 1a6eb54f7..8d6be361f 100644 --- a/package.json +++ b/package.json @@ -118,6 +118,9 @@ "test:auth": "npx tsx test/continuous-test-suite-auth.ts", "test:autoresearch": "npx tsx test/continuous-test-suite-autoresearch.ts", "test:autoresearch:redis": "npx tsx test/continuous-test-suite-autoresearch-redis.ts", + "test:anthropic-tools-policy": "npx tsx test/continuous-test-suite-anthropic-tools-policy.ts", + "test:anthropic-multimodal": "npx tsx test/continuous-test-suite-anthropic-multimodal.ts", + "test:excel-interop": "npx tsx test/continuous-test-suite-excel-interop.ts", "test:envguard": "npx tsx test/helpers/envGuard.test.ts", "test:tool-routing-cli:vitest": "pnpm exec vitest run test/toolRoutingCli.test.ts", "test:tool-routing-cli": "pnpm run test:tool-routing-cli:vitest && npx tsx test/continuous-test-suite-tool-routing-cli.ts", @@ -131,7 +134,7 @@ "test:tool-routing": "pnpm run test:unit:vitest && npx tsx test/continuous-test-suite-tool-routing.ts", "test:tool-routing-semantic:vitest": "pnpm exec vitest run test/toolRoutingSemantic.test.ts", "test:tool-routing-semantic": "pnpm run test:tool-routing-semantic:vitest && npx tsx test/continuous-test-suite-tool-routing-semantic.ts", - "test:unit": "pnpm run test:envguard && pnpm run test:bugfixes && pnpm run test:mcp:infra && pnpm run test:mcp:bash && pnpm run test:mcp:limits && pnpm run test:mcp:spans && pnpm run test:autoresearch:redis && pnpm run test:unit:vitest && pnpm run test:tool-routing-cli:vitest && pnpm run test:tool-dedup:vitest && pnpm run test:model-pool:vitest && pnpm run test:tool-routing-semantic:vitest", + "test:unit": "pnpm run test:envguard && pnpm run test:bugfixes && pnpm run test:mcp:infra && pnpm run test:mcp:bash && pnpm run test:mcp:limits && pnpm run test:mcp:spans && pnpm run test:autoresearch:redis && pnpm run test:unit:vitest && pnpm run test:tool-routing-cli:vitest && pnpm run test:tool-dedup:vitest && pnpm run test:model-pool:vitest && pnpm run test:tool-routing-semantic:vitest && pnpm run test:anthropic-tools-policy && pnpm run test:anthropic-multimodal && pnpm run test:excel-interop", "// CI tier — live providers, runs only when API keys are present (test:credentials and test:dynamic make real provider calls when keys are set, so they live here, not in test:unit)": "", "test:live": "pnpm run test:providers && pnpm run test:mcp:http && pnpm run test:mcp:sdk && pnpm run test:mcp:cli && pnpm run test:observability && pnpm run test:context && pnpm run test:memory && pnpm run test:tool-reliability && pnpm run test:evaluation && pnpm run test:autoresearch && pnpm run test:credentials && pnpm run test:dynamic", "// CI tier — product output (image/video/TTS/PPT) — costs $$ per run": "", @@ -598,6 +601,9 @@ "rollup@>=4.0.0 <4.59.0": ">=4.59.0", "shell-quote@<1.8.4": ">=1.8.4", "undici@>=8.0.0": ">=7.24.0 <8.0.0" + }, + "patchedDependencies": { + "mammoth@1.12.0": "patches/mammoth@1.12.0.patch" } }, "os": [ diff --git a/patches/mammoth@1.12.0.patch b/patches/mammoth@1.12.0.patch new file mode 100644 index 000000000..0862d5b66 --- /dev/null +++ b/patches/mammoth@1.12.0.patch @@ -0,0 +1,26 @@ +diff --git a/lib/xml/xmldom.js b/lib/xml/xmldom.js +index 752c353f..45cc659a 100644 +--- a/lib/xml/xmldom.js ++++ b/lib/xml/xmldom.js +@@ -4,13 +4,18 @@ var dom = require("@xmldom/xmldom/lib/dom"); + function parseFromString(string) { + var error = null; + ++ // @xmldom/xmldom >= 0.9 replaced the `errorHandler` constructor option with ++ // `onError` and made the `mimeType` argument to parseFromString required. ++ // Capture only real (non-warning) errors so benign warnings don't abort. + var domParser = new xmldom.DOMParser({ +- errorHandler: function(level, message) { +- error = {level: level, message: message}; ++ onError: function(level, message) { ++ if (level === "error" || level === "fatalError") { ++ error = {level: level, message: message}; ++ } + } + }); + +- var document = domParser.parseFromString(string); ++ var document = domParser.parseFromString(string, "text/xml"); + + if (error === null) { + return document; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 607b1ad5a..6f0a74468 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -48,6 +48,11 @@ overrides: pnpmfileChecksum: sha256-qFFlrDVMxTAG02VWf3xBAHmYpquoAuUauBBYPNQv57g= +patchedDependencies: + mammoth@1.12.0: + hash: 08d907da666d60e4aca12deab636e3795c9bacbec7f10f258653467aadea0362 + path: patches/mammoth@1.12.0.patch + importers: .: @@ -488,7 +493,7 @@ importers: version: 2.15.4 mammoth: specifier: ^1.11.0 - version: 1.12.0 + version: 1.12.0(patch_hash=08d907da666d60e4aca12deab636e3795c9bacbec7f10f258653467aadea0362) mediabunny: specifier: ^1.40.1 version: 1.40.1 @@ -10643,7 +10648,7 @@ snapshots: inquirer: 13.4.3(@types/node@25.5.0) jose: 6.2.3 json-schema-to-zod: 2.8.1 - mammoth: 1.12.0 + mammoth: 1.12.0(patch_hash=08d907da666d60e4aca12deab636e3795c9bacbec7f10f258653467aadea0362) mathjs: 15.2.0 minisearch: 7.2.0 music-metadata: 11.13.0 @@ -15961,7 +15966,7 @@ snapshots: dependencies: semver: 7.7.4 - mammoth@1.12.0: + mammoth@1.12.0(patch_hash=08d907da666d60e4aca12deab636e3795c9bacbec7f10f258653467aadea0362): dependencies: '@xmldom/xmldom': 0.9.10 argparse: 1.0.10 diff --git a/src/lib/core/modules/GenerationHandler.ts b/src/lib/core/modules/GenerationHandler.ts index b93ddc4bd..5e7e34cc3 100644 --- a/src/lib/core/modules/GenerationHandler.ts +++ b/src/lib/core/modules/GenerationHandler.ts @@ -40,6 +40,7 @@ import { } from "../../utils/tokenUtils.js"; import { DEFAULT_MAX_STEPS } from "../constants.js"; import { + isTemperatureDeprecatedError, isToolsSchemaConflictError, isToolsSchemaExclusionInForce, } from "./structuredOutputPolicy.js"; @@ -531,6 +532,81 @@ export class GenerationHandler { return result; } + // Retry once without `temperature` when the model deprecated it. The + // newest Anthropic models (e.g. claude-opus-4-8 with tools + advanced + // beta features) reject `temperature` — "`temperature` is deprecated + // for this model." — in favour of reasoning-effort controls. Structured + // output is already excluded for the native anthropic surface, so this + // is the dominant failure mode for Opus there. + if ( + isTemperatureDeprecatedError(error) && + typeof options.temperature === "number" + ) { + span.setAttribute("neurolink.has_fallback", true); + span.addEvent("retry.initial_failure", { + "error.message": + error instanceof Error ? error.message : String(error), + "retry.attempt": 1, + "retry.reason": "temperature_deprecated", + }); + logger.debug( + "[GenerationHandler] temperature-deprecated error caught - retrying without temperature", + { + provider: this.providerName, + model: this.modelName, + error: error instanceof Error ? error.message : String(error), + }, + ); + const result = await withProviderRetry( + () => + this.callGenerateText( + model, + messages, + tools, + { ...options, temperature: undefined }, + shouldUseTools, + true, // mirror the initial call; the structured-output policy still applies + ), + span, + "generateText(no-temperature)", + ); + span.addEvent("retry.recovered", { + "retry.attempts": 2, + "retry.strategy": "temperature_omitted", + }); + span.setAttribute("retry.count", 1); + if (result.usage) { + span.setAttribute( + "gen_ai.usage.input_tokens", + result.usage.inputTokens || 0, + ); + span.setAttribute( + "gen_ai.usage.output_tokens", + result.usage.outputTokens || 0, + ); + const noTempCost = calculateCost( + this.providerName, + this.modelName, + { + input: result.usage.inputTokens || 0, + output: result.usage.outputTokens || 0, + total: + (result.usage.inputTokens || 0) + + (result.usage.outputTokens || 0), + }, + ); + span.setAttribute("neurolink.cost", noTempCost ?? 0); + } + if (result.finishReason) { + span.setAttribute( + "gen_ai.response.finish_reason", + result.finishReason, + ); + } + span.setStatus({ code: SpanStatusCode.OK }); + return result; + } + span.setStatus({ code: SpanStatusCode.ERROR, message: error instanceof Error ? error.message : String(error), diff --git a/src/lib/core/modules/structuredOutputPolicy.ts b/src/lib/core/modules/structuredOutputPolicy.ts index 50eb0cc2c..984926672 100644 --- a/src/lib/core/modules/structuredOutputPolicy.ts +++ b/src/lib/core/modules/structuredOutputPolicy.ts @@ -3,11 +3,17 @@ * disabled because the provider cannot combine tool calls with JSON-schema * enforcement. * - * This is a GEMINI-ONLY API limitation. Anthropic Claude — including when - * hosted on Vertex (modelName starts with "claude-") — supports tools and - * structured output simultaneously, so it must NOT be excluded. A gate keyed on - * "any Vertex model" wrongly disables structured output for Vertex+Claude (the - * primary production config) and forces fragile hand-parsed JSON. + * Two provider surfaces have this conflict: + * - Gemini (google-ai, or Vertex with a non-Claude model). + * - The native Anthropic Messages API surface (provider "anthropic"/"bedrock", + * including via a proxy/base-URL override). experimental_output silently + * drops tool_use blocks when tools are also present (finishReason=tool-calls + * but zero parsed tool calls), so structured output must be disabled there too. + * + * Vertex+Claude (provider "vertex", modelName starts with "claude-") uses a + * different transport that supports both simultaneously and must NOT be excluded — + * a gate keyed on "any Vertex model" wrongly disables it for the primary + * production config and forces fragile hand-parsed JSON. */ /** True when the provider+model is a Gemini model (the only family with the tools↔schema conflict). */ @@ -26,9 +32,21 @@ export function isGeminiProvider( return false; } +/** + * True when the provider is the native Anthropic Messages API surface + * (provider "anthropic" — including via a proxy/base-URL override — or "bedrock"). + * experimental_output + tools silently drops tool_use blocks on this surface, so + * structured output must be disabled when tools are active. Vertex+Claude is NOT + * matched here (different transport, no conflict). + */ +export function isNativeAnthropicProvider(providerName: string): boolean { + return providerName === "anthropic" || providerName === "bedrock"; +} + /** * True when structured output must be disabled for this call because tools are - * active on a Gemini provider. Mirrors the AI-SDK constraint exactly. + * active on a provider that cannot combine them (Gemini, or the native Anthropic + * Messages API surface). Mirrors the AI-SDK constraint exactly. */ export function isToolsSchemaExclusionInForce( providerName: string, @@ -37,7 +55,10 @@ export function isToolsSchemaExclusionInForce( toolCount: number, ): boolean { return ( - isGeminiProvider(providerName, modelName) && shouldUseTools && toolCount > 0 + (isGeminiProvider(providerName, modelName) || + isNativeAnthropicProvider(providerName)) && + shouldUseTools && + toolCount > 0 ); } @@ -68,3 +89,41 @@ export function isToolsSchemaConflictError(error: unknown): boolean { /response_format[^.]{0,60}(tool|function)/i.test(message) ); } + +/** + * True when a provider error indicates the request was rejected because the + * `temperature` parameter is deprecated / unsupported for the model. The newest + * Anthropic models (e.g. claude-opus-4-8, with tools + advanced beta features) + * reject `temperature` — "`temperature` is deprecated for this model." — in + * favour of reasoning-effort controls. Detect this so the call can be retried + * once without `temperature` instead of failing the turn. + */ +export function isTemperatureDeprecatedError(error: unknown): boolean { + const message = + error instanceof Error + ? error.message + : typeof error === "string" + ? error + : ""; + return /\btemperature\b[^.]{0,40}\b(deprecated|unsupported|not[\s_-]?(supported|allowed))\b/i.test( + message, + ); +} + +/** + * True when the model is known to reject the `temperature` parameter (the + * reasoning-effort Anthropic models — claude-opus-4-8 and newer — deprecate it + * in favour of effort controls). Used to omit `temperature` proactively so the + * request does not fail-then-retry on every turn: the reactive + * isTemperatureDeprecatedError() retry remains the safety net for any model not + * matched here, but a guaranteed-to-fail first request is pure wasted latency. + * + * Matches opus 4.8+ (4-8, 4-9, 4-10, …) while leaving 4.1/4.5/4.6 and Sonnet/ + * Haiku — which still accept `temperature` — untouched. + */ +export function modelDeprecatesTemperature( + modelName: string | undefined, +): boolean { + const m = (modelName ?? "").toLowerCase(); + return /opus[-_.]?4[-_.]?(?:[89]|\d{2,})\b/.test(m); +} diff --git a/src/lib/processors/document/ExcelProcessor.ts b/src/lib/processors/document/ExcelProcessor.ts index 99576aa94..03d89d4ac 100644 --- a/src/lib/processors/document/ExcelProcessor.ts +++ b/src/lib/processors/document/ExcelProcessor.ts @@ -58,7 +58,17 @@ async function loadExcelJS() { return _exceljs; } try { - _exceljs = await import(/* @vite-ignore */ "exceljs"); + const mod: unknown = await import(/* @vite-ignore */ "exceljs"); + // exceljs is a CommonJS module. Under Node ESM (and some bundlers) the + // `Workbook` constructor is exposed at runtime on the namespace's `default` + // export rather than on the namespace itself — so a bare + // `new ExcelJS.Workbook()` throws "ExcelJS.Workbook is not a constructor" + // (TS still types it as present via esModuleInterop, masking the bug). + // Normalise here so the constructor is reachable regardless of interop style. + const ns = mod as { Workbook?: unknown; default?: unknown }; + _exceljs = ( + ns.Workbook ? ns : (ns.default ?? ns) + ) as typeof import("exceljs"); return _exceljs; } catch (err) { const e = err instanceof Error ? (err as NodeJS.ErrnoException) : null; diff --git a/src/lib/providers/anthropic.ts b/src/lib/providers/anthropic.ts index 0ca154815..4b4f5b95a 100644 --- a/src/lib/providers/anthropic.ts +++ b/src/lib/providers/anthropic.ts @@ -75,6 +75,11 @@ import { } from "../utils/noOutputSentinel.js"; import { convertZodToJsonSchema } from "../utils/schemaConversion.js"; import { resolveClaudeMaxTokens } from "../utils/tokenLimits.js"; +import { + toAnthropicImageBlock, + fileToAnthropicBlock, +} from "./anthropicImageBlocks.js"; +import { modelDeprecatesTemperature } from "../core/modules/structuredOutputPolicy.js"; import { createChunkQueue, createDeferredAnalytics, @@ -313,49 +318,6 @@ const parseRateLimitHeaders = ( // Native Messages-API conversion helpers (NeuroLink/V3 shapes → Anthropic) // ─────────────────────────────────────────────────────────────────────────── -/** - * Convert an image part (data URL, bare base64, https URL, or byte array) - * into an Anthropic image block. Returns undefined for unusable inputs. - */ -const toAnthropicImageBlock = ( - data: unknown, -): Anthropic.Messages.ImageBlockParam | undefined => { - if (data instanceof Uint8Array) { - return { - type: "image", - source: { - type: "base64", - media_type: "image/png", - data: Buffer.from(data).toString("base64"), - }, - }; - } - if (typeof data !== "string" && !(data instanceof URL)) { - return undefined; - } - const str = data instanceof URL ? data.toString() : data; - const dataUrlMatch = str.match(/^data:(image\/[a-z+.-]+);base64,(.+)$/i); - if (dataUrlMatch) { - return { - type: "image", - source: { - type: "base64", - media_type: - dataUrlMatch[1] as Anthropic.Messages.Base64ImageSource["media_type"], - data: dataUrlMatch[2], - }, - }; - } - if (/^https?:\/\//i.test(str)) { - return { type: "image", source: { type: "url", url: str } }; - } - // Bare base64 payload — assume PNG (matches the OpenAI-compat client). - return { - type: "image", - source: { type: "base64", media_type: "image/png", data: str }, - }; -}; - /** * Read an Anthropic cache breakpoint from a message/part/tool carrier. * MessageBuilder marks system messages (and GenerationHandler marks the last @@ -484,6 +446,41 @@ const messagesToAnthropic = ( const cc = cacheControlOf(p); blocks.push(cc ? { ...img, cache_control: cc } : img); } + } else if (p?.type === "file") { + // AI-SDK v6 encodes images AND PDFs as `type:"file"` parts in the + // LanguageModel prompt that `doGenerate` receives. Without this + // branch the image is dropped on the tool-using generate path and + // the model never sees it ("no image detected"). + // + // Runtime guard: p comes from message parsing and may not match the + // expected shape. Verify p is an object and that mediaType, if + // present, is a string (not an object/array from a malformed part). + // Skip gracefully rather than passing a bad shape to fileToAnthropicBlock. + const isValidFilePart = + typeof p === "object" && + p !== null && + ("mediaType" in p + ? typeof (p as Record).mediaType === "string" + : true); + const block = isValidFilePart + ? fileToAnthropicBlock( + p as { mediaType?: string; data?: unknown }, + ) + : undefined; + if (block) { + const cc = cacheControlOf(p); + if (cc) { + // block is a fresh object from fileToAnthropicBlock; mutate in + // place to keep the discriminated-union type (a spread widens it + // past ContentBlockParam). + ( + block as { + cache_control?: Anthropic.Messages.CacheControlEphemeral; + } + ).cache_control = cc; + } + blocks.push(block); + } } } if (blocks.length > 0) { @@ -1454,7 +1451,9 @@ export class AnthropicProvider extends BaseProvider { messages, max_tokens: resolveClaudeMaxTokens(modelId, options.maxOutputTokens), ...(system ? { system } : {}), - ...(options.temperature !== undefined && options.temperature !== null + ...(options.temperature !== undefined && + options.temperature !== null && + !modelDeprecatesTemperature(modelId) ? { temperature: options.temperature } : {}), ...(options.topP !== undefined && options.topP !== null @@ -1783,7 +1782,9 @@ export class AnthropicProvider extends BaseProvider { max_tokens: resolveClaudeMaxTokens(modelId, options.maxTokens), stream: true, ...(payload.system ? { system: payload.system } : {}), - ...(options.temperature !== undefined && options.temperature !== null + ...(options.temperature !== undefined && + options.temperature !== null && + !modelDeprecatesTemperature(modelId) ? { temperature: options.temperature } : {}), ...(anthropicTools && anthropicTools.length > 0 diff --git a/src/lib/providers/anthropicImageBlocks.ts b/src/lib/providers/anthropicImageBlocks.ts new file mode 100644 index 000000000..14e3e2a85 --- /dev/null +++ b/src/lib/providers/anthropicImageBlocks.ts @@ -0,0 +1,261 @@ +/** + * Pure converters from multimodal content parts into native Anthropic + * Messages-API content blocks (image / document). + * + * Two input shapes reach the native Anthropic surface: + * + * 1. NeuroLink/V3 multimodal parts — `{ type: "image", image }` produced by + * the multimodal message builder (base64, data URL, https URL, or bytes). + * + * 2. AI-SDK LanguageModel prompt parts — `{ type: "file", mediaType, data }`. + * This is how `ai@6` encodes BOTH images and PDFs in the prompt that the + * provider's `doGenerate(options.prompt)` receives. Before this module the + * converter only handled `type:"image"`, so on the tool-using generate + * path (which always normalises images to `type:"file"`) the image part + * was silently dropped — the model never saw the image ("no image + * detected"). Gemini/Vertex are immune because they read `input.images` + * directly in their own builders instead of going through this conversion. + * + * Media type is taken from the AI-SDK-provided `mediaType` when present, then + * sniffed from magic bytes, and only then defaulted — a hardcoded `image/png` + * default silently corrupts JPEG/GIF/WebP uploads (the Anthropic API rejects a + * mislabeled base64 image with HTTP 400 and the image vanishes). + */ + +import type Anthropic from "@anthropic-ai/sdk"; + +// The base64 image media types the Anthropic Messages API accepts are exactly +// `Anthropic.Messages.Base64ImageSource["media_type"]` — referenced inline below +// rather than redeclared as a local alias (project rule: type aliases live in +// src/lib/types/, and this provider-internal shape does not warrant a barrel entry). +const SUPPORTED_IMAGE_MEDIA_TYPES: ReadonlySet = new Set([ + "image/jpeg", + "image/png", + "image/gif", + "image/webp", +]); + +/** Map a caller-provided MIME hint onto a supported image media type (or undefined). */ +const normalizeImageMediaType = ( + hint: string | undefined, +): Anthropic.Messages.Base64ImageSource["media_type"] | undefined => { + if (!hint) { + return undefined; + } + const h = hint.toLowerCase().split(";")[0].trim(); + if (h === "image/jpg") { + return "image/jpeg"; + } + return SUPPORTED_IMAGE_MEDIA_TYPES.has(h) + ? (h as Anthropic.Messages.Base64ImageSource["media_type"]) + : undefined; +}; + +/** + * Detect a supported image media type from a buffer's magic bytes. Returns + * undefined when the bytes are not one of the four Anthropic-supported formats. + */ +export function sniffImageMediaType( + bytes: Uint8Array, +): Anthropic.Messages.Base64ImageSource["media_type"] | undefined { + // PNG: 89 50 4E 47 0D 0A 1A 0A + if ( + bytes.length >= 8 && + bytes[0] === 0x89 && + bytes[1] === 0x50 && + bytes[2] === 0x4e && + bytes[3] === 0x47 + ) { + return "image/png"; + } + // JPEG: FF D8 FF + if ( + bytes.length >= 3 && + bytes[0] === 0xff && + bytes[1] === 0xd8 && + bytes[2] === 0xff + ) { + return "image/jpeg"; + } + // GIF: "GIF" + if ( + bytes.length >= 6 && + bytes[0] === 0x47 && + bytes[1] === 0x49 && + bytes[2] === 0x46 + ) { + return "image/gif"; + } + // WebP: "RIFF"...."WEBP" + if ( + bytes.length >= 12 && + bytes[0] === 0x52 && + bytes[1] === 0x49 && + bytes[2] === 0x46 && + bytes[3] === 0x46 && + bytes[8] === 0x57 && + bytes[9] === 0x45 && + bytes[10] === 0x42 && + bytes[11] === 0x50 + ) { + return "image/webp"; + } + return undefined; +} + +// The longest signature `sniffImageMediaType` checks is WebP — "RIFF"…"WEBP", +// which spans the first 12 decoded bytes. Base64 packs 3 bytes per 4 chars, so +// 16 chars already cover those 12 bytes; we slice 32 (a clean 4-char boundary +// that decodes to 24 bytes) for a comfortable margin over every signature. +const SNIFF_BASE64_CHARS = 32; + +/** Sniff the leading bytes of a base64 payload (best-effort, never throws). */ +const sniffBase64 = ( + b64: string, +): Anthropic.Messages.Base64ImageSource["media_type"] | undefined => { + try { + return sniffImageMediaType( + Buffer.from(b64.slice(0, SNIFF_BASE64_CHARS), "base64"), + ); + } catch { + return undefined; + } +}; + +/** + * Convert an image part (data URL, bare base64, https URL, byte array, or + * ArrayBuffer) into an Anthropic image block. Honors `mediaTypeHint` (the + * AI-SDK `mediaType`), falls back to magic-byte sniffing, then to image/png. + * Returns undefined for unusable inputs. + */ +export function toAnthropicImageBlock( + data: unknown, + mediaTypeHint?: string, +): Anthropic.Messages.ImageBlockParam | undefined { + if (data instanceof ArrayBuffer) { + return toAnthropicImageBlock(new Uint8Array(data), mediaTypeHint); + } + if (data instanceof Uint8Array) { + const media_type = + normalizeImageMediaType(mediaTypeHint) ?? + sniffImageMediaType(data) ?? + "image/png"; + return { + type: "image", + source: { + type: "base64", + media_type, + data: Buffer.from(data).toString("base64"), + }, + }; + } + if (typeof data !== "string" && !(data instanceof URL)) { + return undefined; + } + const str = data instanceof URL ? data.toString() : data; + const dataUrlMatch = str.match(/^data:(image\/[a-z0-9+.-]+);base64,(.+)$/i); + if (dataUrlMatch) { + const media_type = + normalizeImageMediaType(dataUrlMatch[1]) ?? sniffBase64(dataUrlMatch[2]); + if (!media_type) { + return undefined; + } + return { + type: "image", + source: { type: "base64", media_type, data: dataUrlMatch[2] }, + }; + } + if (/^https?:\/\//i.test(str)) { + return { type: "image", source: { type: "url", url: str } }; + } + // Bare base64 payload — prefer the hint, then sniff, then assume PNG + // (matches the OpenAI-compat client's historical default). + const media_type = + normalizeImageMediaType(mediaTypeHint) ?? sniffBase64(str) ?? "image/png"; + return { + type: "image", + source: { type: "base64", media_type, data: str }, + }; +} + +/** Convert a PDF file part into an Anthropic document block. */ +const toAnthropicPdfBlock = ( + data: unknown, +): Anthropic.Messages.DocumentBlockParam | undefined => { + if (data instanceof ArrayBuffer) { + return toAnthropicPdfBlock(new Uint8Array(data)); + } + if (data instanceof Uint8Array) { + return { + type: "document", + source: { + type: "base64", + media_type: "application/pdf", + data: Buffer.from(data).toString("base64"), + }, + }; + } + if (data instanceof URL) { + return { type: "document", source: { type: "url", url: data.toString() } }; + } + if (typeof data === "string") { + const m = data.match(/^data:application\/pdf;base64,(.+)$/i); + if (m) { + return { + type: "document", + source: { type: "base64", media_type: "application/pdf", data: m[1] }, + }; + } + if (/^https?:\/\//i.test(data)) { + return { type: "document", source: { type: "url", url: data } }; + } + return { + type: "document", + source: { type: "base64", media_type: "application/pdf", data }, + }; + } + return undefined; +}; + +/** + * Convert an AI-SDK `{ type: "file", mediaType, data }` content part into the + * matching Anthropic content block: image/* → image block (media type honored, + * not hardcoded), application/pdf → document block. Returns undefined for + * unsupported media types (so the caller simply omits them rather than 400ing). + * When `mediaType` is absent, image bytes are still salvaged via sniffing. + */ +export function fileToAnthropicBlock(part: { + mediaType?: string; + data?: unknown; +}): Anthropic.Messages.ContentBlockParam | undefined { + const data = part?.data; + if (data === undefined || data === null) { + return undefined; + } + const mediaType = + typeof part.mediaType === "string" + ? part.mediaType.toLowerCase().split(";")[0].trim() + : ""; + + if (SUPPORTED_IMAGE_MEDIA_TYPES.has(mediaType) || mediaType === "image/jpg") { + // Hint is one of the four Anthropic-supported image types (or the jpg alias). + return toAnthropicImageBlock(data, mediaType); + } + if (mediaType === "application/pdf") { + return toAnthropicPdfBlock(data); + } + // No media type, unknown media type, OR an unsupported image/* hint + // (e.g. image/svg+xml, image/bmp) — attempt magic-byte salvage so that the + // part is included only when the actual bytes sniff to a supported format, + // otherwise omit it rather than mislabeling it as image/png. + const bytes = + data instanceof ArrayBuffer + ? new Uint8Array(data) + : data instanceof Uint8Array + ? data + : undefined; + if (bytes && sniffImageMediaType(bytes)) { + return toAnthropicImageBlock(bytes); + } + return undefined; +} diff --git a/src/lib/proxy/oauthFetch.ts b/src/lib/proxy/oauthFetch.ts index af153e7a5..fc01b7d6f 100644 --- a/src/lib/proxy/oauthFetch.ts +++ b/src/lib/proxy/oauthFetch.ts @@ -20,6 +20,7 @@ import { } from "../auth/anthropicOAuth.js"; import { logger } from "../utils/logger.js"; import { createProxyFetch } from "./proxyFetch.js"; +import { relocateClientSystemIntoMessages } from "./systemRelocation.js"; // Re-export constants for consumers that previously imported them alongside // the function from `providers/anthropic.ts`. @@ -222,23 +223,21 @@ function transformOAuthJsonBody( text: "You are a Claude agent, built on Anthropic's Claude Agent SDK.", }; - // Normalise `system` to an array and APPEND billing + agent blocks. - // IMPORTANT: We append (not prepend) to preserve the client's cache - // prefix chain. Anthropic's prompt caching uses prefix matching — if - // we insert anything before the client's system blocks, we invalidate - // all cached content (tools, system prompt, message history). + // Normalise `system` to an array, then route by client type. // - // Claude Code sends a billing block with a `cch=` value that - // changes on every request. We remove any existing billing/agent - // blocks from their positions and always append our stable - // Claude-Code-shaped versions at the end. + // The subscription/OAuth path only accepts a `system` it recognises as the + // genuine Claude Code prompt. A real CC client sends its own billing + agent + // identity blocks — we keep those and just stabilise the volatile billing + // `cch`. A custom client sends its own arbitrary system prompt with NO agent + // block; left in `system` it is rejected as `rate_limit_error: "Error"`, so we + // relocate it into the message stream and send only the recognised billing + + // agent blocks as `system`. if (parsed.system) { if (typeof parsed.system === "string") { parsed.system = [{ type: "text", text: parsed.system }]; } if (Array.isArray(parsed.system)) { - // Find and remove existing billing/agent blocks from wherever - // the client placed them (typically at system[0]) + // Find existing billing/agent blocks wherever the client placed them. const billingIdx = parsed.system.findIndex( (b: { text?: string }) => typeof b.text === "string" && @@ -255,7 +254,12 @@ function transformOAuthJsonBody( ), }; - // Remove in reverse index order so indices stay valid + // A genuine Claude Code client supplies its own agent-identity block; + // a custom client does not. + const isClaudeCodeClient = agentIdx >= 0; + + // Strip billing/agent from their positions (reverse order so indices + // stay valid). What remains is the client's "extra" system content. const indicesToRemove = [billingIdx, agentIdx] .filter((i) => i >= 0) .sort((a, b) => b - a); @@ -263,8 +267,16 @@ function transformOAuthJsonBody( parsed.system.splice(idx, 1); } - // Always append deterministic billing + agent blocks at the end - parsed.system = [...parsed.system, billingBlock, agentBlock]; + if (!isClaudeCodeClient && parsed.system.length > 0) { + // Non-CC client: relocate its system into the message stream so the + // subscription/OAuth path accepts the request. + relocateClientSystemIntoMessages(parsed, parsed.system); + parsed.system = [billingBlock, agentBlock]; + } else { + // Genuine Claude Code (or no extra blocks): keep system and append the + // deterministic billing + agent blocks at the end. + parsed.system = [...parsed.system, billingBlock, agentBlock]; + } } } else { const billingBlock = { diff --git a/src/lib/proxy/proxyTracer.ts b/src/lib/proxy/proxyTracer.ts index 8b6e89b4d..e474c3cb8 100644 --- a/src/lib/proxy/proxyTracer.ts +++ b/src/lib/proxy/proxyTracer.ts @@ -34,6 +34,7 @@ import type { AccountSelectionContext, ProxyMetrics, ProxyRequestContext, + ResponseInfoContext, UpstreamAttemptContext, UsageContext, } from "../types/index.js"; @@ -312,6 +313,13 @@ class ProxyTracer { if (ctx.userAgent) { rootSpan.setAttribute("http.user_agent", ctx.userAgent); } + if (ctx.toolNames && ctx.toolNames.length > 0) { + // What the caller exposed to the model (tool catalogue for this request). + rootSpan.setAttribute( + "gen_ai.request.tool_names", + JSON.stringify(ctx.toolNames), + ); + } // Read x-neurolink-* context headers from calling SDK (e.g., Curator) const nlSessionId = incomingHeaders?.["x-neurolink-session-id"]; @@ -516,6 +524,38 @@ class ProxyTracer { } } + /** + * Record response-side details parsed from the upstream reply: the model + * that actually answered, the finish reason, and which tools the model + * invoked (tool_use blocks). Uses gen_ai.* semantic-convention attributes. + */ + setResponseInfo(ctx: ResponseInfoContext): void { + if (ctx.responseModel) { + this.rootSpan.setAttribute("gen_ai.response.model", ctx.responseModel); + } + if (ctx.finishReason) { + this.rootSpan.setAttribute( + "gen_ai.response.finish_reason", + ctx.finishReason, + ); + } + if (ctx.stopSequence) { + this.rootSpan.setAttribute( + "gen_ai.response.stop_sequence", + ctx.stopSequence, + ); + } + if (ctx.toolCalls && ctx.toolCalls.length > 0) { + this.rootSpan.setAttributes({ + "gen_ai.response.tool_calls": JSON.stringify(ctx.toolCalls), + "gen_ai.response.tool_call_count": ctx.toolCalls.length, + }); + this.rootSpan.addEvent("proxy.response.tool_calls", { + "gen_ai.response.tool_calls": JSON.stringify(ctx.toolCalls), + }); + } + } + /** Record an error on the root span. */ setError(errorType: string, errorMessage: string): void { this.rootSpan.setAttributes({ diff --git a/src/lib/proxy/systemRelocation.ts b/src/lib/proxy/systemRelocation.ts new file mode 100644 index 000000000..3febcf3c9 --- /dev/null +++ b/src/lib/proxy/systemRelocation.ts @@ -0,0 +1,54 @@ +/** + * Relocate a non-Claude-Code client's `system` blocks into the message stream. + * + * Anthropic's subscription/OAuth path rejects any `system` content it does not + * recognise as the genuine Claude Code system prompt — anti-abuse fingerprinting + * surfaced as a header-less `rate_limit_error: "Error"` (NOT a real rate limit). + * Custom clients (Curator/Tara) send their own system prompt, so we move it into + * a leading user block and keep only the recognised billing+agent blocks in + * `system`. The model still honours the instructions, and any `cache_control` is + * carried over so the prompt prefix stays cacheable. + * + * Shared by both proxy entry points (`oauthFetch.ts` and `claudeProxyRoutes.ts`) + * so the OAuth anti-abuse workaround stays in one place and can't drift between + * the two paths. + */ +export function relocateClientSystemIntoMessages( + parsed: { messages?: unknown }, + instructionBlocks: Array<{ text?: unknown; cache_control?: unknown }>, +): void { + if (instructionBlocks.length === 0) { + return; + } + const blocks = instructionBlocks.map((b) => { + const text = typeof b.text === "string" ? b.text : String(b.text ?? ""); + const out: { type: "text"; text: string; cache_control?: unknown } = { + type: "text", + text, + }; + if (b.cache_control) { + out.cache_control = b.cache_control; + } + return out; + }); + // Wrap the relocated system in an explicit delimiter so the model treats it + // as authoritative instructions, clearly separated from the user's message. + blocks[0].text = `\n${blocks[0].text}`; + const last = blocks.length - 1; + blocks[last].text = `${blocks[last].text}\n`; + + const messages = Array.isArray(parsed.messages) ? parsed.messages : []; + const first = messages[0] as { role?: string; content?: unknown } | undefined; + if (first && first.role === "user") { + const existing = + typeof first.content === "string" + ? [{ type: "text", text: first.content }] + : Array.isArray(first.content) + ? first.content + : []; + first.content = [...blocks, ...existing]; + } else { + messages.unshift({ role: "user", content: blocks }); + } + parsed.messages = messages; +} diff --git a/src/lib/server/routes/claudeProxyRoutes.ts b/src/lib/server/routes/claudeProxyRoutes.ts index 407163ffc..fc04fad78 100644 --- a/src/lib/server/routes/claudeProxyRoutes.ts +++ b/src/lib/server/routes/claudeProxyRoutes.ts @@ -46,6 +46,7 @@ import { tracers } from "../../telemetry/tracers.js"; import { withSpan } from "../../telemetry/withSpan.js"; import { ProxyTracer, recordFallbackAttempt } from "../../proxy/proxyTracer.js"; import { createRawStreamCapture } from "../../proxy/rawStreamCapture.js"; +import { relocateClientSystemIntoMessages } from "../../proxy/systemRelocation.js"; import { logBodyCapture, logRequest, @@ -91,6 +92,7 @@ import type { PreparedAnthropicAccountAttempt, ProxyBodyCaptureLogger, ProxyPassthroughAccount, + ResponseInfoContext, RouteGroup, RuntimeAccountState, ServerContext, @@ -502,6 +504,78 @@ async function maybeRefreshClaudeSnapshot( return next; } +/** + * Parse response-side details (model, finish reason, invoked tools) from a + * non-streaming Anthropic reply so they can be recorded on the trace span. + */ +function extractResponseInfo(responseJson: unknown): ResponseInfoContext { + const info: ResponseInfoContext = {}; + if (!responseJson || typeof responseJson !== "object") { + return info; + } + const r = responseJson as { + model?: unknown; + stop_reason?: unknown; + stop_sequence?: unknown; + content?: unknown; + }; + if (typeof r.model === "string") { + info.responseModel = r.model; + } + if (typeof r.stop_reason === "string") { + info.finishReason = r.stop_reason; + } + if (typeof r.stop_sequence === "string") { + info.stopSequence = r.stop_sequence; + } + if (Array.isArray(r.content)) { + const toolCalls = r.content + .filter( + (b): b is { type: string; name?: unknown } => + !!b && + typeof b === "object" && + (b as { type?: unknown }).type === "tool_use", + ) + .map((b) => String((b as { name?: unknown }).name ?? "")) + .filter((n) => n.length > 0); + if (toolCalls.length > 0) { + info.toolCalls = toolCalls; + } + } + return info; +} + +/** + * Build response-info from streaming telemetry (responding model, finish + * reason, and the tools the model invoked via tool_use content blocks) so the + * streaming path records the same gen_ai.response.* attributes as non-streaming. + */ +function responseInfoFromStream(data: { + model?: string; + stopReason?: string | null; + stopSequence?: string | null; + contentBlocks?: Array<{ type?: string; toolName?: string }>; +}): ResponseInfoContext { + const info: ResponseInfoContext = {}; + if (data.model) { + info.responseModel = data.model; + } + if (data.stopReason) { + info.finishReason = data.stopReason; + } + if (data.stopSequence) { + info.stopSequence = data.stopSequence; + } + const toolCalls = (data.contentBlocks ?? []) + .filter((b) => b.type === "tool_use") + .map((b) => String(b.toolName ?? "")) + .filter((n) => n.length > 0); + if (toolCalls.length > 0) { + info.toolCalls = toolCalls; + } + return info; +} + /** * Polyfill the request body for OAuth accounts. * Claude Code injects a billing header, agent block, and metadata.user_id @@ -528,25 +602,21 @@ function polyfillOAuthBody( "You are a Claude agent, built on Anthropic's Claude Agent SDK.", }; - // Normalise system to array and APPEND billing + agent blocks. - // IMPORTANT: We append (not prepend) to preserve the client's cache - // prefix chain. Anthropic's prompt caching uses prefix matching — if we - // insert anything before the client's system blocks, we invalidate all - // cached content (tools, system prompt, message history). + // Normalise system to an array, then route by client type. // - // Claude Code sends a billing block with a `cch=` value that changes - // on every request. We fix this by: - // 1. Removing the client's billing block from its current position - // 2. Stabilizing it while keeping the official Claude Code shape - // 3. Appending it at the END so the cacheable system blocks stay - // at the front of the prefix chain + // The subscription/OAuth path only accepts a `system` it recognises as the + // genuine Claude Code prompt. A real CC client sends its own billing + agent + // identity blocks alongside the canonical prompt — we keep those in place and + // just stabilise the volatile billing `cch`. A custom client (Curator) sends + // its own arbitrary system prompt with NO agent block; left in `system` it is + // rejected as `rate_limit_error: "Error"`, so we relocate it into the message + // stream and send only the recognised billing + agent blocks as `system`. if (parsed.system) { if (typeof parsed.system === "string") { parsed.system = [{ type: "text", text: parsed.system }]; } if (Array.isArray(parsed.system)) { - // Find and remove existing billing/agent blocks from wherever - // the client placed them (typically at system[0]) + // Find existing billing/agent blocks wherever the client placed them. const billingIdx = parsed.system.findIndex( (b: { text?: string }) => typeof b.text === "string" && @@ -563,7 +633,12 @@ function polyfillOAuthBody( ), }; - // Remove in reverse index order so indices stay valid + // A genuine Claude Code client supplies its own agent-identity block; + // a custom client (Curator) does not. + const isClaudeCodeClient = agentIdx >= 0; + + // Strip billing/agent from their positions (reverse order so indices + // stay valid). What remains is the client's "extra" system content. const indicesToRemove = [billingIdx, agentIdx] .filter((i) => i >= 0) .sort((a, b) => b - a); @@ -571,10 +646,17 @@ function polyfillOAuthBody( parsed.system.splice(idx, 1); } - // Always append a deterministic billing block at the end. - // If the client sent one, we stripped its dynamic cch= and use - // our stable version instead. If not, we add ours. - parsed.system = [...parsed.system, billingBlock, agentBlock]; + if (!isClaudeCodeClient && parsed.system.length > 0) { + // Non-CC client: relocate its system into the message stream so the + // subscription/OAuth path accepts the request, then send only the + // recognised billing + agent blocks as `system`. + relocateClientSystemIntoMessages(parsed, parsed.system); + parsed.system = [billingBlock, agentBlock]; + } else { + // Genuine Claude Code (or no extra blocks): keep the recognised system + // and append the deterministic billing + agent blocks at the end. + parsed.system = [...parsed.system, billingBlock, agentBlock]; + } } } else { const billingBlock = { @@ -969,6 +1051,7 @@ async function handleClaudePassthroughStreamResponse(args: { cacheReadTokens: data.usage.cacheReadInputTokens, }); capturedTracer.logStreamEvents(data.events); + capturedTracer.setResponseInfo(responseInfoFromStream(data)); const rateLimit5h = parseFloat( capturedResponse.headers.get( @@ -1209,6 +1292,7 @@ async function handleClaudePassthroughJsonResponse(args: { tracer.setUsage(usageWithRates); } } + tracer.setResponseInfo(extractResponseInfo(responseJson)); tracer.recordMetrics(); const responseJsonStr = JSON.stringify(responseJson); tracer.recordBodySizes(bodyStr.length, responseJsonStr.length); @@ -2310,6 +2394,7 @@ function attachAnthropicSuccessStreamTelemetry(args: { cacheReadTokens: data.usage.cacheReadInputTokens, }); capturedTracer.logStreamEvents(data.events); + capturedTracer.setResponseInfo(responseInfoFromStream(data)); const rateLimit5h = parseFloat( capturedResponse.headers.get( "anthropic-ratelimit-unified-5h-utilization", @@ -2607,6 +2692,7 @@ async function handleAnthropicJsonSuccessResponse(args: { tracer.setUsage(usageWithRates); } } + tracer.setResponseInfo(extractResponseInfo(responseJson)); tracer.recordMetrics(); const responseJsonStr = JSON.stringify(responseJson); tracer.recordBodySizes(finalBodyStr.length, responseJsonStr.length); @@ -2770,6 +2856,7 @@ async function handleAnthropicSuccessfulRetryResponse(args: { cacheReadTokens: data.usage.cacheReadInputTokens, }); capturedTracer.logStreamEvents(data.events); + capturedTracer.setResponseInfo(responseInfoFromStream(data)); capturedTracer.logUpstreamResponseHeaders( Object.fromEntries([...capturedRetryResp.headers.entries()]), ); @@ -2881,6 +2968,7 @@ async function handleAnthropicSuccessfulRetryResponse(args: { cacheReadTokens: retryUsage.cache_read_input_tokens ?? 0, }); } + tracer.setResponseInfo(extractResponseInfo(retryJson)); tracer.recordMetrics(); const retryJsonStr = JSON.stringify(retryJson); tracer.recordBodySizes(finalBodyStr.length, retryJsonStr.length); @@ -3100,7 +3188,16 @@ async function handleAnthropicAuthRetry(args: { ); recordAttemptError(account.label, account.type, retryStatus); - if (retryStatus === 429) { + // A real rate-limit 429 rotates accounts. But an anti-abuse / construction + // 429 (no rate-limit headers, body "Error") is NOT a real rate limit: + // advancing the primary or rotating cannot help and only burns quota. Skip + // the rotate path for it so it falls through to the terminal error return + // below, surfacing the truthful upstream error — mirroring the initial + // fetch path's fast-fail (isAntiAbuseConstruction429). + if ( + retryStatus === 429 && + !isAntiAbuseConstruction429(retryRespHeaders, retryBody) + ) { currentSawRateLimit = true; advancePrimaryIfCurrent( account.key, @@ -3570,6 +3667,15 @@ function createClaudeRequestRuntimeContext(args: { model: body.model, stream: body.stream ?? false, toolCount: Array.isArray(body.tools) ? body.tools.length : 0, + toolNames: Array.isArray(body.tools) + ? body.tools + .map((t) => + t && typeof t === "object" && "name" in t + ? String((t as { name?: unknown }).name ?? "") + : "", + ) + .filter((n): n is string => n.length > 0) + : undefined, sessionId: ctx.headers["x-neurolink-session-id"] ?? ctx.headers["x-claude-code-session-id"] ?? @@ -3935,6 +4041,32 @@ async function prepareAnthropicAccountAttempt(args: { }; } +/** + * Detect Anthropic's anti-abuse / request-construction 429. + * + * The subscription/OAuth path rejects requests it does not recognise as genuine + * Claude Code traffic with a 429 `rate_limit_error` whose message is literally + * "Error" and which carries NONE of the real rate-limit headers (no retry-after, + * no anthropic-ratelimit-*). This is NOT a capacity limit — retrying or rotating + * accounts cannot fix it and only burns quota, so the caller must fail fast and + * surface the truthful upstream error instead of "all accounts rate-limited". + */ +function isAntiAbuseConstruction429( + headers: Record, + body: string, +): boolean { + const hasRetryAfter = !!headers["retry-after"]; + const hasRateLimitHeaders = Object.keys(headers).some((k) => + k.toLowerCase().startsWith("anthropic-ratelimit-"), + ); + if (hasRetryAfter || hasRateLimitHeaders) { + return false; + } + return ( + body.includes("rate_limit_error") && /"message"\s*:\s*"Error"/.test(body) + ); +} + async function fetchAnthropicAccountResponse(args: { url: string; headers: Record; @@ -4044,6 +4176,35 @@ async function fetchAnthropicAccountResponse(args: { responseStatus: 429, durationMs: Date.now() - fetchStartMs, }); + // Anti-abuse / request-construction 429 (no rate-limit headers, body + // "Error"): rotating accounts cannot help and only burns quota. Fail fast + // and surface the truthful upstream error instead of the misleading + // "all accounts rate-limited" after 44 wasted attempts. + if (isAntiAbuseConstruction429(errRespHeaders, String(lastError))) { + logger.always( + `[proxy] ← 429 account=${account.label} anti-abuse/construction rejection (no ratelimit headers, body="Error") — NOT a real rate limit; returning upstream error without rotating`, + ); + logAttempt(429, "construction_rejection", String(lastError)); + tracer?.setError( + "construction_rejection", + String(lastError).slice(0, 500), + ); + currentUpstreamSpan?.end(); + const passthrough = new Response(String(lastError), { + status: 429, + headers: { + "content-type": errRespHeaders["content-type"] ?? "application/json", + }, + }); + return { + continueLoop: false, + response: passthrough, + lastError, + sawRateLimit, + sawNetworkError, + upstreamSpan: undefined, + }; + } logger.always( `[proxy] ← 429 account=${account.label} retry-after=${retryAfterMs}ms (upstream) ratelimit-status=${errRespHeaders["anthropic-ratelimit-unified-status"] ?? "unknown"}`, ); diff --git a/src/lib/types/proxy.ts b/src/lib/types/proxy.ts index f09b3e6f2..005e0ff6f 100644 --- a/src/lib/types/proxy.ts +++ b/src/lib/types/proxy.ts @@ -937,11 +937,22 @@ export type ProxyRequestContext = { model: string; stream: boolean; toolCount: number; + /** Names of the tools advertised in the request (what the caller exposed). */ + toolNames?: string[]; sessionId?: string; userAgent?: string; clientApp?: string; }; +/** Response-side details parsed from the upstream reply (model, finish, tools). */ +export type ResponseInfoContext = { + responseModel?: string; + finishReason?: string; + stopSequence?: string; + /** Names of the tools the model actually invoked (tool_use blocks). */ + toolCalls?: string[]; +}; + /** Context recorded when an account is selected for a proxy request. */ export type AccountSelectionContext = { strategy: string; diff --git a/test/continuous-test-suite-anthropic-multimodal.ts b/test/continuous-test-suite-anthropic-multimodal.ts new file mode 100644 index 000000000..abf3f9527 --- /dev/null +++ b/test/continuous-test-suite-anthropic-multimodal.ts @@ -0,0 +1,208 @@ +#!/usr/bin/env tsx +/** + * Continuous Test Suite: native-Anthropic multimodal block conversion (pure, + * no API). + * + * Regression guard for the vision defect on the native Anthropic surface: AI-SDK + * v6 encodes images AND PDFs in the LanguageModel prompt as `type:"file"` parts + * (`{ type:"file", mediaType, data }`). The provider's `doGenerate(options.prompt)` + * converts that prompt with messagesToAnthropic, which only handled + * text/image/image_url — so the image was silently dropped on the tool-using + * generate path and the model answered "no image detected". These tests lock in: + * + * - `fileToAnthropicBlock` turns an image file part into an Anthropic image + * block with the CORRECT media type (honoring the AI-SDK `mediaType`, not a + * hardcoded image/png that 400s real JPEG/GIF/WebP uploads). + * - PDF file parts become document blocks. + * - Magic-byte sniffing recovers the media type when no hint is present. + * + * Run: npx tsx test/continuous-test-suite-anthropic-multimodal.ts + */ + +import { defineSuite, assertEqual } from "./helpers/harness.js"; +import { + sniffImageMediaType, + toAnthropicImageBlock, + fileToAnthropicBlock, +} from "../src/lib/providers/anthropicImageBlocks.js"; + +const { test, runSuite } = defineSuite( + "Native-Anthropic multimodal conversion", +); + +// Minimal valid magic-byte headers for each format. +const PNG = Buffer.from([ + 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d, +]); +const JPEG = Buffer.from([0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10, 0x4a, 0x46]); +const GIF = Buffer.from([0x47, 0x49, 0x46, 0x38, 0x39, 0x61]); // GIF89a +const WEBP = Buffer.from([ + 0x52, 0x49, 0x46, 0x46, 0x1a, 0x00, 0x00, 0x00, 0x57, 0x45, 0x42, 0x50, +]); +const PDF = Buffer.from([0x25, 0x50, 0x44, 0x46, 0x2d, 0x31, 0x2e, 0x34]); // %PDF-1.4 + +const isImageBlock = ( + b: unknown, +): b is { + type: "image"; + source: { type: string; media_type?: string; data?: string; url?: string }; +} => !!b && (b as { type?: string }).type === "image"; + +await test("sniffImageMediaType: detects png/jpeg/gif/webp from magic bytes", () => { + assertEqual(sniffImageMediaType(PNG), "image/png", "PNG magic → image/png"); + assertEqual( + sniffImageMediaType(JPEG), + "image/jpeg", + "JPEG magic → image/jpeg", + ); + assertEqual(sniffImageMediaType(GIF), "image/gif", "GIF magic → image/gif"); + assertEqual( + sniffImageMediaType(WEBP), + "image/webp", + "WEBP magic → image/webp", + ); + assertEqual( + sniffImageMediaType(PDF), + undefined, + "non-image bytes → undefined", + ); +}); + +await test("toAnthropicImageBlock: byte array honors mediaType hint", () => { + const block = toAnthropicImageBlock(JPEG, "image/jpeg"); + assertEqual(isImageBlock(block), true, "produces image block"); + assertEqual(block?.source.type, "base64", "base64 source"); + assertEqual( + (block?.source as { media_type?: string }).media_type, + "image/jpeg", + "media_type from hint (NOT hardcoded png)", + ); + assertEqual( + (block?.source as { data?: string }).data, + JPEG.toString("base64"), + "base64 payload preserved", + ); +}); + +await test("toAnthropicImageBlock: byte array with no hint sniffs the type", () => { + const block = toAnthropicImageBlock(GIF); + assertEqual( + (block?.source as { media_type?: string }).media_type, + "image/gif", + "GIF bytes sniffed to image/gif (not defaulted to png)", + ); +}); + +await test("toAnthropicImageBlock: https URL → url source", () => { + const block = toAnthropicImageBlock("https://example.com/cat.png"); + assertEqual(block?.source.type, "url", "url source type"); + assertEqual( + (block?.source as { url?: string }).url, + "https://example.com/cat.png", + "url preserved", + ); +}); + +await test("toAnthropicImageBlock: data URL parses media type + payload", () => { + const b64 = PNG.toString("base64"); + const block = toAnthropicImageBlock(`data:image/png;base64,${b64}`); + assertEqual( + (block?.source as { media_type?: string }).media_type, + "image/png", + "data URL media type", + ); + assertEqual( + (block?.source as { data?: string }).data, + b64, + "data URL payload", + ); +}); + +await test("toAnthropicImageBlock: bare base64 WebP without hint sniffs via the 32-char slice", () => { + // Guards SNIFF_BASE64_CHARS: sniffBase64 decodes only the first 32 base64 + // chars (= 24 bytes). WebP is the longest signature (RIFF…WEBP spans the + // first 12 bytes), so the filler below pushes the base64 well past 32 chars, + // forcing the slice to truncate — yet the 12-byte signature stays within the + // decoded 24 bytes, so detection must still resolve to image/webp. + const webpLong = Buffer.concat([WEBP, Buffer.alloc(48, 0x20)]); + const b64 = webpLong.toString("base64"); + assertEqual(b64.length > 32, true, "payload longer than the sniff slice"); + const block = toAnthropicImageBlock(b64); + assertEqual(isImageBlock(block), true, "bare base64 → image block"); + assertEqual( + (block?.source as { media_type?: string }).media_type, + "image/webp", + "bare base64 WebP sniffed to image/webp (slice reaches the 12th byte)", + ); + assertEqual( + (block?.source as { data?: string }).data, + b64, + "full base64 payload preserved (only sniffing uses the slice)", + ); +}); + +await test("fileToAnthropicBlock: AI-SDK image file part → image block (THE fix)", () => { + // Exactly the shape ai@6 puts in the doGenerate prompt for an uploaded PNG. + const block = fileToAnthropicBlock({ mediaType: "image/png", data: PNG }); + assertEqual(isImageBlock(block), true, "image file part → image block"); + assertEqual( + (block as { source?: { media_type?: string } })?.source?.media_type, + "image/png", + "media type carried through", + ); + assertEqual( + (block as { source?: { data?: string } })?.source?.data, + PNG.toString("base64"), + "image bytes carried through (image is no longer dropped)", + ); +}); + +await test("fileToAnthropicBlock: jpeg file part keeps image/jpeg", () => { + const block = fileToAnthropicBlock({ mediaType: "image/jpeg", data: JPEG }); + assertEqual( + (block as { source?: { media_type?: string } })?.source?.media_type, + "image/jpeg", + "jpeg not mislabeled as png", + ); +}); + +await test("fileToAnthropicBlock: PDF file part → document block", () => { + const block = fileToAnthropicBlock({ + mediaType: "application/pdf", + data: PDF, + }); + assertEqual( + (block as { type?: string })?.type, + "document", + "pdf → document block", + ); + assertEqual( + (block as { source?: { media_type?: string } })?.source?.media_type, + "application/pdf", + "document media type", + ); +}); + +await test("fileToAnthropicBlock: missing mediaType still salvages image bytes", () => { + const block = fileToAnthropicBlock({ data: WEBP }); + assertEqual( + (block as { source?: { media_type?: string } })?.source?.media_type, + "image/webp", + "sniffed image/webp from bytes with no hint", + ); +}); + +await test("fileToAnthropicBlock: unsupported non-image file → undefined (skipped, not 400)", () => { + assertEqual( + fileToAnthropicBlock({ mediaType: "text/plain", data: "hello" }), + undefined, + "text file part is omitted, not forced into an image block", + ); + assertEqual( + fileToAnthropicBlock({ mediaType: "image/png", data: null }), + undefined, + "null data → undefined", + ); +}); + +await runSuite(); diff --git a/test/continuous-test-suite-anthropic-tools-policy.ts b/test/continuous-test-suite-anthropic-tools-policy.ts new file mode 100644 index 000000000..55001f465 --- /dev/null +++ b/test/continuous-test-suite-anthropic-tools-policy.ts @@ -0,0 +1,206 @@ +#!/usr/bin/env tsx +/** + * Continuous Test Suite: native-Anthropic tools policy (pure, no API). + * + * Two related provider-quirk gates for the native Anthropic Messages API + * surface (provider "anthropic"/"bedrock", incl. via a proxy/base-URL override): + * + * 1. structured-output ↔ tools — experimental_output (JSON-schema enforcement) + * silently drops tool_use blocks when combined with tools on this surface + * (finishReason=tool-calls but zero parsed tool calls). isToolsSchema + * ExclusionInForce() must therefore disable structured output for it, + * mirroring the Gemini gate — while leaving Vertex+Claude untouched (a + * different transport that supports both simultaneously). + * + * 2. temperature deprecation — the newest models (e.g. claude-opus-4-8 with + * tools + advanced betas) reject `temperature` ("`temperature` is deprecated + * for this model.") in favour of reasoning-effort controls. isTemperature + * DeprecatedError() detects this so the call can be retried without it. + * + * Run: npx tsx test/continuous-test-suite-anthropic-tools-policy.ts + */ + +import { defineSuite, assertEqual } from "./helpers/harness.js"; +import { + isNativeAnthropicProvider, + isGeminiProvider, + isToolsSchemaExclusionInForce, + isTemperatureDeprecatedError, + modelDeprecatesTemperature, +} from "../src/lib/core/modules/structuredOutputPolicy.js"; + +const { test, runSuite } = defineSuite("Native-Anthropic tools policy"); + +await test("isNativeAnthropicProvider: anthropic + bedrock only", () => { + assertEqual(isNativeAnthropicProvider("anthropic"), true, "anthropic → true"); + assertEqual(isNativeAnthropicProvider("bedrock"), true, "bedrock → true"); + assertEqual(isNativeAnthropicProvider("vertex"), false, "vertex → false"); + assertEqual( + isNativeAnthropicProvider("google-ai"), + false, + "google-ai → false", + ); +}); + +await test("structured-output exclusion now covers native anthropic + tools", () => { + assertEqual( + isToolsSchemaExclusionInForce("anthropic", "claude-opus-4-8", true, 5), + true, + "anthropic + tools → exclude structured output", + ); + assertEqual( + isToolsSchemaExclusionInForce("bedrock", "claude-sonnet-4-6", true, 3), + true, + "bedrock + tools → exclude structured output", + ); + // Reviewer-requested assertions (exact inputs from the review comment): + assertEqual( + isToolsSchemaExclusionInForce("anthropic", "claude-3-sonnet", true, 5), + true, + "anthropic + claude-3-sonnet + 5 tools → exclude structured output", + ); + assertEqual( + isToolsSchemaExclusionInForce( + "bedrock", + "anthropic.claude-3-sonnet", + true, + 5, + ), + true, + "bedrock + anthropic.claude-3-sonnet (Bedrock ARN format) + 5 tools → exclude structured output", + ); +}); + +await test("Vertex+Claude is intentionally NOT excluded (different transport)", () => { + assertEqual( + isGeminiProvider("vertex", "claude-sonnet-4-6"), + false, + "vertex+claude is not a gemini provider", + ); + assertEqual( + isToolsSchemaExclusionInForce("vertex", "claude-sonnet-4-6", true, 9), + false, + "vertex+claude keeps strict structured output", + ); +}); + +await test("Gemini gate still in force", () => { + assertEqual( + isToolsSchemaExclusionInForce("vertex", "gemini-2.5-flash", true, 2), + true, + "vertex+gemini + tools → exclude", + ); + assertEqual( + isToolsSchemaExclusionInForce("google-ai", "gemini-2.5-pro", true, 1), + true, + "google-ai + tools → exclude", + ); +}); + +await test("exclusion requires tools to actually be active", () => { + assertEqual( + isToolsSchemaExclusionInForce("anthropic", "claude-opus-4-8", true, 0), + false, + "no tools → no exclusion", + ); + assertEqual( + isToolsSchemaExclusionInForce("anthropic", "claude-opus-4-8", false, 5), + false, + "tools disabled → no exclusion", + ); +}); + +await test("isTemperatureDeprecatedError matches the real Anthropic 400", () => { + assertEqual( + isTemperatureDeprecatedError( + new Error("`temperature` is deprecated for this model."), + ), + true, + "production message → true", + ); + assertEqual( + isTemperatureDeprecatedError("temperature is not supported"), + true, + "not-supported phrasing → true", + ); + assertEqual( + isTemperatureDeprecatedError( + new Error("temperature parameter not allowed"), + ), + true, + "not-allowed phrasing → true", + ); +}); + +await test("isTemperatureDeprecatedError ignores unrelated errors", () => { + assertEqual( + isTemperatureDeprecatedError(new Error("rate limit exceeded (429)")), + false, + "rate limit → false", + ); + assertEqual( + isTemperatureDeprecatedError(new Error("max_tokens is too large")), + false, + "unrelated 400 → false", + ); + assertEqual(isTemperatureDeprecatedError(""), false, "empty → false"); +}); + +await test("modelDeprecatesTemperature: opus 4.8+ only (proactive omission)", () => { + // Reasoning-effort models that reject `temperature` → omit it proactively. + assertEqual( + modelDeprecatesTemperature("claude-opus-4-8"), + true, + "opus-4-8 → true", + ); + assertEqual( + modelDeprecatesTemperature("claude-opus-4-8-20251101"), + true, + "opus-4-8 dated id → true", + ); + assertEqual( + modelDeprecatesTemperature("claude-opus-4-9"), + true, + "opus-4-9 → true", + ); + assertEqual( + modelDeprecatesTemperature("claude-opus-4-10"), + true, + "opus-4-10 → true", + ); +}); + +await test("modelDeprecatesTemperature: older/other models keep temperature", () => { + assertEqual( + modelDeprecatesTemperature("claude-opus-4-1"), + false, + "opus-4-1 → false", + ); + assertEqual( + modelDeprecatesTemperature("claude-opus-4-6"), + false, + "opus-4-6 → false", + ); + assertEqual( + modelDeprecatesTemperature("claude-opus-4-7"), + false, + "opus-4-7 (boundary one below the 4-8 cutoff) → false", + ); + assertEqual( + modelDeprecatesTemperature("claude-sonnet-4-6"), + false, + "sonnet-4-6 → false", + ); + assertEqual( + modelDeprecatesTemperature("claude-haiku-4-5"), + false, + "haiku-4-5 → false", + ); + assertEqual( + modelDeprecatesTemperature(undefined), + false, + "undefined → false", + ); +}); + +await runSuite(); diff --git a/test/continuous-test-suite-excel-interop.ts b/test/continuous-test-suite-excel-interop.ts new file mode 100644 index 000000000..1381b233c --- /dev/null +++ b/test/continuous-test-suite-excel-interop.ts @@ -0,0 +1,71 @@ +#!/usr/bin/env tsx +/** + * Continuous Test Suite: ExcelProcessor CJS/ESM interop (no API). + * + * Regression guard for "ExcelJS.Workbook is not a constructor": exceljs is a + * CommonJS module, and under Node ESM `await import("exceljs")` exposes the + * `Workbook` constructor on the namespace's `default` export, not the namespace + * itself. A bare `new ExcelJS.Workbook()` therefore throws and every .xlsx + * upload is rejected with a generic "couldn't process this file" placeholder. + * loadExcelJS() now normalises the interop; this proves a real .xlsx round-trips + * through ExcelProcessor.processFile(). + * + * Run: npx tsx test/continuous-test-suite-excel-interop.ts + */ + +import { defineSuite, assertEqual } from "./helpers/harness.js"; +import { excelProcessor } from "../src/lib/processors/document/ExcelProcessor.js"; + +const { test, runSuite } = defineSuite("Excel processor interop"); + +/** Build a minimal valid .xlsx in memory via exceljs (same CJS module). */ +async function makeXlsx(): Promise { + const mod = (await import("exceljs")) as unknown as { + Workbook?: new () => unknown; + default?: { Workbook: new () => unknown }; + }; + const Workbook = mod.Workbook ?? mod.default?.Workbook; + if (!Workbook) { + throw new Error("exceljs Workbook constructor unresolved in test harness"); + } + const wb = new Workbook() as { + addWorksheet: (n: string) => { addRow: (r: unknown[]) => void }; + xlsx: { writeBuffer: () => Promise }; + }; + const ws = wb.addWorksheet("Sheet1"); + ws.addRow(["quarter", "units"]); + ws.addRow(["q1", 50]); + ws.addRow(["q2", 60]); + return Buffer.from(await wb.xlsx.writeBuffer()); +} + +await test("ExcelProcessor.processFile succeeds on a real .xlsx (interop fix)", async () => { + const buffer = await makeXlsx(); + const result = await excelProcessor.processFile({ + id: "t.xlsx", + name: "t.xlsx", + mimetype: + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + size: buffer.length, + buffer, + }); + assertEqual( + result.success, + true, + `processFile must succeed (was: ${ + result.success ? "ok" : result.error?.technicalDetails + })`, + ); + assertEqual( + (result.data?.worksheets?.length ?? 0) > 0, + true, + "extracts at least one worksheet", + ); + assertEqual( + (result.data?.totalRows ?? 0) >= 3, + true, + "reads the data rows (header + 2)", + ); +}); + +await runSuite();