From 0039e3da14572633440e9d804fe49adda4e40401 Mon Sep 17 00:00:00 2001 From: Sachin Sharma Date: Mon, 10 Aug 2026 01:33:24 +0530 Subject: [PATCH] fix(multimodal): deliver file content to the model instead of describing it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The end-to-end format suite attaches a real file hiding an un-guessable code and asks the model to read it back. On this branch's parent it passed 21 of 57 runnable formats. Almost every failure traced to one cause, with four independent defects underneath it. Files over `SIZE_TIER_THRESHOLDS.TINY_MAX` (10 KB) were registered as lazy references and a truncated slice of their raw bytes injected in place of the file. That is faithful for a file that IS text and misleading for everything else — a .rtf sliced raw is RTF control words, .bz2/.xz/.zst are compressed bytes, and the branch that extracts any of them is skipped entirely on that path. Eagerness is now decided by whether a text preview can stand in for the file, rather than by listing types one incident at a time. Plain text and CSV keep the lazy path, which is what it was built for: a truncated sample of a large CSV is a faithful sample, and the file tools can read the rest on demand. Video shows most clearly that this was a delivery problem and not a format one. Keyframe extraction is identical across containers — three frames, ~38 KB each, for mp4/wmv/flv/mpg/m2ts alike — and a frame from any of them shows the test code perfectly legibly. Only mp4 answered, because Gemini decodes an mp4 natively and never needed the frames; the other four depended on frames that were thrown away before extraction ran. Attaching audio produced a metadata block — duration, codec, sample rate — and no audio bytes, while Gemini has accepted inline audio throughout. Three layers each discarded it independently: detection kept only the summary, `buildMultimodalMessagesArray` treated an audio-only turn as text-only, and Gemini's native request shape is assembled in the Vertex client rather than taken from the AI SDK's parts, so audio had to be added there as inlineData the way PDFs and images already were. The gap survived because that metadata block answers exactly the questions a test is most tempted to ask. "How long is this?" and "what sample rate?" both succeed with nothing attached, so the existing audio tests — which assert on duration — passed throughout. Containers Gemini rejects (WMA, CAF, AU, WavPack) are transcoded to MP3; one that cannot be converted is skipped rather than sent, since an unsupported inlineData mimeType fails the whole request while the summary still stands. Providers not known to accept audio keep the previous behaviour. Content-based strategies identify a type from magic bytes and report no extension, which is honest for the type and wrong for routing: several processors are chosen by extension *after* the type is settled, because one routing type covers several formats. With a null extension those branches were unreachable, so an .rtf reached the Word processor and reported "Could not extract content" while its own RtfProcessor extracts it perfectly. The name was dropped too, and TAR has no magic bytes at offset 0 — its "ustar" marker sits at byte 257 — so a .tar arrived unidentifiable. Both are now filled in only when a strategy left them empty, so content still wins over a lying filename. Every other format returned a listing of names and sizes. TAR and GZ now capture member text (a tar entry's bytes stream past exactly once, so they are taken there), BZ2 is no longer refused outright, and XZ and ZST gain extractors — the registry has claimed both as archives since the format work landed. Node ships zstd from v22.15/23; bzip2 and xz use the system tools when present, the same soft-dependency arrangement this codebase has with ffmpeg, and their absence is reported as unsupported on this machine rather than in principle. JPEG 2000 was claimed by the registry with no signature to match it; both the JP2 container and a bare codestream are now recognised, the container check placed before ICO because their leading bytes differ by one. .odp reached the OOXML reader, which finds no ppt/slides parts in an OpenDocument package. It had been passing for the wrong reason — as a lazy reference it was described as a ZIP whose content.xml text happened to satisfy the assertion. Verified live against Vertex: 57 of 57 runnable formats now return their hidden code, from 21. The 9 remaining skips are formats this machine cannot encode (amr, ape, mid, ogv, doc, xls, ppt, 7z, rar) and name the reason. The canonical multimodal suite still passes 18/18. --- src/lib/adapters/audioFormatSupport.ts | 235 +++++++++ .../processors/archive/ArchiveProcessor.ts | 444 ++++++++++++++-- src/lib/providers/googleAiStudio/client.ts | 79 ++- .../providers/googleNativeGemini3/utils.ts | 72 +++ src/lib/providers/googleVertex/client.ts | 16 + src/lib/types/file.ts | 43 ++ src/lib/types/generate.ts | 17 +- src/lib/types/processor.ts | 18 + src/lib/types/providers.ts | 7 + src/lib/utils/fileDetector.ts | 165 +++++- src/lib/utils/imageProcessor.ts | 36 ++ src/lib/utils/messageBuilder.ts | 480 ++++++++++++++++-- test/continuous-test-suite-audio.ts | 319 ++++++++++++ test/continuous-test-suite-file-formats.ts | 97 +++- test/continuous-test-suite-multimodal-sdk.ts | 147 +++++- 15 files changed, 2053 insertions(+), 122 deletions(-) create mode 100644 src/lib/adapters/audioFormatSupport.ts diff --git a/src/lib/adapters/audioFormatSupport.ts b/src/lib/adapters/audioFormatSupport.ts new file mode 100644 index 000000000..74727b5c6 --- /dev/null +++ b/src/lib/adapters/audioFormatSupport.ts @@ -0,0 +1,235 @@ +/** + * Native audio delivery to providers that can listen. + * + * ## The gap this closes + * + * Until this module existed, attaching an audio file produced a message + * containing only a metadata block: + * + * ## Audio File: "recording.mp3" + * Duration: 19s | Codec: MPEG 2 Layer 3 | Bitrate: 32 kbps | + * Sample Rate: 22050 Hz | Channels: 1 (Mono) + * + * No audio bytes were ever handed to the provider. Every question about what + * the recording *says* — transcribe this, who is speaking, what was agreed — + * was answered from a description of the file, and Gemini has accepted inline + * audio the whole time. + * + * The failure was invisible for an instructive reason: that metadata block + * answers precisely the questions a test is most tempted to ask. "How long is + * this audio?" and "what sample rate is it?" both succeed with no audio + * attached, so a suite built on them reports working audio support. It took an + * end-to-end test asking for a spoken word to expose it. + * + * ## Provider scope + * + * Deliberately a capability map rather than "send audio to everyone". A + * provider that cannot accept an audio part responds with an opaque HTTP 400, + * which is worse than the metadata summary it would otherwise have received — + * so an unlisted provider keeps the existing text-only behaviour and loses + * nothing. + * + * @module adapters/audioFormatSupport + */ + +import type { AudioConversionResult } from "../types/index.js"; +import { withTimeout } from "../utils/errorHandling.js"; +import { logger } from "../utils/logger.js"; +import { getFfmpegPath, runFfmpeg } from "./video/ffmpegAdapter.js"; + +/** + * Ceiling for one audio conversion. + * + * Longer than the image equivalent because a lossless hour-long WAV is a + * legitimate input and re-encoding it is not instant, but still bounded so a + * wedged decoder cannot hold a generation request open indefinitely. + */ +const AUDIO_TRANSCODE_TIMEOUT_MS = 120_000; + +/** + * Providers that accept inline audio parts. + * + * Google's Gemini models (both Vertex and AI Studio) take audio as `inlineData` + * alongside text. Other providers are omitted rather than assumed: OpenAI's + * audio models use a different request shape than the chat-completions path + * NeuroLink builds here, and sending an audio part to a provider that does not + * expect one converts a working (if limited) response into a hard failure. + */ +const NATIVE_AUDIO_PROVIDERS: ReadonlySet = new Set([ + "vertex", + "google-vertex", + "googlevertex", + "google-ai-studio", + "googleaistudio", + "google-ai", + "googleai", + "gemini", +]); + +/** + * Audio MIME types the native providers accept as-is. + * + * Gemini's documented set. Anything outside it is transcoded rather than + * rejected, because the container a user happens to have — a voice memo in + * CAF, a Windows recording in WMA — says nothing about whether the audio + * inside is useful. + */ +const NATIVE_AUDIO_MIME_TYPES: ReadonlySet = new Set([ + "audio/wav", + "audio/x-wav", + "audio/mpeg", + "audio/mp3", + "audio/aiff", + "audio/x-aiff", + "audio/aac", + "audio/ogg", + "audio/flac", + "audio/x-flac", +]); + +/** MIME type every transcode targets. Universally accepted and compact. */ +const TRANSCODE_TARGET_MIME = "audio/mpeg"; + +/** Whether `provider` can be handed raw audio bytes. */ +export function supportsNativeAudio(provider: string): boolean { + return NATIVE_AUDIO_PROVIDERS.has(provider.toLowerCase().trim()); +} + +/** Whether `mimeType` must be re-encoded before a native provider will read it. */ +export function needsAudioTranscode(mimeType: string): boolean { + return !NATIVE_AUDIO_MIME_TYPES.has(normalizeAudioMime(mimeType)); +} + +function normalizeAudioMime(mimeType: string): string { + return mimeType.split(";")[0].trim().toLowerCase(); +} + +/** + * Re-encode audio to MP3 with ffmpeg. + * + * Temp files rather than stdin: several of the containers that need converting + * (CAF, WavPack, AU) carry their metadata in a trailer or require seeking, and + * a piped stream leaves ffmpeg unable to find it. The directory is removed in + * `finally` whether or not the conversion succeeded. + * + * Node builtins are imported dynamically because the browser bundle stubs + * `node:fs/promises` without `mkdtemp`; nothing in a browser spawns ffmpeg, so + * the import belongs at the point of use. + */ +async function transcodeToMp3( + buffer: Buffer, + extension: string, +): Promise { + const [ + { randomUUID }, + { mkdtemp, readFile, rm, writeFile }, + { tmpdir }, + { join }, + ] = await Promise.all([ + import("node:crypto"), + import("node:fs/promises"), + import("node:os"), + import("node:path"), + ]); + const workDir = await mkdtemp(join(tmpdir(), "neurolink-audio-")); + const inputPath = join(workDir, `${randomUUID()}${extension}`); + const outputPath = join(workDir, `${randomUUID()}.mp3`); + try { + await writeFile(inputPath, buffer); + await runFfmpeg( + [ + "-y", + "-v", + "error", + "-i", + inputPath, + // Downmix and cap the rate: speech is the point, and a 48 kHz stereo + // re-encode of a mono voice memo triples the payload for nothing. + "-ac", + "1", + "-ar", + "16000", + "-c:a", + "libmp3lame", + "-q:a", + "4", + outputPath, + ], + // Without this the call inherits runFfmpeg's frame-extraction default of + // 30s, which is sized for pulling a single video frame — so the 120s + // ceiling above, chosen precisely because re-encoding a lossless + // hour-long WAV is not instant, could never be reached. ffmpeg killed the + // transcode at 30s and the outer race never got to run. + { timeoutMs: AUDIO_TRANSCODE_TIMEOUT_MS }, + ); + return await readFile(outputPath); + } finally { + await rm(workDir, { recursive: true, force: true }).catch(() => undefined); + } +} + +/** + * Return audio bytes a native provider can read, transcoding when the source + * container is one it does not accept. + * + * Never throws for audio reasons. When conversion is impossible — no ffmpeg, an + * unreadable stream — the original bytes and MIME type come back with + * `converted: false`, and the caller falls back to the metadata summary. That + * keeps this from turning a previously-working (if limited) request into a + * failure. + * + * @param buffer - Raw audio bytes. + * @param mimeType - Detected MIME type of `buffer`. + * @param extension - Source extension, used so ffmpeg picks the right demuxer. + */ +export async function toProviderCompatibleAudio( + buffer: Buffer, + mimeType: string, + extension: string, +): Promise { + const normalized = normalizeAudioMime(mimeType); + if (!needsAudioTranscode(normalized)) { + return { buffer, mimeType: normalized, converted: false }; + } + + // Resolving the binary first turns "ffmpeg is not installed" into one clear + // warning rather than a spawn error surfacing from inside the conversion. + const ffmpegAvailable = await getFfmpegPath() + .then(() => true) + .catch(() => false); + if (!ffmpegAvailable) { + logger.warn( + `[audioFormatSupport] ${normalized} needs conversion before a provider can ` + + `read it, but ffmpeg is unavailable — falling back to a metadata-only ` + + `summary. Install ffmpeg (or set FFMPEG_PATH) to enable this format.`, + ); + return { buffer, mimeType: normalized, converted: false }; + } + + try { + const converted = await withTimeout( + transcodeToMp3(buffer, extension), + AUDIO_TRANSCODE_TIMEOUT_MS, + new Error(`audio transcode exceeded ${AUDIO_TRANSCODE_TIMEOUT_MS}ms`), + ); + if (converted.length === 0) { + throw new Error("produced an empty audio stream"); + } + logger.debug( + `[audioFormatSupport] Transcoded ${normalized} → ${TRANSCODE_TARGET_MIME} ` + + `(${buffer.length} → ${converted.length} bytes) for native delivery`, + ); + return { + buffer: converted, + mimeType: TRANSCODE_TARGET_MIME, + converted: true, + }; + } catch (error) { + logger.warn( + `[audioFormatSupport] Could not convert ${normalized} for native delivery ` + + `— falling back to a metadata-only summary: ` + + `${error instanceof Error ? error.message.split("\n")[0] : String(error)}`, + ); + return { buffer, mimeType: normalized, converted: false }; + } +} diff --git a/src/lib/processors/archive/ArchiveProcessor.ts b/src/lib/processors/archive/ArchiveProcessor.ts index 9ee1abbe2..b5a4040f0 100644 --- a/src/lib/processors/archive/ArchiveProcessor.ts +++ b/src/lib/processors/archive/ArchiveProcessor.ts @@ -40,6 +40,7 @@ import * as path from "path"; import { BaseFileProcessor } from "../base/BaseFileProcessor.js"; import type { + ArchiveDecompressionResult, ArchiveEntry, ArchiveFormat, FileInfo, @@ -161,11 +162,29 @@ const SUPPORTED_ARCHIVE_MIME_TYPES = [ "application/x-gzip", "application/x-compressed-tar", "application/x-bzip2", + // XZ and Zstandard, added alongside the extensions above: a caller that + // uploads bytes with a declared mimetype and no filename was rejected at the + // type gate for the two formats this change teaches the processor to read. + "application/x-xz", + "application/zstd", "application/java-archive", ] as const; +/** + * External decompressor per single-stream format. + * + * One map rather than a ternary repeated at each use: the decompressor and the + * error message that names it were derived separately, so they could disagree + * about which binary the user is being told to install. + */ +const SINGLE_STREAM_TOOLS: Record<"bz2" | "xz" | "zst", string> = { + bz2: "bzip2", + xz: "xz", + zst: "zstd", +}; + /** File extensions recognized as archive formats */ -const SUPPORTED_ARCHIVE_EXTENSIONS = [".zip", ".tar", ".gz", ".tgz", ".bz2", ".tbz2", ".jar"] as const; +const SUPPORTED_ARCHIVE_EXTENSIONS = [".zip", ".tar", ".gz", ".tgz", ".bz2", ".tbz2", ".jar", ".xz", ".txz", ".zst", ".tzst"] as const; // ============================================================================= // MAGIC BYTE SIGNATURES @@ -184,8 +203,20 @@ const MAGIC_BYTES = { ZIP_SPANNED: [0x50, 0x4b, 0x07, 0x08], /** GZIP: \x1f\x8b */ GZIP: [0x1f, 0x8b], - /** BZIP2: BZ */ - BZIP2: [0x42, 0x5a], + /** + * BZIP2: `BZh` — all three bytes, not just `BZ`. + * + * Two bytes was harmless while every BZIP2 match was rejected outright as an + * unsupported format. Now that a match actually spawns `bzip2`, any buffer + * beginning with the ASCII letters "BZ" is handed to the decompressor and + * comes back reported as a corrupt BZ2 stream — a confident wrong answer + * about a file that was never bzip2 at all. + */ + BZIP2: [0x42, 0x5a, 0x68], + /** XZ: \xfd7zXZ\x00 */ + XZ: [0xfd, 0x37, 0x7a, 0x58, 0x5a, 0x00], + /** Zstandard frame: \x28\xb5\x2f\xfd */ + ZSTD: [0x28, 0xb5, 0x2f, 0xfd], /** RAR: Rar!\x1a\x07 */ RAR: [0x52, 0x61, 0x72, 0x21, 0x1a, 0x07], /** 7-Zip: 7z\xbc\xaf\x27\x1c */ @@ -349,7 +380,7 @@ export class ArchiveProcessor extends BaseFileProcessor { error: this.createError(FileErrorCode.UNSUPPORTED_TYPE, { format, reason: `${format.toUpperCase()} archives are not yet supported. Please convert to ZIP or TAR format.`, - supportedFormats: "ZIP, TAR, TAR.GZ, GZ", + supportedFormats: "ZIP, TAR, TAR.GZ, TAR.BZ2, GZ, BZ2, XZ, ZST, JAR", }), }; } @@ -397,7 +428,10 @@ export class ArchiveProcessor extends BaseFileProcessor { // Step 9: Extract content from text-based entries (Phase 2 sub-processing) // For ZIP archives, extract and include content from small text-based files. // Skips nested archives and binary files for safety. - let extractedContents: Map = new Map(); + // ZIP re-parses to pull member text; every other format collects it while + // decompressing, because their bytes stream past exactly once. + let extractedContents: Map = + extractionResult.contents ?? new Map(); if (format === "zip") { extractedContents = await this.extractEntryContents(buffer, entries); } @@ -472,6 +506,18 @@ export class ArchiveProcessor extends BaseFileProcessor { // Could still be a tar.gz without the extension - we'll detect during extraction return "gz"; } + // Bzip2 wraps a tar just as often, and now that a BZIP2 magic match + // resolves to a real format rather than being rejected outright, the + // same filename check has to run here too. Without it every genuine + // `.tar.bz2` reported its format as "BZ2" to the model — extraction was + // right, the label was not — and the `.tar.bz2` extension mapping below + // became unreachable for any well-formed file. + if (magicFormat === "bz2") { + const ext = filename.toLowerCase(); + return ext.endsWith(".tar.bz2") || ext.endsWith(".tbz2") + ? "tar.bz2" + : "bz2"; + } return magicFormat; } @@ -515,9 +561,21 @@ export class ArchiveProcessor extends BaseFileProcessor { return "gz"; } - // Check for BZIP2 (2 bytes) + // Check for BZIP2 (3 bytes: "BZh"). A bare .bz2 is a single compressed + // file; the caller decides whether the decompressed bytes turn out to be a + // tar. if (this.matchesMagic(buffer, MAGIC_BYTES.BZIP2)) { - return "tar.bz2"; + return "bz2"; + } + + // Check for XZ (6 bytes) + if (buffer.length >= 6 && this.matchesMagic(buffer, MAGIC_BYTES.XZ)) { + return "xz"; + } + + // Check for Zstandard (4 bytes) + if (buffer.length >= 4 && this.matchesMagic(buffer, MAGIC_BYTES.ZSTD)) { + return "zst"; } return null; @@ -545,7 +603,21 @@ export class ArchiveProcessor extends BaseFileProcessor { return "gz"; } if (lowerFilename.endsWith(".bz2")) { - return "tar.bz2"; + // A bare `.bz2` is a single compressed file, not a tarball. Reporting it + // as "tar.bz2" sent it down a branch that refused the format outright. + return "bz2"; + } + if (lowerFilename.endsWith(".tar.xz") || lowerFilename.endsWith(".txz")) { + return "xz"; + } + if (lowerFilename.endsWith(".xz")) { + return "xz"; + } + if (lowerFilename.endsWith(".tar.zst") || lowerFilename.endsWith(".tzst")) { + return "zst"; + } + if (lowerFilename.endsWith(".zst")) { + return "zst"; } if (lowerFilename.endsWith(".zip") || lowerFilename.endsWith(".jar")) { return "zip"; @@ -595,6 +667,7 @@ export class ArchiveProcessor extends BaseFileProcessor { success: boolean; entries: ArchiveEntry[]; securityWarnings: string[]; + contents?: Map; error?: FileProcessingError; }> { switch (format) { @@ -605,16 +678,12 @@ export class ArchiveProcessor extends BaseFileProcessor { case "tar.gz": return this.extractTarGzEntries(buffer); case "tar.bz2": - return { - success: false, - entries: [], - securityWarnings: [], - error: this.createError(FileErrorCode.UNSUPPORTED_TYPE, { - format: "tar.bz2", - reason: "TAR.BZ2 archives are not yet supported. Please convert to ZIP or TAR.GZ format.", - supportedFormats: "ZIP, TAR, TAR.GZ, GZ", - }), - }; + case "bz2": + return this.extractSingleStreamEntries(buffer, "bz2"); + case "xz": + return this.extractSingleStreamEntries(buffer, "xz"); + case "zst": + return this.extractSingleStreamEntries(buffer, "zst"); case "gz": return this.extractGzEntries(buffer); default: @@ -625,7 +694,7 @@ export class ArchiveProcessor extends BaseFileProcessor { error: this.createError(FileErrorCode.UNSUPPORTED_TYPE, { format, reason: `${format} archives are not supported`, - supportedFormats: "ZIP, TAR, TAR.GZ, GZ", + supportedFormats: "ZIP, TAR, TAR.GZ, TAR.BZ2, GZ, BZ2, XZ, ZST, JAR", }), }; } @@ -646,6 +715,7 @@ export class ArchiveProcessor extends BaseFileProcessor { success: boolean; entries: ArchiveEntry[]; securityWarnings: string[]; + contents?: Map; error?: FileProcessingError; }> { const entries: ArchiveEntry[] = []; @@ -783,6 +853,7 @@ export class ArchiveProcessor extends BaseFileProcessor { success: boolean; entries: ArchiveEntry[]; securityWarnings: string[]; + contents?: Map; error?: FileProcessingError; }> { try { @@ -815,6 +886,7 @@ export class ArchiveProcessor extends BaseFileProcessor { success: boolean; entries: ArchiveEntry[]; securityWarnings: string[]; + contents?: Map; error?: FileProcessingError; }> { try { @@ -908,13 +980,16 @@ export class ArchiveProcessor extends BaseFileProcessor { success: boolean; entries: ArchiveEntry[]; securityWarnings: string[]; + contents?: Map; error?: FileProcessingError; }> { return new Promise((resolve) => { const entries: ArchiveEntry[] = []; const securityWarnings: string[] = []; + const contents = new Map(); let entryCount = 0; let cumulativeSize = 0; + let extractedBytes = 0; let earlyError: FileProcessingError | null = null; const extract = tarStream.extract(); @@ -989,6 +1064,36 @@ export class ArchiveProcessor extends BaseFileProcessor { isDirectory, }); + // Capture the text of eligible members as they go past. + // + // A listing alone is not an answer. Asked what a .tar contains, the + // model could previously only recite filenames and sizes — the same + // archive as a .zip returned its text, because only the ZIP path + // extracted anything. A tar entry's bytes are available exactly once, + // here, while the stream is open; re-reading later would mean parsing + // the whole archive a second time. + if ( + !isDirectory && + this.isExtractableEntryName(entryName) && + entrySize > 0 && + entrySize <= ARCHIVE_CONFIG.MAX_EXTRACT_ENTRY_SIZE && + contents.size < ARCHIVE_CONFIG.MAX_EXTRACT_ENTRIES && + extractedBytes + entrySize <= ARCHIVE_CONFIG.MAX_TOTAL_EXTRACT_SIZE + ) { + const chunks: Buffer[] = []; + stream.on("data", (chunk: Buffer) => chunks.push(chunk)); + stream.on("end", () => { + const text = this.decodeEntryText(Buffer.concat(chunks)); + if (text !== null) { + contents.set(entryName, text); + extractedBytes += entrySize; + } + next(); + }); + stream.on("error", () => next()); + return; + } + // Consume the stream without buffering (we only need metadata) stream.resume(); next(); @@ -1003,7 +1108,7 @@ export class ArchiveProcessor extends BaseFileProcessor { error: earlyError, }); } else { - resolve({ success: true, entries, securityWarnings }); + resolve({ success: true, entries, securityWarnings, contents }); } }); @@ -1052,6 +1157,7 @@ export class ArchiveProcessor extends BaseFileProcessor { success: boolean; entries: ArchiveEntry[]; securityWarnings: string[]; + contents?: Map; error?: FileProcessingError; }> { try { @@ -1110,7 +1216,18 @@ export class ArchiveProcessor extends BaseFileProcessor { }, ]; - return { success: true, entries, securityWarnings }; + // The decompressed bytes ARE the content here; a listing that says + // "decompressed-content (1.02 MB)" answers nothing about the file. + const contents = new Map(); + const gzText = this.decodeEntryText(decompressed); + if (gzText !== null) { + contents.set( + innerFilename, + gzText.slice(0, ARCHIVE_CONFIG.MAX_EXTRACT_ENTRY_SIZE), + ); + } + + return { success: true, entries, securityWarnings, contents }; } catch (error) { return { success: false, @@ -1192,6 +1309,278 @@ export class ArchiveProcessor extends BaseFileProcessor { * @param entries - Previously extracted entry metadata * @returns Map of entry name to extracted text content */ + /** + * Whether an entry name looks like something worth inlining as text. + * + * Shared by the ZIP and TAR paths so the two cannot drift into disagreeing + * about which members are worth reading — they did, because only ZIP had the + * rule at all. + */ + private isExtractableEntryName(name: string): boolean { + const ext = path.extname(name).toLowerCase(); + if (ARCHIVE_CONFIG.EXTRACTABLE_EXTENSIONS.has(ext)) { + return true; + } + const base = path.basename(name).toLowerCase(); + return ( + base === "readme" || + base === "license" || + base === "makefile" || + base === "dockerfile" + ); + } + + /** + * Decode bytes to text, or null when they are not text. + * + * A NUL byte in the first 512 bytes, or a high proportion of replacement + * characters after decoding, means binary — inlining that would spend the + * extraction budget on mojibake. + */ + private decodeEntryText(data: Buffer): string | null { + if (!data || data.length === 0) { + return null; + } + if (data.subarray(0, Math.min(512, data.length)).includes(0)) { + return null; + } + const text = data.toString("utf-8"); + const replacements = (text.match(/\ufffd/g) || []).length; + return replacements > text.length * 0.05 ? null : text; + } + + /** + * Decompress a single-stream archive (.bz2, .xz, .zst). + * + * Node ships zstd from v22.15/23, so that one needs no help. bzip2 and xz + * have no Node binding, and adding a native module for them would make an + * optional format a build-time dependency for every consumer — so the system + * tools are used when present, the same soft-dependency arrangement this + * codebase already has with ffmpeg. Absent tooling returns null and the + * caller reports the format as unsupported *on this machine* rather than + * unsupported in principle. + */ + private async decompressSingleStream( + buffer: Buffer, + format: "bz2" | "xz" | "zst", + ): Promise { + if (format === "zst") { + // Node gained zstd in v22.15/23. Older runtimes simply lack the export, + // so it is probed at runtime rather than assumed from the typings — + // which is also why this is a property check on an `unknown` module + // instead of a cast asserting it exists. + const zlibModule: unknown = await import("zlib"); + const candidate = + typeof zlibModule === "object" && zlibModule !== null + ? (zlibModule as Record).zstdDecompress + : undefined; + if (typeof candidate === "function") { + const zstdDecompress = candidate as ( + buf: Buffer, + opts: { maxOutputLength: number }, + cb: (err: Error | null, res: Buffer) => void, + ) => void; + return await new Promise((resolve) => { + // Bounded at the decoder rather than after the fact. The size guard + // downstream only runs once the whole buffer exists, which is too + // late for a zip-bomb: a few KB of zstd expands to gigabytes and the + // allocation is what hurts. `maxOutputLength` makes it fail fast. + zstdDecompress( + buffer, + { maxOutputLength: ARCHIVE_SECURITY.MAX_DECOMPRESSED_SIZE }, + (err, res) => + resolve( + err + ? { + status: + (err as NodeJS.ErrnoException).code === + "ERR_BUFFER_TOO_LARGE" + ? "too-large" + : "failed", + } + : { status: "ok", buffer: res }, + ), + ); + }); + } + } + + const tool = SINGLE_STREAM_TOOLS[format]; + try { + const { execFile } = await import("node:child_process"); + return await new Promise((resolve) => { + // ENOENT means the binary is absent; anything else means it ran and + // could not do the job. Only the former justifies telling the caller to + // install something. + // + // Read off the error itself rather than a flag set by the `error` + // listener below, because execFile invokes this callback BEFORE that + // listener runs — measured, not assumed: + // callback(missing=false, code=ENOENT) → errorListener(code=ENOENT) + // The promise is already settled by the time the listener could set the + // flag, so a missing decompressor reported itself as a corrupt stream + // and told the user to re-upload a perfectly good file. + const isMissingTool = (error: unknown): boolean => + (error as NodeJS.ErrnoException | null)?.code === "ENOENT"; + // Hitting `maxBuffer` is the zip-bomb guard firing, not a corrupt + // stream: it will fail identically on every retry, so it must not be + // reported as retryable the way a truncated upload is. + const isTooLarge = (error: unknown): boolean => + (error as NodeJS.ErrnoException | null)?.code === + "ERR_CHILD_PROCESS_STDIO_MAXBUFFER"; + const child = execFile( + tool, + ["-dc"], + { + encoding: "buffer", + maxBuffer: ARCHIVE_SECURITY.MAX_DECOMPRESSED_SIZE, + // Without this a crafted or truncated stream that makes the tool + // block forever never fires the callback, so the promise never + // settles and the request that awaits it hangs indefinitely. + timeout: ARCHIVE_CONFIG.TIMEOUT_MS, + }, + (error, stdout) => { + // Only a non-zero exit means failure. Empty output does not: + // compressing an empty file is legal and round-trips — a 32-byte + // `empty.txt.xz` decompresses to nothing with exit 0 — so treating + // zero length as an error reported a valid archive as corrupt. The + // in-process zstd path above already keyed on the error alone, so + // the two backends disagreed about the same input. + if (error || !stdout) { + resolve({ + status: isMissingTool(error) + ? "tool-unavailable" + : isTooLarge(error) + ? "too-large" + : "failed", + }); + return; + } + resolve({ status: "ok", buffer: Buffer.from(stdout) }); + }, + ); + // Kept as a backstop for spawn failures that never reach the + // callback. Whichever settles first wins; both now classify the same + // way, so the outcome no longer depends on the order. + child.on("error", (error: NodeJS.ErrnoException) => { + resolve({ + status: isMissingTool(error) ? "tool-unavailable" : "failed", + }); + }); + // Swallowed, deliberately not resolved from. A stdin write fails with + // EPIPE precisely because the child already died — including when Node + // killed it for exceeding `maxBuffer`, which is the zip-bomb guard. + // Resolving here raced the exec callback and, when it won, downgraded a + // "too-large" verdict to a retryable "failed", telling the caller to + // retry a bomb. The callback is the authoritative signal: it always + // fires once the process exits or fails to spawn, and the `timeout` + // above bounds the wait. This listener exists only so an unhandled + // 'error' event cannot take the process down. + child.stdin?.on("error", () => undefined); + child.stdin?.end(buffer); + }); + } catch { + return { status: "tool-unavailable" }; + } + } + + /** + * Extract a single-stream archive: decompress, then treat the result as a + * TAR when it is one and as a lone file otherwise. + * + * The tar check matters because `.tar.xz` and `.tar.zst` are how these + * formats are usually met — reporting one opaque "decompressed-content" blob + * for an archive of forty files would be technically true and useless. + */ + private async extractSingleStreamEntries( + buffer: Buffer, + format: "bz2" | "xz" | "zst", + ): Promise<{ + success: boolean; + entries: ArchiveEntry[]; + securityWarnings: string[]; + contents?: Map; + error?: FileProcessingError; + }> { + const result = await this.decompressSingleStream(buffer, format); + if (result.status !== "ok") { + const tool = SINGLE_STREAM_TOOLS[format]; + // Three distinct outcomes, and the codes carry contracts a caller acts + // on. UNSUPPORTED_TYPE is `retryable: false` and advises converting the + // file; DECOMPRESSION_FAILED is `retryable: true` and advises + // re-uploading; SECURITY_VALIDATION_FAILED is how the GZIP and TAR.GZ + // paths in this file already report a decompression bomb. `retryable` is + // read downstream by `isRetryableErrorCode`, so a stream that tripped the + // decoder's size cap must not be advertised as worth retrying — it will + // fail identically every time — and a merely truncated upload must not be + // reported as an unsupported format. + const code = + result.status === "tool-unavailable" + ? FileErrorCode.UNSUPPORTED_TYPE + : result.status === "too-large" + ? FileErrorCode.SECURITY_VALIDATION_FAILED + : FileErrorCode.DECOMPRESSION_FAILED; + return { + success: false, + entries: [], + securityWarnings: [], + error: this.createError(code, { + format, + reason: + result.status === "tool-unavailable" + ? `${format.toUpperCase()} could not be decompressed. Node has no built-in ` + + `decoder for it and the "${tool}" command is unavailable on this machine.` + : result.status === "too-large" + ? `${format.toUpperCase()} expands beyond the ` + + `${this.formatSizeMB(ARCHIVE_SECURITY.MAX_DECOMPRESSED_SIZE)} MB ` + + `decompression limit and was refused before it could be read.` + : `${format.toUpperCase()} could not be decompressed. The stream is ` + + `corrupt or truncated.`, + supportedFormats: "ZIP, TAR, TAR.GZ, TAR.BZ2, GZ, BZ2, XZ, ZST, JAR", + }), + }; + } + const decompressed = result.buffer; + + if (decompressed.length > ARCHIVE_SECURITY.MAX_DECOMPRESSED_SIZE) { + return { + success: false, + entries: [], + securityWarnings: [], + error: this.createError(FileErrorCode.SECURITY_VALIDATION_FAILED, { + reason: `Decompressed size (${this.formatSizeMB(decompressed.length)} MB) exceeds limit (${this.formatSizeMB(ARCHIVE_SECURITY.MAX_DECOMPRESSED_SIZE)} MB)`, + }), + }; + } + + if (this.looksLikeTar(decompressed)) { + const tarStream = await import("tar-stream"); + return await this.parseTarStream(tarStream, decompressed); + } + + const contents = new Map(); + const text = this.decodeEntryText(decompressed); + if (text !== null) { + contents.set( + "decompressed-content", + text.slice(0, ARCHIVE_CONFIG.MAX_EXTRACT_ENTRY_SIZE), + ); + } + return { + success: true, + entries: [ + { + name: "decompressed-content", + uncompressedSize: decompressed.length, + compressedSize: buffer.length, + isDirectory: false, + }, + ], + securityWarnings: [], + contents, + }; + } + private async extractEntryContents(buffer: Buffer, entries: ArchiveEntry[]): Promise> { const contents = new Map(); @@ -1212,18 +1601,7 @@ export class ArchiveProcessor extends BaseFileProcessor { return false; } - const ext = path.extname(e.name).toLowerCase(); - // Check by extension - if (ARCHIVE_CONFIG.EXTRACTABLE_EXTENSIONS.has(ext)) { - return true; - } - // Check for common extensionless config files - const basename = path.basename(e.name).toLowerCase(); - if (basename === "readme" || basename === "license" || basename === "makefile" || basename === "dockerfile") { - return true; - } - - return false; + return this.isExtractableEntryName(e.name); }) // Sort: smaller files first (more likely to fit), then by name .sort((a, b) => a.uncompressedSize - b.uncompressedSize); diff --git a/src/lib/providers/googleAiStudio/client.ts b/src/lib/providers/googleAiStudio/client.ts index 32ed11747..1403e040d 100644 --- a/src/lib/providers/googleAiStudio/client.ts +++ b/src/lib/providers/googleAiStudio/client.ts @@ -10,6 +10,7 @@ import { TOOL_STORAGE_TIMEOUT_MS, } from "../../core/constants.js"; import { + mergeMediaFileAliases, normalizeVisionImageFormats, processUnifiedFilesArray, } from "../../utils/messageBuilder.js"; @@ -707,6 +708,55 @@ export class GoogleAIStudioProvider extends BaseProvider { } // executeGenerate removed - BaseProvider handles all generation with tools + /** + * Run the file preprocessing this provider's native paths depend on. + * + * AI Studio overrides both `generate()` and `executeStream()` and routes + * straight to the native SDK, so neither reaches + * `buildMultimodalMessagesArray` — the place that turns `input.files` into + * text, images, PDFs and `nativeAudioFiles`. `BaseProvider.stream()` does + * build messages, but onto a throwaway clone whose result is discarded, so + * the real `options.input` came through untouched. + * + * The consequence was asymmetric and easy to miss: `generate()` did this + * inline and worked, while `stream()` silently dropped every attached file — + * not just audio, but the metadata summary too. Vertex hit the identical bug + * (#1258) and solved it with exactly this shape, called from both entry + * points. + */ + private async preprocessNativeFileInput( + options: TextGenerationOptions | StreamOptions, + ): Promise { + // The user-facing aliases (`input.audioFiles`, `input.videoFiles`) are + // folded into `input.files` here, exactly as the Vertex client does. Only + // `files` is processed below, so without this a caller who used the + // documented `audioFiles` field had it silently ignored on both of this + // provider's paths. + if (options.input) { + mergeMediaFileAliases(options.input); + } + if (options.input?.files && options.input.files.length > 0) { + try { + // Mutates options.input.text / .images / .pdfFiles / .nativeAudioFiles + // in place. + await processUnifiedFilesArray( + options as Parameters[0], + 100 * 1024 * 1024, + this.providerName, + ); + } catch (fileError) { + logger.warn( + `[GoogleAIStudio] processUnifiedFilesArray threw, continuing without file content: ${fileError instanceof Error ? fileError.message : String(fileError)}`, + ); + } + } + + // Runs even without input.files: a caller can populate input.images + // directly, and this native path never reaches the shared multimodal + // builder that would otherwise normalize the formats. + await normalizeVisionImageFormats(options.input); + } + protected async executeStream( options: StreamOptions, analysisSchema?: ZodUnknownSchema | Schema, @@ -718,6 +768,10 @@ export class GoogleAIStudioProvider extends BaseProvider { return await this.executeAudioStreamViaGeminiLive(options); } + // #1258, for this provider: stream() must run the same file preprocessing + // generate() does, or attached files are dropped on this path alone. + await this.preprocessNativeFileInput(options); + // Structured output (analysisSchema, JSON format, or schema) is incompatible with tools on Gemini. const wantsStructuredOutput = analysisSchema || options.output?.format === "json" || options.schema; @@ -1675,30 +1729,7 @@ export class GoogleAIStudioProvider extends BaseProvider { return this.handleDirectTTSSynthesis(options, Date.now()); } - // Process the unified `input.files` array before routing to the - // native SDK. BaseProvider.generate() runs this preprocessing via - // buildMultimodalMessagesArray, but AI Studio's override skips it, - // which would otherwise drop text-file content (and the - // mimetype-hint contract) on the floor. Mutates options.input.text / - // options.input.images / options.input.pdfFiles in place. - if (options.input?.files && options.input.files.length > 0) { - try { - await processUnifiedFilesArray( - options as Parameters[0], - 100 * 1024 * 1024, - this.providerName, - ); - } catch (fileError) { - logger.warn( - `[GoogleAIStudio] processUnifiedFilesArray threw, continuing without file content: ${fileError instanceof Error ? fileError.message : String(fileError)}`, - ); - } - } - - // Runs even without input.files: a caller can populate input.images - // directly, and this native path never reaches the shared multimodal - // builder that would otherwise normalize the formats. - await normalizeVisionImageFormats(options.input); + await this.preprocessNativeFileInput(options); // Merge registered (built-in / MCP) tools with caller-supplied tools. // AI Studio's generate() bypasses BaseProvider.generate(), so the diff --git a/src/lib/providers/googleNativeGemini3/utils.ts b/src/lib/providers/googleNativeGemini3/utils.ts index b5a9638d4..67b44c05f 100644 --- a/src/lib/providers/googleNativeGemini3/utils.ts +++ b/src/lib/providers/googleNativeGemini3/utils.ts @@ -36,7 +36,12 @@ import type { VertexSegment, VertexToolStep, GeminiMultimodalInput, + MultimodalAudioEntry, } from "../../types/index.js"; +import { + needsAudioTranscode, + toProviderCompatibleAudio, +} from "../../adapters/audioFormatSupport.js"; import { logger } from "../../utils/logger.js"; import { resolveSamplingParams } from "../../models/modelRegistry.js"; import { @@ -1865,6 +1870,66 @@ export function prependConversationMessages( * is skipped rather than aborting the entire request, matching prior * Vertex behaviour. */ +/** + * Append audio to a Gemini request as `inlineData` parts. + * + * Shared by both Gemini front ends. Vertex assembles its request here and AI + * Studio assembles it in `buildUserPartsWithMultimodal`; when this lived only in + * the Vertex client, AI Studio advertised audio support through + * `NATIVE_AUDIO_PROVIDERS` and then silently dropped the bytes. + * + * Gemini's native request shape is assembled directly rather than taken from the + * AI SDK's `file` parts, so audio has to be added explicitly the same way PDFs + * and images are — a `{ type: "file" }` part built upstream simply never + * reaches this request body. That asymmetry is why attaching a recording + * produced only the metadata summary even after the message builder learned to + * carry the bytes. + * + * A container Gemini does not accept is converted first; one that cannot be + * converted is skipped rather than sent, because an unsupported inlineData + * mimeType fails the whole request, and the caller still has the metadata + * summary in the text part. + */ +export async function appendNativeAudioParts( + userParts: VertexNativePart[], + audioFiles: MultimodalAudioEntry[] | undefined, + logPrefix: string = "[GeminiNative]", +): Promise { + if (!audioFiles || audioFiles.length === 0) { + return; + } + for (const audio of audioFiles) { + // Split on both separators: a Windows-style name reaching a POSIX host + // would otherwise keep its whole path, and the extension lookup below + // needs the bare filename. + const base = audio.filename.split(/[\\/]/).pop() ?? audio.filename; + const dot = base.lastIndexOf("."); + const extension = dot > 0 ? base.slice(dot) : ".bin"; + const compatible = await toProviderCompatibleAudio( + audio.buffer, + audio.mimeType, + extension, + ); + if (needsAudioTranscode(compatible.mimeType)) { + logger.warn( + `${logPrefix} Skipping native audio for ${base}: ${compatible.mimeType} ` + + `is not accepted and could not be converted. The metadata summary was ` + + `still included.`, + ); + continue; + } + userParts.push({ + inlineData: { + mimeType: compatible.mimeType, + data: compatible.buffer.toString("base64"), + }, + }); + logger.debug( + `${logPrefix} Added native audio part for ${base} (${compatible.mimeType})`, + ); + } +} + export async function buildUserPartsWithMultimodal( input: GeminiMultimodalInput | undefined, textOverride?: string, @@ -1977,5 +2042,12 @@ export async function buildUserPartsWithMultimodal( } } + // Audio last, and through the same helper the Vertex client uses. AI Studio + // never touches `buildMultimodalMessagesArray` — it overrides generate() and + // stream() and assembles its request here — so wiring audio only into the + // Vertex client left this front end advertising native audio via + // NATIVE_AUDIO_PROVIDERS and then dropping the bytes on the floor. + await appendNativeAudioParts(parts, input?.nativeAudioFiles, logPrefix); + return parts; } diff --git a/src/lib/providers/googleVertex/client.ts b/src/lib/providers/googleVertex/client.ts index e17631712..b35311f3d 100644 --- a/src/lib/providers/googleVertex/client.ts +++ b/src/lib/providers/googleVertex/client.ts @@ -12,6 +12,7 @@ import { } from "../../constants/enums.js"; import { BaseProvider } from "../../core/baseProvider.js"; import { unwrapImagePayload } from "../../adapters/imageFormatSupport.js"; +import { appendNativeAudioParts } from "../googleNativeGemini3/utils.js"; import { getMimeTypeForExtension } from "../../processors/config/mimeConstants.js"; import { DEFAULT_GEMINI_STREAM_TIMEOUT_MS, @@ -54,6 +55,7 @@ import type { VertexSegment, ChatMessage, MinimalChatMessage, + MultimodalAudioEntry, } from "../../types/index.js"; import { AuthenticationError, @@ -1885,6 +1887,7 @@ export class GoogleVertexProvider extends BaseProvider { text: string; pdfFiles?: Array; images?: Array; + nativeAudioFiles?: MultimodalAudioEntry[]; }; if (multimodalInput?.pdfFiles && multimodalInput.pdfFiles.length > 0) { @@ -1918,6 +1921,12 @@ export class GoogleVertexProvider extends BaseProvider { } } + await appendNativeAudioParts( + userParts, + multimodalInput?.nativeAudioFiles, + "[GoogleVertex]", + ); + // Add images as inlineData parts if present if (multimodalInput?.images && multimodalInput.images.length > 0) { logger.debug( @@ -3170,6 +3179,7 @@ export class GoogleVertexProvider extends BaseProvider { text?: string; pdfFiles?: Array; images?: Array; + nativeAudioFiles?: MultimodalAudioEntry[]; } | undefined; @@ -3204,6 +3214,12 @@ export class GoogleVertexProvider extends BaseProvider { } } + await appendNativeAudioParts( + userParts, + multimodalInput?.nativeAudioFiles, + "[GoogleVertex]", + ); + // Add images as inlineData parts if present if (multimodalInput?.images && multimodalInput.images.length > 0) { logger.debug( diff --git a/src/lib/types/file.ts b/src/lib/types/file.ts index 0e4b1cf3a..bb1a0ff86 100644 --- a/src/lib/types/file.ts +++ b/src/lib/types/file.ts @@ -38,6 +38,38 @@ export type VisionImageConversion = { readonly converted: boolean; }; +/** + * Outcome of an audio-compatibility pass over one file. + * + * See `adapters/audioFormatSupport.ts`. As with images, `converted` is false + * both when the container was already acceptable and when nothing could + * re-encode it, so it is not a success flag — the caller decides what to do + * from the resulting `mimeType`. + */ +export type AudioConversionResult = { + readonly buffer: Buffer; + readonly mimeType: string; + /** True when the bytes were re-encoded; false when they were left alone. */ + readonly converted: boolean; +}; + +/** + * One audio file destined for native delivery to a provider. + * + * Carries the bytes rather than a path because the decision to send audio is + * made per provider, after detection has already read the file — re-reading it + * from disk at dispatch time would be a second read of something already in + * memory. + */ +export type MultimodalAudioEntry = { + /** Raw audio bytes, as detected. */ + buffer: Buffer; + /** Display name; may be a full path, so log only its basename. */ + filename: string; + /** Detected MIME type of `buffer`. */ + mimeType: string; +}; + /** * Broad category a file format belongs to, as a human would name it. * @@ -506,6 +538,17 @@ export type FileDetectorOptions = { * hint (the lazy FileReferenceRegistry path has its own hint-handling). */ mimetypeHint?: string; + /** + * Caller-provided filename hint, the companion to {@link mimetypeHint}. + * + * The unified file path unwraps a `FileWithMetadata` to its `buffer` before + * detection runs, so the object's `filename` is gone by the time extension + * resolution looks for one — and TAR in particular cannot be identified any + * other way, because its "ustar" marker sits at byte 257 rather than at + * offset 0. Passing the name alongside the bytes keeps `.odp`, `.rtf` and + * `.tar` routed to the processors that can actually read them. + */ + filenameHint?: string; }; /** diff --git a/src/lib/types/generate.ts b/src/lib/types/generate.ts index 328de8004..a63d30b12 100644 --- a/src/lib/types/generate.ts +++ b/src/lib/types/generate.ts @@ -33,7 +33,11 @@ import type { ZodUnknownSchema, } from "./aliases.js"; import type { NeurolinkCredentials } from "./providers.js"; -import type { CSVProcessorOptions, FileWithMetadata } from "./file.js"; +import type { + CSVProcessorOptions, + FileWithMetadata, + MultimodalAudioEntry, +} from "./file.js"; import type { WorkflowConfig } from "./workflow.js"; import type { Schema, Tool, ToolChoice } from "./tools.js"; import type { StepResult, LanguageModel } from "./providers.js"; @@ -73,6 +77,17 @@ export type GenerateOptions = { csvFiles?: Array; // Explicit CSV files pdfFiles?: Array; // Explicit PDF files audioFiles?: Array; // Explicit audio files (metadata/transcript extraction) + /** + * Audio whose bytes should be delivered to the provider, populated during + * detection rather than by callers. + * + * Separate from `audioFiles` above, which is the caller-facing input that + * yields a metadata summary. This one carries the decoded bytes forward so + * a provider that can actually listen receives the audio instead of a + * description of it; providers that cannot fall back to the summary and + * this is ignored. + */ + nativeAudioFiles?: MultimodalAudioEntry[]; videoFiles?: Array; // Explicit video files files?: Array; // Auto-detect file types content?: Content[]; // Advanced multimodal content diff --git a/src/lib/types/processor.ts b/src/lib/types/processor.ts index ffcfeaff9..d1b321da6 100644 --- a/src/lib/types/processor.ts +++ b/src/lib/types/processor.ts @@ -883,9 +883,27 @@ export type ArchiveFormat = | "tar.gz" | "tar.bz2" | "gz" + | "bz2" + | "xz" + | "zst" | "rar" | "7z"; +/** + * Outcome of decompressing a single-stream archive (.bz2, .xz, .zst). + * + * A plain `Buffer | null` collapsed two very different failures into one: a + * machine that has no `xz` installed and a `.xz` file that is corrupt both + * returned null, and the caller reported both as "the command is unavailable on + * this machine" — actively misleading for the second. The reason is carried so + * the message can match the fact. + */ +export type ArchiveDecompressionResult = + | { readonly status: "ok"; readonly buffer: Buffer } + | { readonly status: "tool-unavailable" } + | { readonly status: "too-large" } + | { readonly status: "failed" }; + /** * Metadata about an individual entry within an archive. */ diff --git a/src/lib/types/providers.ts b/src/lib/types/providers.ts index bba6f9653..f6a527329 100644 --- a/src/lib/types/providers.ts +++ b/src/lib/types/providers.ts @@ -26,6 +26,7 @@ import type { GenerateResult, TextGenerationOptions, } from "./generate.js"; +import type { MultimodalAudioEntry } from "./file.js"; import type { StreamOptions, StreamResult } from "./stream.js"; import type { ExternalMCPToolInfo } from "./externalMcp.js"; @@ -2124,6 +2125,12 @@ export type GeminiMultimodalInput = { text?: string; pdfFiles?: Array; images?: Array; + /** + * Audio collected during file detection, carried through to the native + * request as `inlineData`. Distinct from the user-facing `audioFiles`: these + * are already-materialised bytes with a resolved mime type. + */ + nativeAudioFiles?: MultimodalAudioEntry[]; }; /** diff --git a/src/lib/utils/fileDetector.ts b/src/lib/utils/fileDetector.ts index 01f472ae4..9cf5fec43 100644 --- a/src/lib/utils/fileDetector.ts +++ b/src/lib/utils/fileDetector.ts @@ -362,6 +362,13 @@ export class FileDetector { // These default ensure consistent timeout behavior across all file-detection logic. public static readonly DEFAULT_NETWORK_TIMEOUT = 30000; // 30 seconds public static readonly DEFAULT_HEAD_TIMEOUT = 5000; // 5 seconds + /** + * Ceiling on an in-process document parse (unzip + XML walk). Generous + * relative to the work, because the cost of firing early on a large but + * legitimate file is a lost extraction, while the cost of never firing is a + * held request. + */ + public static readonly DEFAULT_DOCUMENT_TIMEOUT = 30000; // 30 seconds /** * Auto-detect file type and process in one call * @@ -551,7 +558,14 @@ export class FileDetector { if (Buffer.isBuffer(input)) { return "buffer"; } - return "unknown-input"; + // Everything left is a `FileWithMetadata`, which states its own name, and + // `withResolvedExtension` reads this to recover an extension when a + // content-based strategy reported none. Falling straight through to + // "unknown-input" threw that name away, so an `.odp`, `.rtf` or `.tar` + // supplied as bytes-plus-name lost the extension its processor routes on. + // Still defensive about the value: the type says required, callers are + // untyped JavaScript often enough. + return input?.filename || "unknown-input"; } /** @@ -880,7 +894,8 @@ export class FileDetector { source: FileDetector.deriveInputSource(input), metadata: { confidence: 95, - filename: FileDetector.deriveInputFilename(input), + filename: + options?.filenameHint || FileDetector.deriveInputFilename(input), size: FileDetector.deriveInputSize(input), }, }; @@ -909,7 +924,7 @@ export class FileDetector { logger.info( `[FileDetector] Type: ${result.type} (${result.metadata.confidence}%)`, ); - return result; + return FileDetector.withResolvedExtension(result, input, options); } } @@ -918,7 +933,76 @@ export class FileDetector { logger.debug( `[FileDetector] Best-effort type below threshold: ${best?.type ?? "unknown"} (${best?.metadata.confidence ?? 0}%, threshold ${confidenceThreshold}%)`, ); - return best as FileDetectionResult; + return FileDetector.withResolvedExtension( + best as FileDetectionResult, + input, + options, + ); + } + + /** + * Fill in `extension` from the input's name when detection did not set it. + * + * Content-based strategies identify a type from magic bytes and legitimately + * have no extension to report, so they return null. That is fine for the type + * itself but not for routing: several processors are chosen by extension + * *after* detection has settled the type, because one routing type covers + * several formats — `docx` covers .docx, .odt and .rtf. + * + * With a null extension those branches were unreachable. An .rtf scored high + * on its `{\\rtf1` signature, arrived as type "docx" with no extension, and + * fell through to the Word processor, which cannot read RTF — so a file whose + * dedicated processor extracts it perfectly reported "Could not extract + * content". The extension was known the whole time; it was simply dropped on + * the way through. + * + * Only fills a gap — a strategy that did determine an extension keeps it, so + * content still wins over a lying filename. + */ + private static withResolvedExtension( + result: FileDetectionResult, + input: FileInput, + options?: FileDetectorOptions, + ): FileDetectionResult { + if (!result) { + return result; + } + // The caller's hint outranks a name derived from the input, because on the + // unified path the input has already been unwrapped to a bare Buffer and + // derives to the literal "buffer" — carrying no extension at all. + const filename = + options?.filenameHint || + result.metadata?.filename || + FileDetector.deriveInputFilename(input); + if (!filename) { + return result; + } + // Split on both separators so a Windows-style path on a POSIX host still + // yields its basename, then take the final suffix. + const base = filename.split(/[\\/]/).pop() ?? filename; + const dot = base.lastIndexOf("."); + const extension = + result.extension ?? + (dot > 0 && dot < base.length - 1 + ? base.slice(dot + 1).toLowerCase() + : null); + + // The name is carried alongside the extension for the same reason. A + // content strategy reports no filename, so every processor keyed on one + // received the literal fallback "archive" — and archive format detection + // reads the name, because TAR has no magic bytes at offset 0 (its "ustar" + // marker sits at byte 257). A .tar therefore arrived as an unidentifiable + // archive and reported "Could not extract content", while the same bytes + // handed to the processor WITH their name extract perfectly. + const metadata = + result.metadata && !result.metadata.filename + ? { ...result.metadata, filename: base } + : result.metadata; + + if (extension === result.extension && metadata === result.metadata) { + return result; + } + return { ...result, extension, metadata }; } /** @@ -1813,6 +1897,51 @@ export class FileDetector { ): Promise { const pptxFilename = detection.metadata.filename || "presentation"; try { + // ODP is an OpenDocument package, not OOXML — the PPTX reader finds no + // ppt/slides parts in it and returns nothing. It reaches this branch at + // all because one routing type ("pptx") covers every presentation + // format, the same way "docx" covers .odt. + if (detection.extension?.toLowerCase() === "odp") { + const { openDocumentProcessor } = + await import("../processors/document/OpenDocumentProcessor.js"); + // Bounded per the project's async-timeout guideline: this unzips and + // parses attacker-supplied bytes, and a stalled parse would otherwise + // hold the request open with no ceiling. On timeout the throw lands in + // this block's existing catch, which degrades to the placeholder. + const odpResult = await withTimeout( + openDocumentProcessor.processFile({ + id: pptxFilename, + name: pptxFilename, + mimetype: + detection.mimeType || + "application/vnd.oasis.opendocument.presentation", + size: content.length, + buffer: content, + }), + FileDetector.DEFAULT_DOCUMENT_TIMEOUT, + ); + // Gated on success rather than on text, because a presentation of + // nothing but images is a legitimate ODP that extracts to an empty + // string. Requiring text sent that file on to the PPTX reader, which + // cannot read OpenDocument at all — so a successful extraction was + // discarded in favour of a guaranteed failure. Matches how the ODT and + // ODS branches degrade. + if (odpResult.success && odpResult.data) { + return { + type: "pptx", + content: + odpResult.data.textContent || + FileDetector.formatInformativePlaceholder( + "Presentation", + pptxFilename, + content, + detection, + ), + mimeType: detection.mimeType, + metadata: detection.metadata, + }; + } + } const { PptxProcessor } = await import("../processors/document/PptxProcessor.js"); const pptxResult = await PptxProcessor.extractText(content); @@ -2586,14 +2715,36 @@ class MagicBytesStrategy implements DetectionStrategy { if (input.length >= 6 && input.toString("latin1", 0, 5) === "#!AMR") { return this.result("audio", "audio/amr", 95); } - // JPEG 2000: 12-byte signature box. + // JPEG 2000: the full 12-byte signature box, trailing 0D 0A 87 0A + // included. Those four bytes are a deliberate line-ending probe — CR LF, a + // high byte, LF — that any transfer which mangles newlines or strips the + // eighth bit will visibly corrupt, so checking only the length and brand + // accepts exactly the damaged files the signature exists to reject. if ( - input.length >= 8 && + input.length >= 12 && input[0] === 0x00 && input[1] === 0x00 && input[2] === 0x00 && input[3] === 0x0c && - input.toString("latin1", 4, 8) === "jP " + input.toString("latin1", 4, 8) === "jP " && + input[8] === 0x0d && + input[9] === 0x0a && + input[10] === 0x87 && + input[11] === 0x0a + ) { + return this.result("image", "image/jp2", 95); + } + // The other shape JPEG 2000 ships in: a bare codestream (.j2k/.j2c) opening + // with the SOC + SIZ markers. `ImageProcessor.detectImageType` learned both + // shapes; this strategy knew only the container, so a codestream uploaded + // as bytes-plus-filename was typed "unknown" and delivered as a binary + // blob — the codestream branch over there was unreachable from this path. + if ( + input.length >= 4 && + input[0] === 0xff && + input[1] === 0x4f && + input[2] === 0xff && + input[3] === 0x51 ) { return this.result("image", "image/jp2", 95); } diff --git a/src/lib/utils/imageProcessor.ts b/src/lib/utils/imageProcessor.ts index 4a2d676c4..c46af197f 100644 --- a/src/lib/utils/imageProcessor.ts +++ b/src/lib/utils/imageProcessor.ts @@ -573,6 +573,42 @@ export class ImageProcessor { return isoBmffMimeType; } + // JPEG 2000, in both shapes it ships in. The JP2 container opens with + // the 12-byte signature box `00 00 00 0C 6A 50 20 20 0D 0A 87 0A`; a + // bare codestream (.j2k/.j2c) opens with the SOC+SIZ markers FF 4F FF + // 51. Checked BEFORE ICO because the container's first three bytes are + // 00 00 00, which is one byte away from ICO's 00 00 01 00 and shares + // its leading zeros — an ordering mistake here would classify every + // JPEG 2000 as an icon rather than merely failing to recognise it. + // All twelve bytes are checked, not just the length and brand: the + // trailing 0D 0A 87 0A is the signature's whole point. It is a + // line-ending probe — CR LF, a high byte, LF — that a transfer which + // mangles newlines or strips the eighth bit will visibly corrupt, so + // skipping it accepts exactly the damaged files it exists to reject. + if ( + input.length >= 12 && + input[0] === 0x00 && + input[1] === 0x00 && + input[2] === 0x00 && + input[3] === 0x0c && + input.subarray(4, 8).toString("latin1") === "jP " && + input[8] === 0x0d && + input[9] === 0x0a && + input[10] === 0x87 && + input[11] === 0x0a + ) { + return "image/jp2"; + } + if ( + input.length >= 4 && + input[0] === 0xff && + input[1] === 0x4f && + input[2] === 0xff && + input[3] === 0x51 + ) { + return "image/jp2"; + } + // ICO: 00 00 01 00 (icon type=1) if ( input[0] === 0x00 && diff --git a/src/lib/utils/messageBuilder.ts b/src/lib/utils/messageBuilder.ts index 54dbfb6dd..90e46b1b7 100644 --- a/src/lib/utils/messageBuilder.ts +++ b/src/lib/utils/messageBuilder.ts @@ -42,6 +42,11 @@ import { import { ErrorFactory, NeuroLinkError, withTimeout } from "./errorHandling.js"; import { FileDetector } from "./fileDetector.js"; import { detectIsoBmffImageMimeType } from "./isoBmff.js"; +import { + needsAudioTranscode, + supportsNativeAudio, + toProviderCompatibleAudio, +} from "../adapters/audioFormatSupport.js"; import { getImageCache } from "./imageCache.js"; import { ImageProcessor, imageUtils } from "./imageProcessor.js"; import { logger } from "./logger.js"; @@ -58,6 +63,7 @@ import type { FilePart, ImagePart, TextPart, + MultimodalAudioEntry, MultimodalPdfEntry, } from "../types/index.js"; @@ -83,6 +89,18 @@ const EXTENSION_TYPE_MAP: Record = Object.fromEntries( ), ); +/** + * MIME type → routing type, derived from the same registry as + * {@link EXTENSION_TYPE_MAP} so the two can never disagree about a format. + */ +const MIMETYPE_TYPE_MAP: Record = Object.fromEntries( + FILE_TYPE_REGISTRY.flatMap((entry) => + entry.mimeTypes.map( + (mime) => [mime.toLowerCase(), entry.fileType] as const, + ), + ), +); + /** * Infer file type from extension in a file path or URL. * Returns undefined if no extension or unrecognized. @@ -822,11 +840,89 @@ function enforceFileBudget( } } +/** + * Per input, the file entries already folded into text and media. + * + * A WeakMap so a long-lived process cannot accumulate references to request + * payloads: the record vanishes with the input object it is keyed on. + */ +const PREPROCESSED_FILES = new WeakMap>(); + +/** + * Ceiling on reading one already-detected local file back off disk. + * + * Sized for the 100 MB this path admits from cold storage, not for the warm + * page cache the read usually hits — detection has just read the same bytes. + */ +const FILE_READ_TIMEOUT_MS = 30_000; + +/** + * Read a file input's bytes, or null when they cannot be had. + * + * Asynchronous because this path admits files up to 100 MB: a synchronous read + * of one blocks the event loop for every other in-flight request on the + * process, which for a server handling concurrent generations is not a + * micro-optimisation to trade away. + * + * A URL or data URI yields null rather than a fetch: those arrive already + * materialised by the time detection runs, and re-fetching a remote URL here + * would issue a second network request behind the caller's back. + */ +async function readFileInputBytes(file: FileInput): Promise { + try { + if (isFileWithMetadata(file)) { + return file.buffer; + } + if (Buffer.isBuffer(file)) { + return file; + } + if (typeof file === "string") { + const { readFile, stat } = await import("node:fs/promises"); + // Two different hangs live here, and they need different guards. + // + // A FIFO or device node blocks inside the read syscall, where neither a + // timeout nor an abort can reach it — measured: with a signal attached, + // a blocked FIFO read stays pending through both open and mid-read. The + // timeout would return while that read sat there forever. So refuse + // anything that is not a regular file up front; that is the only thing + // that actually prevents this case. + // + // A regular file on a slow or hung mount does return control between + // chunks, so there the signal works (measured: rejects with AbortError + // in flight) — and it matters, because racing the promise alone leaves + // the read filling a buffer nobody will collect. Aborting in `finally` + // covers both exits; after a resolved read it is a no-op. + // + // stat-then-read is a TOCTOU window, but a narrow one, and it is strictly + // better than the unbounded read it replaces. + const stats = await withTimeout(stat(file), FILE_READ_TIMEOUT_MS); + if (!stats.isFile()) { + return null; + } + const controller = new AbortController(); + try { + return await withTimeout( + readFile(file, { signal: controller.signal }), + FILE_READ_TIMEOUT_MS, + ); + } finally { + controller.abort(); + } + } + } catch { + // An unreadable file is not an error here: the metadata summary was + // already appended, so the caller degrades to previous behaviour. This + // covers the missing-file case that an `existsSync` pre-check used to + // (without the TOCTOU gap between check and read) and the timeout above. + } + return null; +} + /** * Append a detected file result to options.input based on its type. * Handles CSV, SVG, image, PDF, video, audio, archive, xlsx, docx, pptx, text, and unknown types. */ -function appendDetectedFileResult( +async function appendDetectedFileResult( result: { type: string; content: string | Buffer; @@ -836,7 +932,7 @@ function appendDetectedFileResult( }, file: FileInput, options: GenerateOptions, -): void { +): Promise { options.input ??= {}; const filename = extractFilename(file); @@ -887,6 +983,16 @@ function appendDetectedFileResult( if (result.content) { options.input.text += `\n\n## Audio File: "${filename}"\n${result.content}\n`; } + // Carry the bytes forward as well as the summary. Whether they are used is + // decided later, per provider: one that can listen receives the audio, one + // that cannot still gets the summary above and is no worse off than before. + const audioBytes = await readFileInputBytes(file); + if (audioBytes) { + options.input.nativeAudioFiles = [ + ...(options.input.nativeAudioFiles || []), + { buffer: audioBytes, filename, mimeType: result.mimeType }, + ]; + } if (result.images && result.images.length > 0) { options.input.images = [ ...(options.input.images || []), @@ -1023,6 +1129,24 @@ function warnIfVideoTranscriptionRequested( * `input.files` — without this, mimetype-hint and text-file inputs * would silently never reach the model on those paths. */ +/** + * Record that one file entry has been folded into an input. + * + * Marked per entry as each completes, not per run: the loop throws on the + * first file it cannot process (#273, fail loud), and the SDK's own retry path + * re-invokes this function with the same input. Marking the whole run on entry + * would make that retry a silent no-op — permanently skipping the failed file + * and shipping a half-populated request with nothing surfaced. Marking the + * whole run on exit would instead re-process the files that had already + * succeeded, duplicating them. Per entry is the only version that is right in + * both directions. + */ +function markFileProcessed(input: object, entry: unknown): void { + const processed = PREPROCESSED_FILES.get(input) ?? new Set(); + processed.add(entry); + PREPROCESSED_FILES.set(input, processed); +} + export async function processUnifiedFilesArray( options: GenerateOptions, maxSize: number, @@ -1033,8 +1157,36 @@ export async function processUnifiedFilesArray( return; } - const totalFiles = options.input.files.length; - const files = options.input.files; + // Every result this function produces is *appended* — the summary onto + // `text`, the bytes onto `nativeAudioFiles`/`images`/`pdfFiles` — and + // `files` is only ever read, never consumed. Running it twice over the same + // entry therefore doubles the injected text and attaches the same recording + // twice, which a provider sees as two distinct files. + // + // That is reachable: providers whose native paths preprocess in both + // `generate()` and `executeStream()` share one `options.input` reference + // with `BaseProvider`'s real-stream → fake-stream fallback, so a retried + // stream runs this a second time over the same object. + // + // Tracked per *entry* rather than per input, because a caller that appends a + // file to an input it has already used must still get the new one processed + // — treating the whole input as done would silently drop it. Entries are + // compared by identity (or by value for a path string), which is what a + // repeat of the same attachment actually looks like. + const alreadyProcessed = + PREPROCESSED_FILES.get(options.input) ?? new Set(); + const pending = options.input.files.filter( + (entry) => !alreadyProcessed.has(entry), + ); + if (pending.length === 0) { + logger.debug( + "[NEUROLINK] Every attached file has already been processed for this input — skipping to avoid duplicate attachments", + ); + return; + } + + const totalFiles = pending.length; + const files = pending; warnIfVideoTranscriptionRequested(options.videoOptions); @@ -1086,6 +1238,7 @@ export async function processUnifiedFilesArray( `[NEUROLINK] File lazily registered: ${filename} (${fileSize} bytes) — deferred processing`, ); includedCount++; + markFileProcessed(inp2, file); continue; } } @@ -1099,6 +1252,16 @@ export async function processUnifiedFilesArray( const fileMimetypeHint = isFileWithMetadata(file) ? file.mimetype : undefined; + // The name has to travel the same way, and for the same reason: the + // line above unwraps the object to its buffer, so by the time + // detection resolves an extension there is no name left to read one + // from. Without this a `.tar` supplied as bytes-plus-name is + // unidentifiable — its "ustar" marker sits at byte 257, not at + // offset 0 — and reports "Could not extract content" for an archive + // that extracts perfectly when handed its filename. + const fileFilenameHint = isFileWithMetadata(file) + ? file.filename + : undefined; const result = await FileDetector.detectAndProcess(rawFileInput, { maxSize: genericFileMaxSize, allowedTypes: [ @@ -1127,9 +1290,10 @@ export async function processUnifiedFilesArray( : undefined, provider: provider, mimetypeHint: fileMimetypeHint, + filenameHint: fileFilenameHint, }); - appendDetectedFileResult(result, file, options); + await appendDetectedFileResult(result, file, options); includedCount++; // Log what content type was added to the message @@ -1137,6 +1301,7 @@ export async function processUnifiedFilesArray( logger.info( `[NEUROLINK] File added to message: ${filename} as ${contentType} (type: ${result.type})`, ); + markFileProcessed(inp2, file); } catch (error) { const errMsg = error instanceof Error ? error.message : String(error); // #273: don't silently drop a failed file — log, then throw so the @@ -1600,8 +1765,16 @@ export async function buildMultimodalMessagesArray( pdfFiles.length > 0 || !!(inp.content && inp.content.some((c) => c.type === "pdf")); - // If no images or PDFs, use standard message building and convert to MultimodalChatMessage[] - if (!hasImages && !hasPDFs) { + // Audio that is to be delivered natively is multimodal for the same reason a + // PDF is: it becomes a non-text part. Without this an audio-only turn — the + // ordinary "transcribe this recording" request — took the text-only branch + // below, where the collected bytes have nowhere to go and only the metadata + // summary survives. + const hasNativeAudio = + (inp.nativeAudioFiles?.length ?? 0) > 0 && supportsNativeAudio(provider); + + // If no images, PDFs or audio, use standard message building and convert to MultimodalChatMessage[] + if (!hasImages && !hasPDFs && !hasNativeAudio) { // #289: CSV content[] items don't need vision, so they never reach the // multimodal converter below — process them into the prompt text here // (otherwise a `content: [{type:"csv"}]`-only request silently drops it). @@ -1730,19 +1903,35 @@ export async function buildMultimodalMessagesArray( let userContent: string | unknown; if (inp.content && inp.content.length > 0) { + // Audio detected from `input.files` has to reach this branch too. A + // caller that supplies structured `content` AND attaches an audio file + // is not asking for the audio to be discarded — but this branch bypasses + // the multimodal converter below, so the bytes were dropped and only the + // metadata summary folded into `text` survived. Exactly the failure this + // change exists to remove, reintroduced through the other door. userContent = await convertContentToProviderFormat( inp.content, provider, model, options.pdfOptions, + inp.nativeAudioFiles ?? [], ); - } else if ((inp.images && inp.images.length > 0) || pdfFiles.length > 0) { + } else if ( + (inp.images && inp.images.length > 0) || + pdfFiles.length > 0 || + // Audio alone must still take the multimodal path. Without this clause an + // audio-only turn fell through to the plain-text branch below, so the + // recording was dropped and only its metadata summary — already folded + // into `text` — ever reached the model. + (inp.nativeAudioFiles?.length ?? 0) > 0 + ) { userContent = await convertMultimodalToProviderFormat( inp.text ?? "", inp.images || [], pdfFiles, provider, model, + inp.nativeAudioFiles ?? [], ); } else { userContent = inp.text; @@ -1836,6 +2025,7 @@ async function convertContentToProviderFormat( provider: string, _model: string, pdfOptions?: GenerateOptions["pdfOptions"], + audioFiles: MultimodalAudioEntry[] = [], ): Promise { const textContent = content.find((c) => c.type === "text"); const imageContent = content.filter((c) => c.type === "image"); @@ -1850,15 +2040,28 @@ async function convertContentToProviderFormat( text = await appendCsvContentToText(csvContent, text); } - const hasMultimodal = imageContent.length > 0 || pdfContent.length > 0; + // Audio the provider can actually read counts as multimodal content, on both + // of the checks below. Computed before the validation rather than after it: + // a request whose structured `content` carries no text but does carry an + // attached recording is a complete request, and leaving audio out of + // `hasMultimodal` rejected it as empty before delivery could be considered. + // + // The same flag then keeps it off the text-only early return, which would + // otherwise hand back a plain string and lose the bytes — the exact drop this + // change exists to stop, reached through a different branch. Gated on the + // provider accepting audio so one that cannot keeps the cheaper plain-text + // shape rather than an array carrying a part it will ignore. + const deliversAudio = audioFiles.length > 0 && supportsNativeAudio(provider); + const hasMultimodal = + imageContent.length > 0 || pdfContent.length > 0 || deliversAudio; // Validate that we have at least some content if (!hasMultimodal && !text) { throw new Error("Content must include either text or multimodal content"); } - // Text-only case (CSV has already been folded into `text`) - if (imageContent.length === 0 && pdfContent.length === 0) { + // Text-only case (CSV has already been folded into `text`). + if (!hasMultimodal) { return text; } @@ -1893,6 +2096,7 @@ async function convertContentToProviderFormat( pdfFiles, provider, _model, + audioFiles, ); } @@ -2120,14 +2324,21 @@ function detectMimeTypeFromBuffer(buffer: Buffer): string | undefined { return "image/svg+xml"; } - // JPEG 2000: 12-byte signature box "....jP ". + // JPEG 2000: the full 12-byte signature box, trailing 0D 0A 87 0A included. + // Those four bytes are a line-ending probe that a transfer mangling newlines + // or stripping the eighth bit corrupts, so checking only length and brand + // accepts exactly the damaged files the signature exists to reject. if ( - buffer.length >= 8 && + buffer.length >= 12 && buffer[0] === 0x00 && buffer[1] === 0x00 && buffer[2] === 0x00 && buffer[3] === 0x0c && - buffer.toString("latin1", 4, 8) === "jP " + buffer.toString("latin1", 4, 8) === "jP " && + buffer[8] === 0x0d && + buffer[9] === 0x0a && + buffer[10] === 0x87 && + buffer[11] === 0x0a ) { return "image/jp2"; } @@ -2257,10 +2468,17 @@ async function processImageToBase64( // arrive as URLs are downloaded further downstream and only become bytes // here. A no-op for the universal formats, so the common path is unaffected. if (needsVisionTranscode(mimeType)) { - const compatible = await toVisionCompatibleImage( - Buffer.from(imageData, "base64"), - mimeType, - ); + // Guard the decoded bytes, not just the Buffer input. The buffer branch + // above is already checked, but a data: URI reaches here having only been + // regex-matched — so an oversized one was handed straight to sharp/ffmpeg, + // which decode it in full. + // + // Sized before the decode rather than after: checking the Buffer would mean + // allocating the very thing the limit exists to refuse. + const context = `image input at index ${index}`; + ImageProcessor.validateSize(base64DecodedByteLength(imageData), context); + const rawImage = Buffer.from(imageData, "base64"); + const compatible = await toVisionCompatibleImage(rawImage, mimeType); if (compatible.converted) { imageData = compatible.buffer.toString("base64"); mimeType = compatible.mimeType; @@ -2329,21 +2547,95 @@ export async function normalizeVisionImageFormats( * conversion. Reading every attached .png off disk just to confirm it is * already compatible would double the I/O of the common case for no benefit. */ +/** + * Byte length `Buffer.from(b64, "base64")` will allocate, computed without + * allocating it. + * + * Base64 encodes 3 bytes per 4 characters, so the encoded length settles the + * decoded length up front. The point is ordering: a size guard applied to the + * decoded Buffer has already paid for the allocation it exists to prevent, so + * every base64 site here checks this first and decodes second — the same shape + * as the file-path branch, which stats before it reads. + * + * This bounds the decode, not the whole request: the encoded string is already + * resident by the time we see it, so an oversized payload still costs its own + * length in memory. What it removes is the second, larger allocation on top. + * + * Whitespace is skipped because the decoder ignores it, which keeps the count + * exact rather than a conservative over-estimate that would reject legitimate + * payloads sitting just under the limit. + */ +function base64DecodedByteLength(base64: string): number { + let significant = 0; + let padding = 0; + for (let i = 0; i < base64.length; i++) { + const code = base64.charCodeAt(i); + if (code === 32 || code === 9 || code === 10 || code === 13) { + continue; + } + significant++; + if (code === 61) { + padding++; + } + } + return Math.max(0, Math.floor(significant / 4) * 3 - padding); +} + +/** + * Whether a payload is small enough to hand to an image decoder, reported + * rather than thrown. See {@link readImageSourceForConversion} for why this + * pass degrades instead of failing. + */ +function withinConversionByteLimit(bytes: number, context: string): boolean { + try { + ImageProcessor.validateSize(bytes, context); + return true; + } catch (error) { + logger.warn( + `[messageBuilder] Skipping vision-format conversion for ${context}: ` + + `${error instanceof Error ? error.message : String(error)}`, + ); + return false; + } +} + +function withinConversionLimit(buffer: Buffer, context: string): boolean { + return withinConversionByteLimit(buffer.length, context); +} + async function readImageSourceForConversion( payload: Buffer | string, ): Promise<{ buffer: Buffer; mimeType: string } | undefined> { + // Both in-memory shapes are size-checked before they can reach a decoder, + // the same way the file-path branch below is. `withinConversionLimit` + // reports rather than throws, because this whole pass is best-effort: an + // image too large to convert is left in its original format for the + // downstream guard to reject, which is what an unconvertible image already + // does. Throwing here would turn a normalisation step into a hard failure. if (Buffer.isBuffer(payload)) { const mimeType = detectMimeTypeFromBuffer(payload); - return mimeType ? { buffer: payload, mimeType } : undefined; + if (!mimeType || !withinConversionLimit(payload, "image buffer")) { + return undefined; + } + return { buffer: payload, mimeType }; } if (typeof payload !== "string") { return undefined; } if (payload.startsWith("data:")) { const match = payload.match(/^data:([^;]+);base64,(.+)$/); - return match - ? { buffer: Buffer.from(match[2], "base64"), mimeType: match[1] } - : undefined; + if (!match) { + return undefined; + } + if ( + !withinConversionByteLimit( + base64DecodedByteLength(match[2]), + "image data URI", + ) + ) { + return undefined; + } + return { buffer: Buffer.from(match[2], "base64"), mimeType: match[1] }; } if (isInternetUrl(payload)) { return undefined; @@ -2489,6 +2781,7 @@ async function convertMultimodalToProviderFormat( pdfFiles: MultimodalPdfEntry[], provider: string, model: string, + audioFiles: MultimodalAudioEntry[] = [], ): Promise> { const content: Array = [ { type: "text", text }, @@ -2511,6 +2804,46 @@ async function convertMultimodalToProviderFormat( } } + // Attach audio to providers that can listen. The metadata summary was + // already appended to `text` during detection, so this adds the recording + // itself rather than replacing the description — a model asked "how long is + // this?" keeps the exact answer, and one asked "what is said?" can now + // answer at all. + if (audioFiles.length > 0 && supportsNativeAudio(provider)) { + for (const audio of audioFiles) { + // Derived from the trimmed basename so a directory containing a dot + // (`/srv/v1.2/recording`) cannot be mistaken for the file's extension. + const base = safeBasename(audio.filename); + const dot = base.lastIndexOf("."); + const extension = dot > 0 ? base.slice(dot) : ".bin"; + const compatible = await toProviderCompatibleAudio( + audio.buffer, + audio.mimeType, + extension, + ); + // A conversion that could not run leaves a container the provider does + // not accept. Sending it anyway turns a metadata answer into an opaque + // HTTP 400, so it is skipped and the summary stands. + if (needsAudioTranscode(compatible.mimeType)) { + logger.warn( + `[Audio] Skipping native delivery of ${base}: ` + + `${compatible.mimeType} is not accepted by ${provider} and could ` + + `not be converted. The metadata summary was still included.`, + ); + continue; + } + content.push({ + type: "file" as const, + data: compatible.buffer, + mediaType: compatible.mimeType, + }); + logger.info( + `[Audio] ✅ Added to content (native audio): ${base}` + + `${compatible.converted ? ` (converted to ${compatible.mimeType})` : ""}`, + ); + } + } + // Check if provider supports native PDF processing const supportsNativePDF = PDFProcessor.supportsNativePDF(provider); @@ -2766,17 +3099,18 @@ function getFileSource(file: FileInput): "buffer" | "path" | "url" | "datauri" { * 10 KB is far below any real photo, so this affected essentially every image * attached by path — an ordinary JPEG, not just unusual formats. * - * Scoped to images deliberately, and audio deserves spelling out because the - * reason is not the one it appears to be. An audio file's message contains only - * a metadata block — duration, codec, sample rate — on BOTH paths; no audio - * bytes are handed to the provider either way. Moving audio to the eager path - * would therefore change nothing about what the model receives. That audio - * content never reaches the model at all is a separate and pre-existing gap, - * not something this threshold decision can repair, and it is easy to mistake - * for working code because the metadata block answers exactly the questions - * ("how long is it?", "what sample rate?") a test is most tempted to ask. - * Video is left alone for the opposite reason: its frames are measurably - * present in the message and a model reads them correctly. + * Images were only the most visible case. The same reasoning generalises to + * every type whose content is not text, which is why the rule below is stated + * as an exclusion — preview lazily what a text slice can faithfully represent, + * process everything else — rather than as a list of media types. See + * {@link isEagerType} for what that costs and buys per modality. + * + * Audio deserves a note because it is the case most likely to look fine while + * being broken: its message carries a metadata block — duration, codec, sample + * rate — that answers exactly the questions ("how long is it?", "what sample + * rate?") a test is most tempted to ask, and answers them correctly with no + * audio whatsoever attached. A test asking those questions passes against a + * model that received nothing. * * Note this costs no extra memory: `tryRegisterFileReference` already calls * `getFileBuffer()` and reads the whole file to register it. The lazy path was @@ -2785,48 +3119,72 @@ function getFileSource(file: FileInput): "buffer" | "path" | "url" | "datauri" { */ function isEagerMultimodalFile(file: FileInput): boolean { if (typeof file === "string") { - return isImageLikeType(inferFileTypeFromExtension(file)); + return isEagerType(inferFileTypeFromExtension(file)); } if (Buffer.isBuffer(file)) { - return isImageLikeType(inferFileTypeFromBuffer(file)); + return isEagerType(inferFileTypeFromBuffer(file)); } - // A `FileWithMetadata` carries two independent declarations, and either one - // alone is enough: the shape exists for Slack/Curator-style uploads that + // A `FileWithMetadata` carries two independent declarations, and either is + // enough on its own: the shape exists for Slack/Curator-style uploads that // arrive as bytes plus a mimetype, so its `filename` may be extensionless or - // simply wrong. Reading them as a `??` chain meant the first *recognised* - // name won outright — `upload.pdf` with `mimetype: "image/png"` classified as - // a PDF and lost its pixels down the lazy path. Any declaration of an image - // is therefore decisive. + // simply wrong. Reading them as a `??` chain let the first *recognised* name + // win outright — `recording.txt` with `mimetype: "audio/mpeg"` classified as + // text, stayed lazy, and never reached the native-audio path this change + // exists to feed. A file is kept lazy only when nothing about it disagrees. const declared = [ inferFileTypeFromExtension(file.filename), inferFileTypeFromMimetype(file.mimetype), ]; - if (declared.some(isImageLikeType)) { + if (declared.some(isEagerType)) { return true; } - // The buffer sniff is the last resort rather than a third vote, because it is - // a substring scan: an HTML page with an inline `` icon in its head - // would otherwise be pulled onto the eager path and sent in full, which is - // the opposite of what the size tiers are for. It only speaks when nothing - // else did. + // The buffer sniff is a last resort rather than a third vote, because it is + // partly a substring scan: an HTML page with an inline `` icon in its + // head would otherwise be pulled onto the eager path and sent in full, which + // is the opposite of what the size tiers are for. It speaks only when neither + // declaration did. return declared.every((type) => type === undefined) - ? isImageLikeType(inferFileTypeFromBuffer(file.buffer)) + ? isEagerType(inferFileTypeFromBuffer(file.buffer)) : false; } /** - * Whether a routing type should have its bytes preserved rather than previewed. + * Whether a routing type must be processed rather than previewed. * * "svg" is a separate routing type rather than a sub-case of "image" (it goes * to the sanitizer, not to a vision encoder), but it is still an image as far * as this decision is concerned: its markup IS its content, and previewing it * away leaves the model with nothing. Accepting both keeps this correct * whichever of the two type vocabularies the caller's map uses. + * + * Audio and video join them, for one shared reason: the lazy path never runs + * the detection branch that produces their model-visible content — the decoded + * audio buffer, the extracted video keyframes — so a lazily registered file is + * summarised away no matter what the dispatch side is willing to send. + * + * Video is the clearest demonstration that this is a delivery problem and not a + * format one. Keyframe extraction is identical across containers (three frames, + * ~38 KB each, for every one of mp4/wmv/flv/mpg/m2ts), and a frame pulled from + * any of them shows the test token perfectly legibly. Yet only mp4 answered + * correctly, because Gemini accepts an mp4 natively and never needed the + * frames; every container it cannot decode returned NOTHING_RECEIVED, since the + * frames that would have carried the answer were never extracted. Making video + * eager fixed all four at once. + * + * Documents and archives are here for the same reason once removed. The lazy + * preview is a truncated slice of the raw bytes, so it is only ever faithful + * for a file that IS text. A .rtf sliced raw is RTF control words, and a + * .bz2/.xz/.zst is compressed bytes — the model was told "binary file of + * unknown type" about files whose processors extract them cleanly. + * + * Which leaves plain text and CSV on the lazy path, and they are exactly the + * cases it was built for: a truncated sample of a large CSV is a faithful + * sample, and the file tools can read the rest on demand. */ -function isImageLikeType(type: string | undefined): boolean { - return type === "image" || type === "svg"; +function isEagerType(type: string | undefined): boolean { + return type !== undefined && type !== "text" && type !== "csv"; } /** @@ -2842,10 +3200,24 @@ function inferFileTypeFromMimetype(mimetype?: string): string | undefined { return undefined; } const normalized = mimetype.split(";")[0].trim().toLowerCase(); - if (normalized === "image/svg+xml") { - return "svg"; + // "application/octet-stream" is deliberately absent from the registry side of + // this lookup: it is the opaque sentinel a caller sends when it knows + // nothing, not a claim about content, and treating it as one would let a + // shrugging uploader force a routing decision. + if (normalized === "application/octet-stream") { + return undefined; } - return normalized.startsWith("image/") ? "image" : undefined; + const exact = MIMETYPE_TYPE_MAP[normalized]; + if (exact) { + return exact; + } + // A family fallback for types the registry does not enumerate — `audio/webm`, + // `image/x-something`. The leading segment is enough to decide eager vs lazy + // even when the exact codec is unknown to us. + const family = normalized.split("/")[0]; + return family === "image" || family === "audio" || family === "video" + ? family + : undefined; } /** diff --git a/test/continuous-test-suite-audio.ts b/test/continuous-test-suite-audio.ts index 95ff057ab..449195e48 100644 --- a/test/continuous-test-suite-audio.ts +++ b/test/continuous-test-suite-audio.ts @@ -34,6 +34,12 @@ import { isAudioFile, } from "../src/lib/processors/media/AudioProcessor.js"; import { FileDetector } from "../src/lib/utils/fileDetector.js"; +import { buildUserPartsWithMultimodal } from "../src/lib/providers/googleNativeGemini3/utils.js"; +import { + needsAudioTranscode, + supportsNativeAudio, + toProviderCompatibleAudio, +} from "../src/lib/adapters/audioFormatSupport.js"; const { test, runSuite } = defineSuite("Audio file support"); @@ -301,6 +307,319 @@ await test("textContent names the file so the model has context", async () => { ); }); +// --- Native audio delivery: capability map and conversion contract --------- +// +// The live format suite proves audio *arrives*, but it can only do so for the +// containers this machine can encode and only when credentials are present. +// These run offline and pin the decision logic itself — which provider is +// offered raw bytes, which container is re-encoded first, and the promise the +// converter makes to its caller when it cannot do the job. + +await test("file preprocessing is idempotent for a reused input object", async () => { + // Everything this produces is appended — the summary onto `text`, the bytes + // onto `nativeAudioFiles` — and `files` is never consumed, so a second pass + // over the same input attaches the same recording twice and doubles the + // injected text. That is reachable: a provider preprocessing in both + // generate() and executeStream() shares one input reference with + // BaseProvider's real-stream → fake-stream fallback, so a retried stream + // runs it again over the same object. + await ensureFixtures(); + const { processUnifiedFilesArray } = + await import("../src/lib/utils/messageBuilder.js"); + const options = { + input: { + text: "Describe this audio.", + files: [path.join(dir, "tone.mp3")], + }, + }; + const run = async () => + processUnifiedFilesArray( + options as unknown as Parameters[0], + 100 * 1024 * 1024, + "google-ai-studio", + ); + + await run(); + const input = options.input as { + text?: string; + nativeAudioFiles?: unknown[]; + }; + const audioAfterFirst = input.nativeAudioFiles?.length ?? 0; + assert(audioAfterFirst === 1, "the first pass collects the recording once"); + + await run(); + assertEqual( + input.nativeAudioFiles?.length ?? 0, + audioAfterFirst, + "a second pass does not attach the same recording again", + ); + assertEqual( + (input.text ?? "").split("## Audio File").length - 1, + 1, + "a second pass does not duplicate the injected metadata summary", + ); +}); + +await test("a failed file still fails loud on retry, and good files are not re-added", async () => { + // The dedup guard must not swallow a retry. The loop throws on the first + // file it cannot process (#273, fail loud) and the SDK's own retry path + // re-invokes with the same input, so marking the whole run as done on entry + // would turn attempt two into a silent success with the bad file simply + // missing — the failure class this PR exists to remove. Marking on exit + // instead would re-add the files that already succeeded. Per entry is the + // only version that is right both ways, and this pins both directions. + await ensureFixtures(); + const { processUnifiedFilesArray } = + await import("../src/lib/utils/messageBuilder.js"); + const good = path.join(dir, "good-marker.txt"); + fs.writeFileSync(good, "GOODMARKER line\n".repeat(60)); + const options = { + input: { + text: "Read these.", + files: [good, path.join(dir, "definitely-absent.pdf")], + }, + }; + const run = () => + processUnifiedFilesArray( + options as unknown as Parameters[0], + 100 * 1024 * 1024, + "google-ai-studio", + ); + + let firstThrew = false; + try { + await run(); + } catch { + firstThrew = true; + } + assert(firstThrew, "the first attempt fails loud on the unreadable file"); + const input = options.input as { text?: string }; + const occurrencesAfterFirst = + (input.text ?? "").split("GOODMARKER").length - 1; + + let secondThrew = false; + try { + await run(); + } catch { + secondThrew = true; + } + assert( + secondThrew, + "the retry fails loud too rather than silently skipping the bad file", + ); + assertEqual( + (input.text ?? "").split("GOODMARKER").length - 1, + occurrencesAfterFirst, + "the file that already succeeded is not added a second time", + ); +}); + +await test("AI Studio's stream path preprocesses files like its generate path", async () => { + // AI Studio overrides both entry points and goes straight to the native SDK, + // so neither reaches the shared message builder. generate() ran the file + // preprocessing inline and stream() did not, which dropped every attached + // file on the streaming path — the audio bytes AND the metadata summary. + // Asserted offline: preprocessing mutates options.input in place before any + // network call, so the credential failure that follows is irrelevant here. + await ensureFixtures(); + const previousKey = process.env.GOOGLE_AI_API_KEY; + process.env.GOOGLE_AI_API_KEY = + previousKey?.trim() || "not-a-real-key-no-request-succeeds"; + try { + const { GoogleAIStudioProvider } = + await import("../src/lib/providers/googleAiStudio/client.js"); + const provider = new GoogleAIStudioProvider(); + const options = { + input: { + text: "Describe this audio.", + files: [path.join(dir, "tone.mp3")], + }, + disableTools: true, + }; + try { + await provider.stream( + options as unknown as Parameters[0], + ); + } catch { + // Expected: the request itself cannot succeed here. Preprocessing has + // already run by then, which is the whole point. + } + const input = options.input as { + text?: string; + nativeAudioFiles?: unknown[]; + }; + assert( + (input.nativeAudioFiles?.length ?? 0) > 0, + "the streaming path collected the audio bytes rather than dropping them", + ); + assertIncludes( + input.text ?? "", + "## Audio File", + "the streaming path also injected the metadata summary", + ); + } finally { + if (previousKey === undefined) { + delete process.env.GOOGLE_AI_API_KEY; + } else { + process.env.GOOGLE_AI_API_KEY = previousKey; + } + } +}); + +await test("the AI Studio request builder attaches audio, not just Vertex", async () => { + // The capability map claims Gemini on BOTH front ends, but AI Studio + // overrides generate()/stream() and assembles its request through + // buildUserPartsWithMultimodal rather than the shared message builder — so + // wiring audio into the Vertex client alone left this path advertising + // support and then dropping the bytes. Asserting on the parts the provider + // would actually send is the only way to catch that; a supportsNativeAudio() + // check passes either way, which is exactly why it went unnoticed. + const parts = await buildUserPartsWithMultimodal( + { + text: "Please transcribe this.", + nativeAudioFiles: [ + { + buffer: Buffer.from("pretend mp3 bytes, enough of them to measure"), + filename: "memo.mp3", + mimeType: "audio/mpeg", + }, + ], + }, + "Please transcribe this.", + "[audio-suite]", + ); + const audioPart = parts.find( + (part) => + "inlineData" in part && + part.inlineData && + (part.inlineData as { mimeType: string }).mimeType.startsWith("audio/"), + ); + assert( + audioPart !== undefined, + "the AI Studio part builder attaches the recording as inlineData", + ); + const payload = + audioPart && "inlineData" in audioPart + ? ((audioPart.inlineData as { data: string }).data ?? "") + : ""; + assert( + Buffer.from(payload, "base64").length > 0, + "the inlineData part carries the audio bytes rather than an empty payload", + ); +}); + +await test("provider capability is matched on normalised names", () => { + // Every spelling the registry and CLI accept for the same provider, because + // this gates whether audio bytes are sent at all: a missed alias silently + // downgrades the turn to a metadata summary. + for (const provider of [ + "vertex", + "google-vertex", + "GoogleVertex", + " gemini ", + "google-ai-studio", + ]) { + assert( + supportsNativeAudio(provider), + `a Gemini-family provider spelling is recognised as accepting audio`, + ); + } + for (const provider of ["openai", "anthropic", "bedrock", "mistral", ""]) { + assert( + !supportsNativeAudio(provider), + `a provider with no inline-audio support is not offered raw bytes`, + ); + } +}); + +await test("transcode is required exactly outside Gemini's accepted set", () => { + // Gemini's documented set passes through untouched... + for (const mime of [ + "audio/wav", + "audio/mpeg", + "audio/aiff", + "audio/aac", + "audio/ogg", + "audio/flac", + "AUDIO/MPEG", + "audio/wav; codecs=1", + ]) { + assert( + !needsAudioTranscode(mime), + `an accepted container is passed through without re-encoding`, + ); + } + // ...and everything else is converted rather than rejected, which is the + // whole point: the container a voice memo happens to use says nothing about + // whether the audio inside is worth hearing. + for (const mime of [ + "audio/x-caf", + "audio/x-ms-wma", + "audio/x-wavpack", + "audio/basic", + "audio/amr", + ]) { + assert( + needsAudioTranscode(mime), + `an unsupported container is routed through conversion`, + ); + } +}); + +await test("an already-native container is returned untouched", async () => { + // No ffmpeg involved on this path, so it holds on any machine. + const bytes = Buffer.from("not really audio, and deliberately so"); + const result = await toProviderCompatibleAudio(bytes, "audio/mpeg", ".mp3"); + assert(!result.converted, "an accepted container reports no conversion"); + assertEqual(result.mimeType, "audio/mpeg", "the MIME type is preserved"); + assert(result.buffer === bytes, "the original buffer is passed through"); +}); + +await test("a mimetype parameter does not defeat the native check", async () => { + const bytes = Buffer.from("still not audio"); + const result = await toProviderCompatibleAudio( + bytes, + "audio/mpeg; codecs=mp3", + ".mp3", + ); + assert(!result.converted, "the parameter is stripped before the lookup"); + assertEqual(result.mimeType, "audio/mpeg", "the normalised type comes back"); +}); + +await test("an unconvertible input degrades instead of throwing", async () => { + // The contract the caller depends on: this never throws for audio reasons. + // Bytes that are not audio in a container that would need conversion is the + // worst case — ffmpeg fails, or is absent entirely — and both must surface + // as `converted: false` so the caller falls back to the metadata summary + // rather than failing the generation. + const garbage = Buffer.from("00000000 definitely not a WMA stream"); + const result = await toProviderCompatibleAudio( + garbage, + "audio/x-ms-wma", + ".wma", + ); + assert(!result.converted, "a failed conversion reports converted: false"); + assert(result.buffer === garbage, "the original bytes are handed back"); + assertEqual( + result.mimeType, + "audio/x-ms-wma", + "the original MIME type is preserved so the caller can still skip delivery", + ); +}); + +await test("an empty buffer is handled on the same degrading path", async () => { + // Raised in review as a case worth pre-checking. It needs no special guard: + // an empty stream either fails in ffmpeg or produces empty output, and both + // are already funnelled into the same fallback. + const result = await toProviderCompatibleAudio( + Buffer.alloc(0), + "audio/x-ms-wma", + ".wma", + ); + assert(!result.converted, "an empty input cannot be converted"); + assertEqual(result.buffer.length, 0, "the empty buffer is returned as-is"); +}); + // Best-effort cleanup; the OS reclaims the temp dir regardless. try { fs.rmSync(dir, { recursive: true, force: true }); diff --git a/test/continuous-test-suite-file-formats.ts b/test/continuous-test-suite-file-formats.ts index 6d9ed3701..470bd67ca 100644 --- a/test/continuous-test-suite-file-formats.ts +++ b/test/continuous-test-suite-file-formats.ts @@ -349,11 +349,19 @@ for (const { ext, modality } of FIXTURE_FORMATS) { } requireLive(); + // Kept, but no longer about which path the file takes: this change routes + // every recognised non-text type eagerly whatever its size, so the tier + // threshold no longer decides anything for these formats. What the floor + // still buys is that each fixture is a realistic file rather than a toy + // one — the size class that was actually broken, and the class a fixture + // would silently drift out of if a generator started emitting stubs. const size = fs.statSync(file).size; + if (size <= SIZE_TIER_THRESHOLDS.TINY_MAX) { + console.error(` ↳ ${ext} fixture size: ${size} bytes`); + } assert( size > SIZE_TIER_THRESHOLDS.TINY_MAX, - `the ${ext} fixture is ${size} bytes, under the tier threshold — it ` + - `would exercise the eager path and prove nothing about the lazy one`, + `the ${ext} fixture is below the realistic-size floor`, ); const nl = new NeuroLink(); @@ -420,6 +428,91 @@ for (const ext of [".png", ".pdf"]) { }); } +// --- Classification from a declaration rather than a name ------------------ + +await test("a mislabelled filename loses to the declared mimetype", async () => { + // The two declarations on a `FileWithMetadata` were read as a `??` chain, so + // the first *recognised* one won outright: a name ending `.txt` beat a + // `mimetype` of `audio/mpeg`, the file stayed on the lazy path, and the bytes + // that native audio delivery exists to send were never collected. Naming is + // the least trustworthy field on that shape, which is why it cannot veto. + const file = fixtures.get(".mp3"); + if (!file) { + throw new Skip("mp3 cannot be encoded in this environment"); + } + requireLive(); + + const buffer = fs.readFileSync(file); + assert( + buffer.length > SIZE_TIER_THRESHOLDS.TINY_MAX, + "the mp3 fixture is under the tier threshold — it would exercise the " + + "eager path and prove nothing about the lazy one", + ); + + const nl = new NeuroLink(); + const content = await generateNonEmpty(nl, { + input: { + text: PROMPTS.audio, + files: [{ buffer, filename: "recording.txt", mimetype: "audio/mpeg" }], + }, + provider: PROVIDER, + maxTokens: 256, + }); + + assertReply( + !content.includes("NOTHING_RECEIVED"), + "the mislabelled audio never reached the model", + content, + ); + assertReply( + normalizeDigits(content).includes(TOKEN), + "the model did not report the mislabelled audio's spoken code", + content, + ); +}); + +await test("a bytes-plus-name upload keeps the name detection routes on", async () => { + // A `.tar` is the sharpest case: it has no magic bytes at offset 0 — its + // "ustar" marker sits at byte 257 — so its name is the only thing that + // identifies it. The unified path unwraps a `FileWithMetadata` to its buffer + // before detection runs, which threw the name away and left the archive + // unidentifiable ("Could not extract content") even though the same bytes + // extract perfectly when the name travels with them. + const file = fixtures.get(".tar"); + if (!file) { + throw new Skip("tar cannot be built in this environment"); + } + requireLive(); + + const buffer = fs.readFileSync(file); + assert( + buffer.length > SIZE_TIER_THRESHOLDS.TINY_MAX, + "the tar fixture is under the tier threshold — it would exercise the " + + "eager path and prove nothing about the lazy one", + ); + + const nl = new NeuroLink(); + const content = await generateNonEmpty(nl, { + input: { + text: PROMPTS.archive, + files: [{ buffer, filename: "bundle.tar" }], + }, + provider: PROVIDER, + maxTokens: 256, + }); + + assertReply( + !content.includes("NOTHING_RECEIVED"), + "the bytes-plus-name archive never reached the model", + content, + ); + assertReply( + normalizeDigits(content).includes(TOKEN), + "the model did not report the bytes-plus-name archive's hidden code", + content, + ); +}); + // Cleanup must precede runSuite(): it prints the summary and then calls // process.exit, so anything after it never runs. try { diff --git a/test/continuous-test-suite-multimodal-sdk.ts b/test/continuous-test-suite-multimodal-sdk.ts index c32c960d3..5af839fbd 100644 --- a/test/continuous-test-suite-multimodal-sdk.ts +++ b/test/continuous-test-suite-multimodal-sdk.ts @@ -43,9 +43,13 @@ import { import { hasPackage, makeDocx, makeXlsx } from "./helpers/officeFixtures.js"; import { FileDetector } from "../src/lib/utils/fileDetector.js"; import { redactUrlsInText } from "../src/lib/utils/logSanitize.js"; -import { buildMultimodalMessagesArray } from "../src/lib/utils/messageBuilder.js"; +import { + buildMultimodalMessagesArray, + normalizeVisionImageFormats, +} from "../src/lib/utils/messageBuilder.js"; import { NeuroLink } from "../src/lib/neurolink.js"; import { SIZE_TIER_THRESHOLDS } from "../src/lib/types/index.js"; +import { SIZE_LIMITS_BYTES } from "../src/lib/processors/config/sizeLimits.js"; const { test, runSuite } = defineSuite("Multimodal through the SDK"); @@ -330,6 +334,115 @@ await test("buildMultimodalMessagesArray carries audio into message content", as ); }); +await test("audio survives alongside structured input.content", async () => { + // A caller supplying `content` took a different branch that never received + // the detected audio, and that branch returns early as a plain string when + // there is no image or PDF — so an attached recording was silently reduced + // to its metadata summary. No API involved: the assertion is on the message + // shape, which is where the bytes were being lost. + await ensureMedia(); + const messages = await buildMultimodalMessagesArray( + { + input: { + text: "Describe this audio.", + content: [{ type: "text", text: "What is said in the recording?" }], + files: [path.join(dir, "tone.mp3")], + }, + } as Parameters[0], + "vertex", + "gemini-2.5-flash", + ); + const user = messages.find((message) => message.role === "user"); + const parts = user?.content; + assert( + Array.isArray(parts), + "the content branch produces parts rather than collapsing to a string", + ); + // The payload is asserted, not just the part's existence. A `file` part + // carrying no bytes is exactly the metadata-only delivery this whole change + // exists to eliminate, and it would satisfy a type-only check — the same + // shape of false pass this suite's header warns about. + const audioPart = ( + parts as Array<{ type?: string; mediaType?: string; data?: unknown }> + ).find( + (part) => part.type === "file" && part.mediaType?.startsWith("audio/"), + ); + assert( + audioPart !== undefined, + "the recording is attached as an audio file part, not merely described", + ); + assertEqual( + audioPart?.mediaType, + "audio/mpeg", + "the part declares the mp3 media type the fixture was encoded as", + ); + const payload = audioPart?.data; + const payloadBytes = + typeof payload === "string" + ? Buffer.from(payload, "base64").length + : Buffer.isBuffer(payload) + ? payload.length + : 0; + assert( + payloadBytes > 1024, + "the part carries the actual audio bytes rather than an empty placeholder", + ); +}); + +await test("structured content carrying only audio is a complete request", async () => { + // `hasMultimodal` counted images and PDFs but not audio, so a `content` array + // with no text plus an attached recording was rejected as empty — the request + // threw before delivery could even be considered. Audio the provider can read + // is content. + await ensureMedia(); + const messages = await buildMultimodalMessagesArray( + { + input: { + text: "", + content: [{ type: "text", text: "" }], + files: [path.join(dir, "tone.mp3")], + }, + } as Parameters[0], + "vertex", + "gemini-2.5-flash", + ); + const user = messages.find((message) => message.role === "user"); + const parts = user?.content; + assert( + Array.isArray(parts), + "an audio-only content array produces parts rather than throwing", + ); + assert( + (parts as Array<{ type?: string; mediaType?: string }>).some( + (part) => part.type === "file" && part.mediaType?.startsWith("audio/"), + ), + "the recording is delivered rather than rejected as empty content", + ); +}); + +await test("a provider without native audio keeps the plain-text shape", async () => { + // The counterpart guard: making audio multimodal must not push every other + // provider onto the array shape carrying a part it cannot read. + await ensureMedia(); + const messages = await buildMultimodalMessagesArray( + { + input: { + text: "Describe this audio.", + content: [{ type: "text", text: "What is said in the recording?" }], + files: [path.join(dir, "tone.mp3")], + }, + } as Parameters[0], + "openai", + "gpt-4o", + ); + const user = messages.find((message) => message.role === "user"); + assertEqual( + typeof user?.content, + "string", + "a provider that cannot listen still receives plain text", + ); +}); + // --- AUDIO-032 (#491): audio through the SDK ------------------------------- // // Assertions here use a value that exists ONLY inside the file, or an explicit @@ -839,6 +952,38 @@ await test("a mislabelled filename loses to the declared mimetype", async () => ); }); +await test("an oversized image data URI is refused before it is decoded", async () => { + // The size guard was added on the wrong side of the allocation it exists to + // prevent: `Buffer.from(base64)` ran first and the decoded length was checked + // second, so a 24MB payload was fully materialised before a 10MB limit turned + // it away. Same verdict either way — the cost is what changed. + const imageMax = SIZE_LIMITS_BYTES.IMAGE_MAX; + const oversized: number[] = []; + const realFrom = Buffer.from.bind(Buffer); + const spy = (...args: Parameters) => { + const out = realFrom(...args); + if (Buffer.isBuffer(out) && out.length > imageMax) { + oversized.push(out.length); + } + return out; + }; + Buffer.from = spy as unknown as typeof Buffer.from; + + try { + // heic needs transcoding, so the entry reaches the conversion path rather + // than being passed straight through as an already-universal format. + const base64 = "A".repeat(Math.ceil((imageMax * 2) / 3) * 4); + const input = { images: [`data:image/heic;base64,${base64}`] }; + await normalizeVisionImageFormats(input); + assert( + oversized.length === 0, + "an oversized data URI was decoded in full before the size limit rejected it", + ); + } finally { + Buffer.from = realFrom as unknown as typeof Buffer.from; + } +}); + // HEIC, HEIF, ICO and JPEG 2000 are deliberately NOT asserted end-to-end here. // The extension→type entries this change adds for them are necessary but not // sufficient: `ImageProcessor.validateImageFormat` rejects those MIME types at