diff --git a/package.json b/package.json index 102a9c7c7..1b9a7c0c7 100644 --- a/package.json +++ b/package.json @@ -218,7 +218,13 @@ "pre-push": "pnpm run validate:commit && pnpm run validate:env && pnpm run validate && pnpm run test:ci", "check:all": "pnpm run lint && pnpm run format --check && pnpm run validate && pnpm run validate:commit", "test:litellm-context:vitest": "pnpm exec vitest run test/litellmContextWindows.test.ts", - "test:step-budget-guard:vitest": "pnpm exec vitest run test/stepBudgetGuard.test.ts" + "test:step-budget-guard:vitest": "pnpm exec vitest run test/stepBudgetGuard.test.ts", + "test:audio": "npx tsx test/continuous-test-suite-audio.ts", + "test:office": "npx tsx test/continuous-test-suite-office.ts", + "test:tts:unit": "npx tsx test/continuous-test-suite-tts-unit.ts", + "test:video": "npx tsx test/continuous-test-suite-video.ts", + "test:multimodal": "pnpm run test:audio && pnpm run test:video && pnpm run test:office && pnpm run test:tts:unit && pnpm run test:multimodal:sdk", + "test:multimodal:sdk": "npx tsx test/continuous-test-suite-multimodal-sdk.ts" }, "files": [ "dist", diff --git a/test/continuous-test-suite-audio.ts b/test/continuous-test-suite-audio.ts new file mode 100644 index 000000000..8f7f86e78 --- /dev/null +++ b/test/continuous-test-suite-audio.ts @@ -0,0 +1,242 @@ +#!/usr/bin/env tsx +/** + * Continuous Test Suite: audio file support (no API). + * + * Covers AUDIO-029 (#477), AUDIO-030 (#483), AUDIO-032 (#491) and AUDIO-033 + * (#496). `AudioProcessor` and its FileDetector routing shipped without any + * suite of their own — `ls test/ | grep audio` returned nothing — so every + * assertion here is new ground rather than a restatement of existing coverage. + * + * Fixtures are minted with ffmpeg at run time (see helpers/mediaFixtures.ts). + * `music-metadata` parses real container headers, so only real files exercise + * the code path that matters. + * + * Run: npx tsx test/continuous-test-suite-audio.ts + */ + +import * as fs from "node:fs"; +import * as path from "node:path"; +import { + defineSuite, + assert, + assertEqual, + assertIncludes, + tempDir, + Skip, +} from "./helpers/harness.js"; +import { + hasFfmpeg, + makeAudioFile, + makeCorruptFile, +} from "./helpers/mediaFixtures.js"; +import { + audioProcessor, + isAudioFile, +} from "../src/lib/processors/media/AudioProcessor.js"; +import { FileDetector } from "../src/lib/utils/fileDetector.js"; + +const { test, runSuite } = defineSuite("Audio file support"); + +const dir = tempDir("neurolink-audio-"); +let ffmpegReady = false; + +/** Mint the fixture set once; individual tests skip when ffmpeg is absent. */ +async function ensureFixtures(): Promise { + if (ffmpegReady) { + return; + } + if (!(await hasFfmpeg())) { + throw new Skip("ffmpeg not available — cannot synthesise audio fixtures"); + } + await makeAudioFile(dir, "tone.mp3", 2); + await makeAudioFile(dir, "tone.wav", 1); + await makeAudioFile(dir, "tone.flac", 1); + makeCorruptFile(dir, "broken.mp3"); + ffmpegReady = true; +} + +function fileInfo(file: string, mimetype: string) { + const full = path.join(dir, file); + return { + id: `audio-${file}`, + name: file, + mimetype, + size: fs.statSync(full).size, + buffer: fs.readFileSync(full), + }; +} + +// --- AUDIO-030 (#483): FileDetector recognises audio ------------------------ + +await test("isAudioFile accepts audio MIME types", () => { + assert(isAudioFile("audio/mpeg", "song.mp3"), "audio/mpeg is audio"); + assert(isAudioFile("audio/wav", "clip.wav"), "audio/wav is audio"); + assert(isAudioFile("audio/flac", "track.flac"), "audio/flac is audio"); +}); + +await test("isAudioFile accepts a bare extension when MIME is missing", () => { + // Uploads routinely arrive with an empty or generic MIME type; extension is + // the only signal left, and rejecting those would silently drop real audio. + assert(isAudioFile("", "recording.flac"), "extension alone identifies flac"); + assert( + isAudioFile("application/octet-stream", "voice.m4a"), + "generic MIME falls back to the extension", + ); +}); + +await test("isAudioFile rejects non-audio", () => { + assertEqual(isAudioFile("image/png", "cat.png"), false, "png is not audio"); + assertEqual(isAudioFile("text/csv", "rows.csv"), false, "csv is not audio"); +}); + +await test("FileDetector routes a real mp3 through the audio path", async () => { + await ensureFixtures(); + const detected = await FileDetector.detectAndProcess( + path.join(dir, "tone.mp3"), + ); + assertIncludes( + JSON.stringify(detected).toLowerCase(), + "audio", + "detector reports an audio type for a real mp3", + ); +}); + +// --- AUDIO-029 (#477): AudioProcessor metadata extraction ------------------- + +await test("processes a real mp3 and reports duration, codec and size", async () => { + await ensureFixtures(); + const result = await audioProcessor.processFile( + fileInfo("tone.mp3", "audio/mpeg"), + ); + assert(result.success, `mp3 processing failed: ${JSON.stringify(result)}`); + if (!result.success) { + return; + } + const { metadata, textContent } = result.data; + + // 2s requested; encoders pad, so assert a band rather than equality. + assert( + metadata.duration > 1.5 && metadata.duration < 3.5, + `duration ${metadata.duration}s outside the expected band for a 2s tone`, + ); + assert(metadata.codec.length > 0, "codec is reported"); + assert(metadata.fileSize > 0, "file size is reported"); + assertIncludes( + metadata.durationFormatted, + ":", + "duration is formatted for humans (m:ss)", + ); + assert(textContent.length > 0, "LLM-facing text content is produced"); +}); + +await test("lossless flag distinguishes flac from mp3", async () => { + await ensureFixtures(); + const flac = await audioProcessor.processFile( + fileInfo("tone.flac", "audio/flac"), + ); + const mp3 = await audioProcessor.processFile( + fileInfo("tone.mp3", "audio/mpeg"), + ); + assert(flac.success && mp3.success, "both fixtures process"); + if (!flac.success || !mp3.success) { + return; + } + assertEqual(flac.data.metadata.lossless, true, "flac is lossless"); + assertEqual(mp3.data.metadata.lossless, false, "mp3 is lossy"); +}); + +await test("wav reports sample rate and channels", async () => { + await ensureFixtures(); + const result = await audioProcessor.processFile( + fileInfo("tone.wav", "audio/wav"), + ); + assert(result.success, "wav processing succeeds"); + if (!result.success) { + return; + } + const { sampleRate, channels } = result.data.metadata; + assert((sampleRate ?? 0) > 0, "sample rate is reported for wav"); + assert((channels ?? 0) > 0, "channel count is reported for wav"); +}); + +// --- AUDIO-032 (#491): degraded input --------------------------------------- +// +// AudioProcessor documents "graceful degradation for corrupt or partially +// readable files", and that is what it does: unreadable input still returns +// success with zeroed metadata rather than an error. These tests pin that +// contract down, and pin down the part a caller needs — that the degradation +// is *detectable* (codec "unknown", duration 0) rather than silent. + +await test("a corrupt file degrades instead of throwing", async () => { + await ensureFixtures(); + const result = await audioProcessor.processFile( + fileInfo("broken.mp3", "audio/mpeg"), + ); + // A mislabelled or truncated upload is routine input, so the contract is a + // structured result either way — never a thrown parser error. + assert(result.success, "corrupt audio degrades rather than throwing"); + if (!result.success) { + return; + } + assertEqual( + result.data.metadata.codec, + "unknown", + "an unparseable stream reports codec 'unknown' so callers can detect it", + ); + assertEqual( + result.data.metadata.duration, + 0, + "no duration is invented for an unparseable stream", + ); +}); + +await test("an empty buffer degrades and reports zero size", async () => { + const result = await audioProcessor.processFile({ + id: "audio-empty", + name: "empty.mp3", + mimetype: "audio/mpeg", + size: 0, + buffer: Buffer.alloc(0), + }); + assert(result.success, "zero-byte audio degrades rather than throwing"); + if (!result.success) { + return; + } + assertEqual(result.data.metadata.fileSize, 0, "zero bytes reported as zero"); + assertEqual( + result.data.metadata.codec, + "unknown", + "zero-byte input reports codec 'unknown'", + ); + // NOTE: a 0-byte file and a valid silent recording currently produce the + // same shape — nothing on the result marks it as degraded, so the text handed + // to the model reads as a genuine audio file of zero length. That is the + // audio analogue of #293 (IMG-010, empty-image handling) and is worth a + // dedicated `degraded: true` flag; asserted here as the behaviour that + // exists, so a future fix has to update this test deliberately. +}); + +await test("textContent names the file so the model has context", async () => { + await ensureFixtures(); + const result = await audioProcessor.processFile( + fileInfo("tone.mp3", "audio/mpeg"), + ); + assert(result.success, "processing succeeds"); + if (!result.success) { + return; + } + assertIncludes( + result.data.textContent, + "tone.mp3", + "the filename appears in the text handed to the model", + ); +}); + +// Best-effort cleanup; the OS reclaims the temp dir regardless. +try { + fs.rmSync(dir, { recursive: true, force: true }); +} catch { + /* ignore */ +} + +await runSuite(); diff --git a/test/continuous-test-suite-multimodal-sdk.ts b/test/continuous-test-suite-multimodal-sdk.ts new file mode 100644 index 000000000..5875300c8 --- /dev/null +++ b/test/continuous-test-suite-multimodal-sdk.ts @@ -0,0 +1,610 @@ +#!/usr/bin/env tsx +/** + * Continuous Test Suite: multimodal through the SDK (live). + * + * Covers AUDIO-030 (#483), AUDIO-032 (#491), OFFICE-017 (#493), VIDEO-026 + * (#498), VIDEO-027 (#502) and VIDEO-028 (#510). + * + * The processor-level suites (audio/video/office) prove each processor reads + * its format. They do NOT prove a file handed to `generate()` reaches the model + * as usable content — which is what these six issues actually asked for, and + * the seam where multimodal support really breaks: detection picks the wrong + * processor, MessageBuilder drops the part, or the adapter formats it for the + * wrong provider. + * + * Every assertion here goes through the public surface — `generate()`, + * `stream()`, `FileDetector` and `buildMultimodalMessagesArray()` — with real + * files, and checks BOTH sides: that the input was carried through, and that + * the output reflects it. + * + * Live tests SKIP without credentials rather than failing. + * + * Run: npx tsx test/continuous-test-suite-multimodal-sdk.ts + */ + +import "dotenv/config"; +import * as fs from "node:fs"; +import * as path from "node:path"; +import { + defineSuite, + assert, + assertEqual, + assertIncludes, + tempDir, + Skip, +} from "./helpers/harness.js"; +import { + hasFfmpeg, + makeAudioFile, + makeVideoFile, +} from "./helpers/mediaFixtures.js"; +import { hasPackage, makeDocx, makeXlsx } from "./helpers/officeFixtures.js"; +import { FileDetector } from "../src/lib/utils/fileDetector.js"; +import { buildMultimodalMessagesArray } from "../src/lib/utils/messageBuilder.js"; +import { NeuroLink } from "../src/lib/neurolink.js"; + +const { test, runSuite } = defineSuite("Multimodal through the SDK"); + +const dir = tempDir("neurolink-sdk-mm-"); +let media = false; + +/** Live provider used for the generate()/stream() assertions. */ +const PROVIDER = process.env.MM_TEST_PROVIDER ?? "vertex"; + +/** + * Hard bound on every live provider call. + * + * Without one a stalled provider hangs the suite until the harness's own + * per-test timeout fires, which is a far blunter signal. 60s is generous for a + * multimodal turn that uploads a file and still fails fast on a wedged request. + */ +const LIVE_TIMEOUT_MS = 60_000; + +/** + * Duration of the fixture used for the Buffer-input assertion. + * + * A Buffer carries no filename, so the filename check the path-based test + * relies on is unavailable and the assertion has to come from the audio's own + * metadata instead. The value matters: asking for the *sample rate* was tried + * first and proved worthless, because the model answers "44100" from priors + * even with no file attached at all — a guessable fact is not evidence. + * Duration has no such prior, and 7 is neither round nor a plausible default, + * so a correct "7" can only have come from the container header. + */ +const ODD_SECONDS = 7; + +/** + * Turn-ending messages NeuroLink substitutes for model output when a turn hits + * its wall-clock deadline, stalls, or is aborted (see + * `buildTurnTimeoutMessage` and siblings in + * `src/lib/providers/googleNativeGemini3/utils.ts`). + * + * These are non-empty and perfectly plausible prose, so they satisfy a + * "response is not empty" check and any assertion phrased as the *absence* of + * a refusal — while carrying no answer at all. Observed live at roughly one + * call in three during this suite's development. Treated as a non-answer and + * retried rather than asserted against. + */ +const TURN_ENDED_MARKERS = [ + "hit its processing time limit", + "made no progress for", + "ask me to continue and I'll pick up from there", +]; + +function isNonAnswer(content: string): boolean { + if (content.trim().length === 0) { + return true; + } + return TURN_ENDED_MARKERS.some((marker) => content.includes(marker)); +} + +/** + * Credential check for the provider actually under test. + * + * This has to follow `PROVIDER`, not assume Vertex. `MM_TEST_PROVIDER` makes + * the provider configurable, so a gate hardcoded to Vertex is wrong in both + * directions: with Vertex credentials absent but the chosen provider's present + * every live test skips for no reason, and with Vertex present but the chosen + * provider's absent they run and die on provider auth instead of skipping + * cleanly — which reads as a product failure rather than a missing key. + * + * An unrecognised provider returns false (skip) rather than true (attempt): + * a skip naming the provider is diagnosable, an auth error from a live call is + * not. + */ +function hasCredentialsFor(provider: string): boolean { + switch (provider) { + case "vertex": + case "google-vertex": + return Boolean( + process.env.GOOGLE_VERTEX_PROJECT || + process.env.GOOGLE_APPLICATION_CREDENTIALS, + ); + case "openai": + return Boolean(process.env.OPENAI_API_KEY); + case "anthropic": + return Boolean(process.env.ANTHROPIC_API_KEY); + case "google-ai": + case "googleaistudio": + return Boolean(process.env.GOOGLE_AI_API_KEY); + case "bedrock": + case "amazonbedrock": + return Boolean( + process.env.AWS_ACCESS_KEY_ID && process.env.AWS_SECRET_ACCESS_KEY, + ); + default: + return false; + } +} + +async function ensureMedia(): Promise { + if (media) { + return; + } + if (!(await hasFfmpeg())) { + throw new Skip("ffmpeg not available — cannot synthesise media fixtures"); + } + await makeAudioFile(dir, "tone.mp3", 2); + // 7s, deliberately not a round or common value — see ODD_SECONDS. + await makeAudioFile(dir, "odd.mp3", ODD_SECONDS); + await makeAudioFile(dir, "tone.wav", 1); + await makeAudioFile(dir, "tone.m4a", 1); + await makeAudioFile(dir, "tone.ogg", 1); + await makeVideoFile(dir, "clip.mp4", 2); + media = true; +} + +function requireLive(): void { + if (!hasCredentialsFor(PROVIDER)) { + throw new Skip( + `no credentials for provider "${PROVIDER}" — skipping live SDK assertions`, + ); + } +} + +/** + * Run a live call, retrying past responses that carry no answer. + * + * Two distinct non-answers occur here. Vertex intermittently returns an empty + * completion — the same request that yields a real answer can come back as "". + * Separately, NeuroLink substitutes a turn-ended message when a turn exceeds + * its deadline (see TURN_ENDED_MARKERS); that one is *non-empty*, so an + * emptiness check alone lets it through to an assertion that then measures + * nothing. Neither is a NeuroLink defect the multimodal path is responsible + * for, so retry rather than redden the suite. + * + * If every attempt is a non-answer the caller's assertion still fails, which + * is what we want if it becomes systematic. + */ +async function generateNonEmpty( + nl: NeuroLink, + options: Parameters[0], +): Promise { + let last = ""; + for (let attempt = 0; attempt < 3; attempt++) { + // Merge rather than replace, so a caller-supplied timeout still wins. + const result = await nl.generate({ timeout: LIVE_TIMEOUT_MS, ...options }); + last = result.content; + if (!isNonAnswer(last)) { + return last; + } + } + return last; +} + +// --- VIDEO-026 (#498) / AUDIO-030 (#483): FileDetector on real bytes --------- +// +// The detector is the first thing every multimodal call hits. These assert it +// against real container headers rather than an extension, because a file whose +// extension lies is the case that actually reaches production. + +await test("FileDetector identifies each audio container from its bytes", async () => { + await ensureMedia(); + for (const [file, expect] of [ + ["tone.mp3", "audio"], + ["tone.wav", "audio"], + ["tone.m4a", "audio"], + ["tone.ogg", "audio"], + ] as const) { + const result = await FileDetector.detectAndProcess(path.join(dir, file)); + assertIncludes( + JSON.stringify(result).toLowerCase(), + expect, + `${file} detected as ${expect}`, + ); + } +}); + +await test("FileDetector identifies mp4 from its bytes", async () => { + await ensureMedia(); + const result = await FileDetector.detectAndProcess( + path.join(dir, "clip.mp4"), + ); + assertIncludes( + JSON.stringify(result).toLowerCase(), + "video", + "mp4 detected as video", + ); +}); + +await test("FileDetector trusts bytes over a lying extension", async () => { + await ensureMedia(); + // An mp3 renamed .mp4 must not be routed to the video processor: magic-byte + // detection is the whole point of not trusting the filename. + const lying = path.join(dir, "actually-audio.mp4"); + fs.copyFileSync(path.join(dir, "tone.mp3"), lying); + const result = await FileDetector.detectAndProcess(lying); + const json = JSON.stringify(result).toLowerCase(); + assertIncludes(json, "audio", "content wins over the extension"); +}); + +// --- VIDEO-027 (#502): MessageBuilder carries media into the message --------- + +await test("buildMultimodalMessagesArray carries a video into message content", async () => { + await ensureMedia(); + const messages = await buildMultimodalMessagesArray( + { + input: { + text: "What is in this clip?", + files: [path.join(dir, "clip.mp4")], + }, + } as Parameters[0], + "vertex", + "gemini-2.5-flash", + ); + + assert(Array.isArray(messages) && messages.length > 0, "messages produced"); + const serialised = JSON.stringify(messages); + assertIncludes(serialised, "What is in this clip?", "the prompt survives"); + // The whole point of the builder is that the file becomes model-visible + // content; a message array carrying only the prompt means the video was + // silently dropped. + assert( + serialised.length > 500, + `expected media content in the message, got ${serialised.length} chars`, + ); +}); + +await test("buildMultimodalMessagesArray carries audio into message content", async () => { + await ensureMedia(); + const messages = await buildMultimodalMessagesArray( + { + input: { + text: "Describe this audio.", + files: [path.join(dir, "tone.mp3")], + }, + } as Parameters[0], + "vertex", + "gemini-2.5-flash", + ); + const serialised = JSON.stringify(messages); + assertIncludes(serialised, "Describe this audio.", "the prompt survives"); + assertIncludes( + serialised.toLowerCase(), + "tone.mp3", + "the audio file is named in the message handed to the model", + ); +}); + +// --- AUDIO-032 (#491): audio through the SDK ------------------------------- +// +// Assertions here use a value that exists ONLY inside the file, or an explicit +// sentinel. Asking the model to "reply with the word AUDIO" and asserting the +// response contains AUDIO is worthless: the refusal "No audio file is +// attached." contains it too, and a prompt that asserts a file is attached +// gets an obedient "RECEIVED" from a model that received nothing. Three tests +// here passed for exactly those reasons until they were checked against a +// live provider. + +await test("audio reaches the model via input.files", async () => { + await ensureMedia(); + requireLive(); + const nl = new NeuroLink(); + const content = await generateNonEmpty(nl, { + input: { + text: "Describe the attached file. If no file reached you, reply exactly: NOTHING_RECEIVED", + files: [path.join(dir, "tone.mp3")], + }, + provider: PROVIDER, + maxTokens: 512, + }); + assert( + !content.includes("NOTHING_RECEIVED"), + `the model reported receiving nothing; got: ${content.slice(0, 200)}`, + ); + // Only obtainable by reading the container header. + assertIncludes( + content.toLowerCase(), + "tone.mp3", + `the model described the actual file; got: ${content.slice(0, 200)}`, + ); +}); + +await test("audio reaches the model as a Buffer via input.files", async () => { + await ensureMedia(); + requireLive(); + const nl = new NeuroLink(); + // Asks for the duration rather than a description. A Buffer has no filename, + // so the neighbouring path-based test's "tone.mp3" check is unavailable here + // — and asserting only the ABSENCE of the sentinel proves nothing, because a + // model that received no file still answers with plausible prose instead of + // the sentinel. Verified against a live negative control: with no file + // attached the reply was "I apologize for the confusion…", which satisfies a + // sentinel-only assertion while containing no answer. + const content = await generateNonEmpty(nl, { + input: { + text: "How many seconds long is the attached audio? Answer with the number only. If no file reached you, reply exactly: NOTHING_RECEIVED", + files: [fs.readFileSync(path.join(dir, "odd.mp3"))], + }, + provider: PROVIDER, + maxTokens: 512, + }); + assert( + !content.includes("NOTHING_RECEIVED"), + `buffer input reached the model; got: ${content.slice(0, 200)}`, + ); + // Word-bounded so a stray "7" inside a larger number cannot satisfy it. + assert( + new RegExp(`\\b${ODD_SECONDS}\\b`).test(content), + `the model read the real duration (${ODD_SECONDS}s) from the buffer; got: ${content.slice(0, 200)}`, + ); +}); + +await test("input.audioFiles delivers audio (regression for #1259)", async () => { + await ensureMedia(); + requireLive(); + // #1259: audioFiles used to be dropped on every path that bypasses + // buildMultimodalMessagesArray, so this returned NOTHING_RECEIVED while the + // identical call through input.files answered correctly. Asserting the real + // duration rather than the absence of the sentinel, for the reason given on + // the Buffer test above. + const nl = new NeuroLink(); + const content = await generateNonEmpty(nl, { + input: { + text: "How many seconds long is the attached audio? Answer with the number only. If no file reached you, reply exactly: NOTHING_RECEIVED", + audioFiles: [path.join(dir, "odd.mp3")], + }, + provider: PROVIDER, + maxTokens: 512, + }); + assert( + !content.includes("NOTHING_RECEIVED"), + `audioFiles reached the model; got: ${content.slice(0, 200)}`, + ); + assert( + new RegExp(`\\b${ODD_SECONDS}\\b`).test(content), + `the model read the real duration (${ODD_SECONDS}s) via audioFiles; got: ${content.slice(0, 200)}`, + ); +}); + +// --- OFFICE-017 (#493): generate() with office documents --------------------- + +await test("generate() reads content out of a .docx", async () => { + if (!(await hasPackage("mammoth"))) { + throw new Skip("mammoth not installed"); + } + requireLive(); + const docx = path.join(dir, "report.docx"); + fs.writeFileSync( + docx, + makeDocx(["The project codename is FALCON.", "Budget approved."]), + ); + const nl = new NeuroLink(); + const result = await nl.generate({ + input: { + text: "What is the project codename in this document? Answer with the codename only.", + files: [docx], + }, + provider: PROVIDER, + maxTokens: 512, + timeout: LIVE_TIMEOUT_MS, + }); + // Asserting on content the model could only know by reading the file is the + // difference between "the call succeeded" and "the document arrived". + assertIncludes( + result.content.toUpperCase(), + "FALCON", + `docx content reached the model; got: ${result.content.slice(0, 160)}`, + ); +}); + +await test("generate() reads cell values out of an .xlsx", async () => { + if (!(await hasPackage("exceljs"))) { + throw new Skip("exceljs not installed"); + } + requireLive(); + const xlsx = path.join(dir, "book.xlsx"); + fs.writeFileSync( + xlsx, + await makeXlsx({ + Revenue: [ + ["region", "amount"], + ["Zanzibar", 4242], + ], + }), + ); + const nl = new NeuroLink(); + const result = await nl.generate({ + input: { + text: "What amount is listed for Zanzibar in this spreadsheet? Answer with the number only.", + files: [xlsx], + }, + provider: PROVIDER, + maxTokens: 512, + timeout: LIVE_TIMEOUT_MS, + }); + assertIncludes( + result.content, + "4242", + `xlsx cell reached the model; got: ${result.content.slice(0, 160)}`, + ); +}); + +// --- VIDEO-028 (#510): video through the SDK -------------------------------- + +await test("video reaches the model via input.files", async () => { + await ensureMedia(); + requireLive(); + const nl = new NeuroLink(); + const content = await generateNonEmpty(nl, { + input: { + text: "What is the pixel resolution of the attached video? Answer with WIDTHxHEIGHT only, e.g. 640x480. If no video reached you, reply exactly: NOTHING_RECEIVED", + files: [path.join(dir, "clip.mp4")], + }, + provider: PROVIDER, + maxTokens: 512, + }); + assert( + !content.includes("NOTHING_RECEIVED"), + `the model reported seeing nothing; got: ${content.slice(0, 200)}`, + ); + // Asking for a specific fact rather than a free-form description: the model's + // prose varies run to run (an earlier version asserted on whichever details it + // happened to mention, and flaked). 320x240 is the fixture's real resolution, + // obtainable only from the probed stream, and no refusal contains it. + assertIncludes( + content, + "320x240", + `the model read the real resolution; got: ${content.slice(0, 200)}`, + ); +}); + +await test("input.videoFiles delivers video (regression for #1259)", async () => { + await ensureMedia(); + requireLive(); + const nl = new NeuroLink(); + const content = await generateNonEmpty(nl, { + input: { + text: "What is the pixel resolution of the attached video? Answer with WIDTHxHEIGHT only, e.g. 640x480. If no video reached you, reply exactly: NOTHING_RECEIVED", + videoFiles: [path.join(dir, "clip.mp4")], + }, + provider: PROVIDER, + maxTokens: 512, + }); + assert( + !content.includes("NOTHING_RECEIVED"), + `videoFiles reached the model; got: ${content.slice(0, 200)}`, + ); + assertIncludes( + content, + "320x240", + `the model read the real resolution via videoFiles; got: ${content.slice(0, 200)}`, + ); +}); + +// --- stream() parity ---------------------------------------------------------- + +await test("stream() carries a document through the same path as generate()", async () => { + if (!(await hasPackage("mammoth"))) { + throw new Skip("mammoth not installed"); + } + // Pinned to anthropic, so the gate names that provider rather than + // following PROVIDER. + if (!hasCredentialsFor("anthropic")) { + throw new Skip("no anthropic credentials — skipping stream parity"); + } + const docx = path.join(dir, "stream.docx"); + fs.writeFileSync(docx, makeDocx(["The access code is ORCHID."])); + const nl = new NeuroLink(); + // Deliberately a *second* provider. #1258 was a provider-specific break — + // Vertex's native override ran file preprocessing in generate() but not in + // executeStream() — so one provider's stream path passing says nothing about + // another's. The PROVIDER-based parity test below covers the configured + // provider; this one keeps a non-Vertex path under continuous assertion. + const result = await nl.stream({ + input: { + text: "What is the access code in this document? Answer with the code only.", + files: [docx], + }, + provider: "anthropic", + maxTokens: 60, + timeout: LIVE_TIMEOUT_MS, + }); + let text = ""; + for await (const chunk of result.stream) { + text += typeof chunk === "string" ? chunk : (chunk.content ?? ""); + } + // generate() and stream() share MessageBuilder but not the whole path, so a + // document that works in one can still be dropped in the other. + assertIncludes( + text.toUpperCase(), + "ORCHID", + `docx content reached the model via stream(); got: ${text.slice(0, 160)}`, + ); +}); + +await test("mixed multimodal input keeps every part", async () => { + await ensureMedia(); + if (!(await hasPackage("exceljs"))) { + throw new Skip("exceljs not installed"); + } + requireLive(); + const xlsx = path.join(dir, "mixed.xlsx"); + fs.writeFileSync(xlsx, await makeXlsx({ Data: [["token"], ["PELICAN"]] })); + const nl = new NeuroLink(); + const content = await generateNonEmpty(nl, { + input: { + text: "You have two attached files. Reply with the token from the spreadsheet, then the name of the audio file.", + files: [xlsx, path.join(dir, "tone.mp3")], + }, + provider: PROVIDER, + maxTokens: 512, + timeout: LIVE_TIMEOUT_MS, + }); + const upper = content.toUpperCase(); + assertIncludes( + upper, + "PELICAN", + `spreadsheet part survived; got: ${content.slice(0, 200)}`, + ); + // The filename, not the word "AUDIO": a refusal such as "no audio file is + // attached" contains "AUDIO" too, so that assertion passed whether or not + // the audio ever arrived. Same false-pass shape this file's header warns + // about — it survived one round of fixing those and was caught in review. + assertIncludes( + upper, + "TONE.MP3", + `audio part survived; got: ${content.slice(0, 200)}`, + ); +}); + +await test("stream() file parity on the configured provider (regression for #1258)", async () => { + if (!(await hasPackage("mammoth"))) { + throw new Skip("mammoth not installed"); + } + requireLive(); + // #1258: Vertex overrides both generate() and executeStream() to reach the + // native SDKs, but only generate() ran file preprocessing — so this exact + // call answered "no documents attached" while generate() returned the code. + // Runs against PROVIDER (not a provider known to be correct), which is what + // makes it a parity test rather than a restatement of the test above. + const docx = path.join(dir, "parity.docx"); + fs.writeFileSync(docx, makeDocx(["The access code is MERIDIAN."])); + const nl = new NeuroLink(); + const result = await nl.stream({ + input: { + text: "What is the access code in this document? Answer with the code only.", + files: [docx], + }, + provider: PROVIDER, + maxTokens: 200, + timeout: LIVE_TIMEOUT_MS, + }); + let text = ""; + for await (const chunk of result.stream) { + text += typeof chunk === "string" ? chunk : (chunk.content ?? ""); + } + assertIncludes( + text.toUpperCase(), + "MERIDIAN", + `docx content reached the model via stream() on ${PROVIDER}; got: ${text.slice(0, 160)}`, + ); +}); + +try { + fs.rmSync(dir, { recursive: true, force: true }); +} catch { + /* ignore */ +} + +await runSuite(); diff --git a/test/continuous-test-suite-office.ts b/test/continuous-test-suite-office.ts new file mode 100644 index 000000000..42885568c --- /dev/null +++ b/test/continuous-test-suite-office.ts @@ -0,0 +1,212 @@ +#!/usr/bin/env tsx +/** + * Continuous Test Suite: Office document support (no API). + * + * Covers OFFICE-015 (#485), OFFICE-016 (#487), OFFICE-017 (#493) and + * OFFICE-019 (#499). Word and Excel processing shipped with only a narrow + * interop regression guard (continuous-test-suite-excel-interop.ts) and nothing + * for .docx at all, so this is the first coverage of extraction, sheet + * handling, detection and the error paths. + * + * Fixtures are built in memory (see helpers/officeFixtures.ts). exceljs and + * mammoth are optionalDependencies, so tests SKIP when they are absent — + * mirroring how the processors themselves behave. + * + * Run: npx tsx test/continuous-test-suite-office.ts + */ + +import { + defineSuite, + assert, + assertEqual, + assertIncludes, + Skip, +} from "./helpers/harness.js"; +import { hasPackage, makeDocx, makeXlsx } from "./helpers/officeFixtures.js"; +import { + wordProcessor, + isWordFile, +} from "../src/lib/processors/document/WordProcessor.js"; +import { + excelProcessor, + isExcelFile, +} from "../src/lib/processors/document/ExcelProcessor.js"; + +const { test, runSuite } = defineSuite("Office document support"); + +const DOCX_MIME = + "application/vnd.openxmlformats-officedocument.wordprocessingml.document"; +const XLSX_MIME = + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; + +async function requireMammoth(): Promise { + if (!(await hasPackage("mammoth"))) { + throw new Skip("mammoth not installed (optional dependency)"); + } +} +async function requireExcel(): Promise { + if (!(await hasPackage("exceljs"))) { + throw new Skip("exceljs not installed (optional dependency)"); + } +} + +function info(name: string, mimetype: string, buffer: Buffer) { + return { id: `office-${name}`, name, mimetype, size: buffer.length, buffer }; +} + +// --- Detection --------------------------------------------------------------- + +await test("isWordFile / isExcelFile accept their own formats", () => { + assert(isWordFile(DOCX_MIME, "report.docx"), "docx recognised"); + assert(isWordFile("application/msword", "legacy.doc"), "doc recognised"); + assert(isExcelFile(XLSX_MIME, "book.xlsx"), "xlsx recognised"); + assert(isExcelFile("application/vnd.ms-excel", "old.xls"), "xls recognised"); +}); + +await test("the two processors do not claim each other's formats", () => { + // A shared "office" bucket would route a spreadsheet into mammoth, which + // fails deep inside the parser rather than at the boundary. + assertEqual(isWordFile(XLSX_MIME, "book.xlsx"), false, "word rejects xlsx"); + assertEqual( + isExcelFile(DOCX_MIME, "report.docx"), + false, + "excel rejects docx", + ); +}); + +// --- OFFICE-016 (#487): DOCX extraction -------------------------------------- + +await test("extracts text from a real .docx", async () => { + await requireMammoth(); + const buf = makeDocx([ + "Quarterly Review", + "Revenue grew 12% year over year.", + ]); + const result = await wordProcessor.processFile( + info("report.docx", DOCX_MIME, buf), + ); + assert(result.success, `docx processing failed: ${JSON.stringify(result)}`); + if (!result.success) { + return; + } + assertIncludes( + result.data.textContent, + "Quarterly Review", + "first paragraph is extracted", + ); + assertIncludes( + result.data.textContent, + "Revenue grew 12%", + "second paragraph is extracted", + ); +}); + +await test("produces HTML alongside plain text", async () => { + await requireMammoth(); + const buf = makeDocx(["Heading text"]); + const result = await wordProcessor.processFile( + info("report.docx", DOCX_MIME, buf), + ); + assert(result.success, "docx processing succeeds"); + if (!result.success) { + return; + } + assertIncludes( + result.data.htmlContent, + "

", + "HTML conversion emits paragraph markup", + ); +}); + +// --- OFFICE-016 (#487): XLSX sheets ------------------------------------------ + +await test("reads every worksheet, not just the first", async () => { + await requireExcel(); + const buf = await makeXlsx({ + Revenue: [ + ["quarter", "amount"], + ["q1", 100], + ], + Costs: [ + ["quarter", "amount"], + ["q1", 60], + ], + }); + const result = await excelProcessor.processFile( + info("book.xlsx", XLSX_MIME, buf), + ); + assert(result.success, `xlsx processing failed: ${JSON.stringify(result)}`); + if (!result.success) { + return; + } + assertEqual(result.data.sheetCount, 2, "both worksheets are read"); + const names = result.data.worksheets.map((w) => w.name); + assert( + names.includes("Revenue") && names.includes("Costs"), + `worksheet names preserved, got ${JSON.stringify(names)}`, + ); +}); + +await test("counts data rows across sheets", async () => { + await requireExcel(); + const buf = await makeXlsx({ + Sheet1: [ + ["a", "b"], + [1, 2], + [3, 4], + ], + }); + const result = await excelProcessor.processFile( + info("book.xlsx", XLSX_MIME, buf), + ); + assert(result.success, "xlsx processing succeeds"); + if (!result.success) { + return; + } + assert( + result.data.totalRows >= 3, + `expected at least the 3 authored rows, got ${result.data.totalRows}`, + ); +}); + +// --- OFFICE-019 (#499): error handling --------------------------------------- + +await test("a non-ZIP payload is rejected before reaching the parser", async () => { + await requireMammoth(); + // .docx and .xlsx are ZIP archives; the PK signature check is the cheap + // boundary that keeps an HTML error page or truncated download from being + // handed to a parser that will fail obscurely. + const notZip = Buffer.from("error page", "utf8"); + const result = await wordProcessor.processFile( + info("report.docx", DOCX_MIME, notZip), + ); + assertEqual(result.success, false, "non-ZIP content is rejected"); +}); + +await test("an HTML error page masquerading as .xlsx is rejected", async () => { + await requireExcel(); + const notZip = Buffer.from("404 Not Found", "utf8"); + const result = await excelProcessor.processFile( + info("book.xlsx", XLSX_MIME, notZip), + ); + assertEqual(result.success, false, "non-ZIP content is rejected"); +}); + +await test("a truncated file is rejected rather than half-parsed", async () => { + await requireExcel(); + const full = await makeXlsx({ Sheet1: [["a"], [1]] }); + const truncated = full.subarray(0, Math.floor(full.length / 3)); + const result = await excelProcessor.processFile( + info("book.xlsx", XLSX_MIME, truncated), + ); + assertEqual(result.success, false, "a truncated archive is rejected"); +}); + +await test("an empty buffer is rejected", async () => { + const result = await wordProcessor.processFile( + info("report.docx", DOCX_MIME, Buffer.alloc(0)), + ); + assertEqual(result.success, false, "zero-byte docx is rejected"); +}); + +await runSuite(); diff --git a/test/continuous-test-suite-tts-unit.ts b/test/continuous-test-suite-tts-unit.ts new file mode 100644 index 000000000..0873da8c5 --- /dev/null +++ b/test/continuous-test-suite-tts-unit.ts @@ -0,0 +1,290 @@ +#!/usr/bin/env tsx +/** + * Continuous Test Suite: TTSProcessor unit tests (no API). + * + * Covers TTS-027 (#527), and the no-key half of TTS-028 (#528). + * + * `continuous-test-suite-tts.ts` already exercises TTS end to end, but every + * assertion in it needs live Google TTS credentials — so on a machine or CI job + * without keys, none of the registry, validation or dispatch logic is checked + * at all. This suite covers that logic with a stub handler, so the parts that + * can be tested without a network call always are. + * + * Run: npx tsx test/continuous-test-suite-tts-unit.ts + */ + +import { + defineSuite, + assert, + assertEqual, + assertIncludes, +} from "./helpers/harness.js"; +import { + TTSProcessor, + TTSError, + TTS_ERROR_CODES, +} from "../src/lib/utils/ttsProcessor.js"; +import type { TTSHandler } from "../src/lib/types/index.js"; + +const { test, runSuite } = defineSuite("TTSProcessor (unit)"); + +/** Records what it was asked to do so dispatch can be asserted, not inferred. */ +function makeStubHandler(overrides: Partial = {}) { + const calls: Array<{ text: string; options: unknown }> = []; + const audio = Buffer.from("fake-audio"); + const handler = { + isConfigured: () => true, + synthesize: async (text: string, options: unknown) => { + calls.push({ text, options }); + return { + buffer: audio, + format: "mp3", + size: audio.length, + voice: "stub-voice", + }; + }, + getVoices: async () => [{ name: "stub-voice", languageCode: "en-US" }], + ...overrides, + } as unknown as TTSHandler; + return { handler, calls }; +} + +const PROVIDER = "stub-tts-provider"; + +await test("registerHandler makes a provider resolvable", () => { + const { handler } = makeStubHandler(); + TTSProcessor.registerHandler(PROVIDER, handler); + assertEqual(TTSProcessor.supports(PROVIDER), true, "supports() sees it"); + assert( + TTSProcessor.getHandler(PROVIDER) !== undefined, + "getHandler() returns it", + ); +}); + +await test("an unregistered provider is not claimed", () => { + assertEqual( + TTSProcessor.supports("provider-that-was-never-registered"), + false, + "supports() is false for unknown providers", + ); + assertEqual( + TTSProcessor.getHandler("provider-that-was-never-registered"), + undefined, + "getHandler() returns undefined rather than throwing", + ); +}); + +await test("synthesize dispatches to the registered handler", async () => { + const { handler, calls } = makeStubHandler(); + TTSProcessor.registerHandler(PROVIDER, handler); + const result = await TTSProcessor.synthesize("hello world", PROVIDER, {}); + assertEqual(calls.length, 1, "handler invoked exactly once"); + assertEqual(calls[0].text, "hello world", "text forwarded verbatim"); + assert(Buffer.isBuffer(result.buffer), "audio buffer returned"); + assertEqual(result.size, result.buffer.length, "size matches the buffer"); +}); + +await test("empty text is rejected before any handler runs", async () => { + const { handler, calls } = makeStubHandler(); + TTSProcessor.registerHandler(PROVIDER, handler); + let code: string | undefined; + try { + await TTSProcessor.synthesize("", PROVIDER, {}); + } catch (err) { + code = err instanceof TTSError ? err.code : undefined; + } + assertEqual( + code, + TTS_ERROR_CODES.EMPTY_TEXT, + "empty text raises TTS_EMPTY_TEXT", + ); + // The point of validating first is not to spend a paid API call proving the + // input was empty. + assertEqual(calls.length, 0, "handler was never invoked"); +}); + +await test("whitespace-only text counts as empty", async () => { + const { handler } = makeStubHandler(); + TTSProcessor.registerHandler(PROVIDER, handler); + let code: string | undefined; + try { + await TTSProcessor.synthesize(" \n\t ", PROVIDER, {}); + } catch (err) { + code = err instanceof TTSError ? err.code : undefined; + } + assertEqual( + code, + TTS_ERROR_CODES.EMPTY_TEXT, + "whitespace-only input is treated as empty", + ); +}); + +await test("an unsupported provider raises a typed error", async () => { + let code: string | undefined; + let message = ""; + try { + await TTSProcessor.synthesize("hello", "no-such-provider", {}); + } catch (err) { + code = err instanceof TTSError ? err.code : undefined; + message = err instanceof Error ? err.message : ""; + } + assertEqual( + code, + TTS_ERROR_CODES.PROVIDER_NOT_SUPPORTED, + "unknown provider raises TTS_PROVIDER_NOT_SUPPORTED", + ); + assertIncludes( + message, + "no-such-provider", + "the error names the provider that was asked for", + ); +}); + +await test("text beyond the handler's limit is rejected", async () => { + const { handler, calls } = makeStubHandler({ + maxTextLength: 10, + } as Partial); + TTSProcessor.registerHandler(PROVIDER, handler); + let code: string | undefined; + try { + await TTSProcessor.synthesize("x".repeat(50), PROVIDER, {}); + } catch (err) { + code = err instanceof TTSError ? err.code : undefined; + } + assertEqual( + code, + TTS_ERROR_CODES.TEXT_TOO_LONG, + "over-long text raises TTS_TEXT_TOO_LONG", + ); + assertEqual(calls.length, 0, "handler was never invoked"); +}); + +await test("an unconfigured provider is rejected before synthesis", async () => { + // Registration and configuration are separate states: a handler can be + // registered at startup and only later discover it has no credentials. + // Failing here rather than inside the provider is what turns a vendor auth + // error into an actionable "set the API keys". + const { handler, calls } = makeStubHandler({ + isConfigured: () => false, + } as Partial); + TTSProcessor.registerHandler(PROVIDER, handler); + let code: string | undefined; + try { + await TTSProcessor.synthesize("hello", PROVIDER, {}); + } catch (err) { + code = err instanceof TTSError ? err.code : undefined; + } + assertEqual( + code, + TTS_ERROR_CODES.PROVIDER_NOT_CONFIGURED, + "unconfigured provider raises TTS_PROVIDER_NOT_CONFIGURED", + ); + assertEqual(calls.length, 0, "handler was never invoked"); +}); + +await test("length is validated before configuration", async () => { + // Ordering is observable, so pin it: an over-long text sent to an + // unconfigured provider must report the text problem the caller can fix + // from the input, not a credentials problem that is beside the point. + const { handler } = makeStubHandler({ + isConfigured: () => false, + maxTextLength: 10, + } as Partial); + TTSProcessor.registerHandler(PROVIDER, handler); + let code: string | undefined; + try { + await TTSProcessor.synthesize("x".repeat(50), PROVIDER, {}); + } catch (err) { + code = err instanceof TTSError ? err.code : undefined; + } + assertEqual( + code, + TTS_ERROR_CODES.TEXT_TOO_LONG, + "the text-length failure wins over the configuration failure", + ); +}); + +await test("a handler failure surfaces as a typed TTSError", async () => { + // A raw vendor error escaping synthesize() would force every caller to + // string-match on provider-specific messages. + const { handler } = makeStubHandler({ + synthesize: async () => { + throw new Error("upstream vendor exploded"); + }, + } as Partial); + TTSProcessor.registerHandler(PROVIDER, handler); + let code: string | undefined; + let isTyped = false; + let message = ""; + try { + await TTSProcessor.synthesize("hello", PROVIDER, {}); + } catch (err) { + isTyped = err instanceof TTSError; + code = err instanceof TTSError ? err.code : undefined; + message = err instanceof Error ? err.message : ""; + } + assert(isTyped, "a raw handler error is wrapped rather than leaked"); + assertEqual( + code, + TTS_ERROR_CODES.SYNTHESIS_FAILED, + "handler failure raises TTS_SYNTHESIS_FAILED", + ); + // The original cause has to survive the wrapping, or the wrap has destroyed + // the only information that explains the failure. + assertIncludes( + message, + "upstream vendor exploded", + "the underlying reason is preserved in the wrapped error", + ); +}); + +await test("a voice-resolution failure propagates rather than being swallowed", async () => { + // An empty voice list and a failed voice lookup are different outcomes; a + // handler that swallowed the error would render the second as the first, + // and a caller would show the user "no voices available" for what is really + // an outage. + const { handler } = makeStubHandler({ + getVoices: async () => { + throw new Error("voice catalogue unavailable"); + }, + } as Partial); + TTSProcessor.registerHandler(PROVIDER, handler); + const resolved = TTSProcessor.getHandler(PROVIDER); + let message = ""; + try { + await resolved?.getVoices?.("en-US"); + } catch (err) { + message = err instanceof Error ? err.message : ""; + } + assertIncludes( + message, + "voice catalogue unavailable", + "the failure reaches the caller instead of becoming an empty list", + ); +}); + +await test("re-registering a provider replaces the previous handler", () => { + // Registration is a Map insert, so a second registration must win rather + // than silently keeping the first — otherwise a host that swaps credentials + // at runtime keeps using the stale handler. + const first = makeStubHandler(); + const second = makeStubHandler(); + TTSProcessor.registerHandler(PROVIDER, first.handler); + TTSProcessor.registerHandler(PROVIDER, second.handler); + assertEqual( + TTSProcessor.getHandler(PROVIDER), + second.handler, + "the later registration wins", + ); +}); + +await test("getVoices reaches the handler", async () => { + const { handler } = makeStubHandler(); + TTSProcessor.registerHandler(PROVIDER, handler); + const resolved = TTSProcessor.getHandler(PROVIDER); + assert(resolved !== undefined, "handler resolves"); + const voices = await resolved?.getVoices?.("en-US"); + assert(Array.isArray(voices) && voices.length > 0, "voices are returned"); +}); + +await runSuite(); diff --git a/test/continuous-test-suite-video.ts b/test/continuous-test-suite-video.ts new file mode 100644 index 000000000..52f7b6c7c --- /dev/null +++ b/test/continuous-test-suite-video.ts @@ -0,0 +1,238 @@ +#!/usr/bin/env tsx +/** + * Continuous Test Suite: video file support (no API). + * + * Covers VIDEO-025 (#495), VIDEO-026 (#498), VIDEO-027 (#502), VIDEO-028 (#510) + * and VIDEO-030 (#518). `VideoProcessor` — probing, metadata, keyframe + * extraction — shipped with no suite of its own. + * + * Fixtures are minted with ffmpeg at run time (helpers/mediaFixtures.ts). + * ffprobe reads real container headers and keyframe extraction genuinely + * decodes, so synthetic bytes would exercise nothing; committing sample video + * would add binaries to the repo for every container under test. + * + * Tests SKIP where ffmpeg is absent. CI installs it (AnimMouse/setup-ffmpeg). + * + * Run: npx tsx test/continuous-test-suite-video.ts + */ + +import * as fs from "node:fs"; +import * as path from "node:path"; +import { + defineSuite, + assert, + assertEqual, + assertIncludes, + tempDir, + Skip, +} from "./helpers/harness.js"; +import { + hasFfmpeg, + makeVideoFile, + makeCorruptFile, +} from "./helpers/mediaFixtures.js"; +import { + videoProcessor, + isVideoFile, +} from "../src/lib/processors/media/VideoProcessor.js"; + +const { test, runSuite } = defineSuite("Video file support"); + +const dir = tempDir("neurolink-video-"); +let ready = false; + +async function ensureFixtures(): Promise { + if (ready) { + return; + } + if (!(await hasFfmpeg())) { + throw new Skip("ffmpeg not available — cannot synthesise video fixtures"); + } + await makeVideoFile(dir, "clip.mp4", 2); + makeCorruptFile(dir, "broken.mp4"); + ready = true; +} + +function info(file: string, mimetype: string) { + const full = path.join(dir, file); + return { + id: `video-${file}`, + name: file, + mimetype, + size: fs.statSync(full).size, + buffer: fs.readFileSync(full), + }; +} + +// --- VIDEO-026 (#498): detection --------------------------------------------- + +await test("isVideoFile accepts the common containers", () => { + assert(isVideoFile("video/mp4", "clip.mp4"), "mp4"); + assert(isVideoFile("video/webm", "clip.webm"), "webm"); + assert(isVideoFile("video/quicktime", "clip.mov"), "mov"); + assert(isVideoFile("video/x-matroska", "clip.mkv"), "mkv"); +}); + +await test("isVideoFile falls back to the extension", () => { + assert(isVideoFile("", "clip.mp4"), "extension alone identifies mp4"); + assert( + isVideoFile("application/octet-stream", "clip.mkv"), + "generic MIME falls back to the extension", + ); +}); + +await test("isVideoFile rejects audio and images", () => { + assertEqual(isVideoFile("audio/mpeg", "song.mp3"), false, "mp3 is not video"); + assertEqual(isVideoFile("image/png", "cat.png"), false, "png is not video"); +}); + +// --- VIDEO-025 (#495): probing and metadata ---------------------------------- + +await test("probes a real mp4 for dimensions, duration and codec", async () => { + await ensureFixtures(); + const result = await videoProcessor.processFile( + info("clip.mp4", "video/mp4"), + ); + assert(result.success, `mp4 processing failed: ${JSON.stringify(result)}`); + if (!result.success) { + return; + } + const { metadata } = result.data; + assertEqual(metadata.width, 320, "width read from the stream"); + assertEqual(metadata.height, 240, "height read from the stream"); + assert( + metadata.duration > 1.5 && metadata.duration < 3.5, + `duration ${metadata.duration}s outside the band for a 2s clip`, + ); + assert(metadata.codec.length > 0, "video codec reported"); + assert(metadata.fps > 0, "frame rate reported"); + assert( + metadata.durationFormatted.length > 0, + "a human-readable duration is produced", + ); + // NOTE: VideoProcessor renders this as "2s" while AudioProcessor renders the + // same field as "0:00" (m:ss). Both are reasonable in isolation, but they are + // sibling processors feeding the same model, so the inconsistency is worth + // settling. Asserted loosely here rather than blessing either format. + assertEqual(metadata.audioCodec, "aac", "muxed audio codec identified"); +}); + +await test("reports the audio track alongside the video track", async () => { + await ensureFixtures(); + const result = await videoProcessor.processFile( + info("clip.mp4", "video/mp4"), + ); + assert(result.success, "processing succeeds"); + if (!result.success) { + return; + } + // The fixture is muxed with a sine track; a processor that only probed the + // video stream would silently drop the audio a transcription path needs. + assert( + Boolean(result.data.metadata.audioCodec), + "audio codec is reported for a muxed file", + ); +}); + +await test("handles a second container (webm) the same way", async () => { + await ensureFixtures(); + // Not every ffmpeg build ships the VP8/Vorbis encoders — a fixture we cannot + // mint is a missing local codec, not a product defect, so skip rather than + // fail and rather than let it poison the mp4 tests. + try { + await makeVideoFile(dir, "clip.webm", 2, [ + "-c:v", + "libvpx", + "-c:a", + "libvorbis", + ]); + } catch { + throw new Skip("this ffmpeg build cannot encode VP8/Vorbis webm"); + } + const result = await videoProcessor.processFile( + info("clip.webm", "video/webm"), + ); + assert(result.success, `webm processing failed: ${JSON.stringify(result)}`); + if (!result.success) { + return; + } + assertEqual(result.data.metadata.width, 320, "webm width read"); + assert(result.data.metadata.duration > 0, "webm duration read"); +}); + +// --- VIDEO-027 (#502) / VIDEO-028 (#510): keyframes for frame-based providers - + +await test("extracts keyframes for frame-based providers", async () => { + await ensureFixtures(); + const result = await videoProcessor.processFile( + info("clip.mp4", "video/mp4"), + ); + assert(result.success, "processing succeeds"); + if (!result.success) { + return; + } + // Providers without native video (OpenAI, Anthropic) receive frames as + // images, so an empty keyframe array means those providers get nothing. + assert(Array.isArray(result.data.keyframes), "keyframes array is present"); + assert( + result.data.keyframes.length > 0, + "at least one keyframe is extracted — an empty array means frame-based providers receive nothing", + ); + for (const frame of result.data.keyframes) { + assert(Buffer.isBuffer(frame), "each keyframe is a Buffer"); + assert(frame.length > 0, "keyframe buffers are non-empty"); + } +}); + +await test("textContent describes the clip for the model", async () => { + await ensureFixtures(); + const result = await videoProcessor.processFile( + info("clip.mp4", "video/mp4"), + ); + assert(result.success, "processing succeeds"); + if (!result.success) { + return; + } + assertIncludes( + result.data.textContent, + "clip.mp4", + "the filename appears in the text handed to the model", + ); +}); + +// --- error paths -------------------------------------------------------------- + +await test("a corrupt file with a video extension does not throw", async () => { + await ensureFixtures(); + const result = await videoProcessor.processFile( + info("broken.mp4", "video/mp4"), + ); + // Whatever the verdict, the contract is a structured result — a mislabelled + // upload must not surface as an ffprobe stack trace. + assert( + typeof result.success === "boolean", + "returns a structured result rather than throwing", + ); +}); + +await test("an empty buffer does not throw", async () => { + const result = await videoProcessor.processFile({ + id: "video-empty", + name: "empty.mp4", + mimetype: "video/mp4", + size: 0, + buffer: Buffer.alloc(0), + }); + assert( + typeof result.success === "boolean", + "returns a structured result rather than throwing", + ); +}); + +try { + fs.rmSync(dir, { recursive: true, force: true }); +} catch { + /* ignore */ +} + +await runSuite(); diff --git a/test/helpers/mediaFixtures.ts b/test/helpers/mediaFixtures.ts new file mode 100644 index 000000000..df108245a --- /dev/null +++ b/test/helpers/mediaFixtures.ts @@ -0,0 +1,142 @@ +/** + * Synthesised audio/video fixtures for the multimodal suites. + * + * The fixtures are generated with ffmpeg at test time rather than committed. + * Real media is the only thing that exercises these processors — `music-metadata` + * and `ffprobe` both read container headers, so a hand-rolled byte string proves + * nothing — but committing binaries to grow the repo for every codec and edge + * case is a poor trade when ffmpeg can mint them deterministically. + * + * CI already installs ffmpeg (`AnimMouse/setup-ffmpeg` in ci.yml). Where it is + * absent, callers should skip rather than fail: a missing local tool is not a + * product defect. + */ + +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import * as fs from "node:fs"; +import * as path from "node:path"; + +const execFileAsync = promisify(execFile); + +/** + * Resolve an ffmpeg binary. + * + * Absolute candidates are checked BEFORE the bare name. Returning "ffmpeg" + * first would end the search there, so a machine whose PATH lacks ffmpeg but + * which has it at /usr/bin/ffmpeg would report "not available" and silently + * skip every media suite. + */ +export function findFfmpeg(): string | null { + const explicit = process.env.FFMPEG_PATH; + if (explicit && fs.existsSync(explicit)) { + return explicit; + } + + for (const absolute of [ + "/opt/homebrew/bin/ffmpeg", + "/usr/local/bin/ffmpeg", + "/usr/bin/ffmpeg", + ]) { + if (fs.existsSync(absolute)) { + return absolute; + } + } + + // Last resort: let execFile resolve it through PATH. hasFfmpeg() proves it + // actually runs, so a bare name that does not resolve fails there, not here. + return "ffmpeg"; +} + +/** True when ffmpeg can actually be invoked, not merely located. */ +export async function hasFfmpeg(): Promise { + const bin = findFfmpeg(); + if (!bin) { + return false; + } + try { + await execFileAsync(bin, ["-version"]); + return true; + } catch { + return false; + } +} + +/** + * Generate a tone as a real audio file. + * + * A sine source rather than silence: some encoders emit a degenerate stream for + * pure silence, and the point is to exercise a normal decode path. + */ +export async function makeAudioFile( + dir: string, + name: string, + seconds = 2, + extraArgs: string[] = [], +): Promise { + const bin = findFfmpeg(); + if (!bin) { + throw new Error("ffmpeg unavailable"); + } + const out = path.join(dir, name); + await execFileAsync(bin, [ + "-y", + "-f", + "lavfi", + "-i", + `sine=frequency=440:duration=${seconds}`, + ...extraArgs, + out, + ]); + return out; +} + +/** Generate a short colour-bar video with an audio track. */ +export async function makeVideoFile( + dir: string, + name: string, + seconds = 2, + extraArgs: string[] = [], +): Promise { + const bin = findFfmpeg(); + if (!bin) { + throw new Error("ffmpeg unavailable"); + } + const out = path.join(dir, name); + // The audio encoder is pinned rather than left to the muxer default: ffmpeg + // does not guarantee a particular default encoder for MP4, so a test that + // asserts on the resulting audio codec would be asserting on the local build + // configuration. `extraArgs` still wins, since it is appended after. + await execFileAsync(bin, [ + "-y", + "-f", + "lavfi", + "-i", + `testsrc=size=320x240:rate=10:duration=${seconds}`, + "-f", + "lavfi", + "-i", + `sine=frequency=440:duration=${seconds}`, + "-shortest", + "-pix_fmt", + "yuv420p", + "-c:a", + "aac", + ...extraArgs, + out, + ]); + return out; +} + +/** + * A file with a valid extension whose bytes are not that format. + * + * The interesting failure is not "no file" but "file that lies about itself" — + * a truncated upload or a mislabelled download, which is what reaches these + * processors in practice. + */ +export function makeCorruptFile(dir: string, name: string): string { + const out = path.join(dir, name); + fs.writeFileSync(out, Buffer.from("not really media, just ascii", "utf8")); + return out; +} diff --git a/test/helpers/officeFixtures.ts b/test/helpers/officeFixtures.ts new file mode 100644 index 000000000..25bdca279 --- /dev/null +++ b/test/helpers/officeFixtures.ts @@ -0,0 +1,113 @@ +/** + * Synthesised Office fixtures (DOCX / XLSX) for the office suite. + * + * Built in memory rather than committed. A .docx is a ZIP of XML parts and a + * .xlsx is written by the same exceljs the processor reads back, so both can be + * minted deterministically — and a generated fixture cannot drift out of sync + * with the format the processor expects the way a checked-in binary can. + * + * `exceljs` and `mammoth` are optionalDependencies, exactly as the processors + * treat them, so callers must skip when they are absent instead of failing. + * `adm-zip` is a direct dependency and always present. + */ + +import AdmZip from "adm-zip"; + +/** + * True when an optional package can actually be loaded. + * + * Only a genuine "this package is not installed" counts as absent. Swallowing + * every import failure would turn an installed-but-broken dependency — a + * syntax error, a bad ESM/CJS interop, a missing transitive dep — into a quiet + * SKIP, which is precisely the regression these suites exist to catch. Anything + * else rethrows so the suite fails loudly. + */ +export async function hasPackage(name: string): Promise { + try { + await import(/* @vite-ignore */ name); + return true; + } catch (err) { + const code = (err as NodeJS.ErrnoException | undefined)?.code; + const isMissing = + (code === "ERR_MODULE_NOT_FOUND" || code === "MODULE_NOT_FOUND") && + (err instanceof Error + ? err.message.includes(`'${name}'`) || err.message.includes(`"${name}"`) + : false); + if (isMissing) { + return false; + } + throw err; + } +} + +const CONTENT_TYPES = ` + + + + +`; + +const ROOT_RELS = ` + + +`; + +/** + * Build a minimal but genuinely valid .docx containing the given paragraphs. + * + * The three parts below are the smallest set Word and mammoth both accept: + * the content-type map, the package relationship pointing at the main part, + * and the document body itself. + */ +export function makeDocx(paragraphs: string[]): Buffer { + const body = paragraphs + .map( + (text) => + `${escapeXml(text)}`, + ) + .join(""); + + const documentXml = ` + + ${body} +`; + + const zip = new AdmZip(); + zip.addFile("[Content_Types].xml", Buffer.from(CONTENT_TYPES, "utf8")); + zip.addFile("_rels/.rels", Buffer.from(ROOT_RELS, "utf8")); + zip.addFile("word/document.xml", Buffer.from(documentXml, "utf8")); + return zip.toBuffer(); +} + +/** Build a real .xlsx via exceljs, normalising its CJS/ESM interop. */ +export async function makeXlsx( + sheets: Record, +): Promise { + const mod = (await import("exceljs")) as unknown as { + Workbook?: new () => unknown; + default?: { Workbook: new () => unknown }; + }; + const Workbook = mod.Workbook ?? mod.default?.Workbook; + if (!Workbook) { + throw new Error("exceljs Workbook constructor unresolved"); + } + const wb = new Workbook() as { + addWorksheet: (n: string) => { addRow: (r: unknown[]) => void }; + xlsx: { writeBuffer: () => Promise }; + }; + for (const [name, rows] of Object.entries(sheets)) { + const ws = wb.addWorksheet(name); + for (const row of rows) { + ws.addRow(row); + } + } + return Buffer.from(await wb.xlsx.writeBuffer()); +} + +function escapeXml(s: string): string { + return s + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """); +}