From d2b31f8e983f2398421fb6d8140f169f1e5370da Mon Sep 17 00:00:00 2001 From: LaZzyMan Date: Mon, 3 Aug 2026 18:30:26 +0800 Subject: [PATCH 01/14] =?UTF-8?q?feat(omni):=20S2=20input=20expansion=20?= =?UTF-8?q?=E2=80=94=20three=20modalities,=20URL=20sources,=20token=20guar?= =?UTF-8?q?d?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extends the S1 video-only upload delivery to the full S2 input surface: - recognition: generalized to image/audio/video with content sniffing (magic bytes for png/jpeg/webp/gif, mp3/wav/flac/ogg/m4a, plus the S1 video set) and per-modality ffprobe metadata; modality mismatches between sniff and reference fail closed - delivery: all modalities upload ORIGINAL bytes — no resizing or transcoding on the default path (lossy transforms are the job of omni policies, which must disclose); images skip renderImageOverview when omni is active and carry a dimensions+zoom-hint text part instead - converter: new fileData→input_audio branch passing the bare oss:// URL (previously audio fileData silently textified as 'Unsupported file media type' after a successful upload); getAudioFormat widened to flac/ogg/m4a in lockstep with the recognizer - URL inputs: @https://… is intercepted ahead of filesystem resolution (previously silently dropped via the ENOENT skip), streamed to downloads/ with SSRF/redirect policy mirroring fetchWithPolicy, hashed while writing, re-recognized from local bytes, and delivered through the same pipeline; per-URL failure isolation keeps the turn alive - token estimation: versioned raw-resource estimator (method raw-resource-v1) attached to read results as the single tokenEstimate location; formula is a swappable slot pending provider confirmation - transport guard: byte ceiling (S1) plus omni.transport.maxEstimatedTokens (default 0 = disabled — observability only until the formula is confirmed; positive values fail closed before any copy/upload) - second normalization trigger point: processToolResultOmniMedia converts inline tool-result media to oss:// fileData at both physical funnels (CoreToolScheduler terminal sites and ACP Session.runTool), as a sibling of the vision bridge — converted parts are invisible to isImagePart so bridge behavior is untouched Part of the omni-experiment S2 slice (#8184). --- .../src/acp-integration/session/Session.ts | 10 + packages/cli/src/config/config.ts | 2 + packages/cli/src/config/settingsSchema.ts | 61 +++++ .../cli/src/ui/hooks/atCommandProcessor.ts | 108 ++++++++ packages/core/src/config/config.ts | 16 ++ packages/core/src/core/coreToolScheduler.ts | 13 +- .../openaiContentGenerator/converter.test.ts | 94 ++++++- .../core/openaiContentGenerator/converter.ts | 43 +++- packages/core/src/index.ts | 23 ++ packages/core/src/omni/download.test.ts | 169 +++++++++++++ packages/core/src/omni/download.ts | 220 ++++++++++++++++ packages/core/src/omni/estimation.test.ts | 74 ++++++ packages/core/src/omni/estimation.ts | 95 +++++++ packages/core/src/omni/ffmpeg.ts | 92 +++++-- packages/core/src/omni/guard.test.ts | 107 ++++++++ packages/core/src/omni/guard.ts | 85 +++++++ packages/core/src/omni/index.ts | 234 +++++++++++++----- packages/core/src/omni/recognition.test.ts | 71 ++++++ packages/core/src/omni/recognition.ts | 211 ++++++++++++---- packages/core/src/omni/tool-result-media.ts | 123 +++++++++ packages/core/src/utils/fileUtils.ts | 46 +++- .../schemas/settings.schema.json | 24 ++ 22 files changed, 1759 insertions(+), 162 deletions(-) create mode 100644 packages/core/src/omni/download.test.ts create mode 100644 packages/core/src/omni/download.ts create mode 100644 packages/core/src/omni/estimation.test.ts create mode 100644 packages/core/src/omni/estimation.ts create mode 100644 packages/core/src/omni/guard.test.ts create mode 100644 packages/core/src/omni/guard.ts create mode 100644 packages/core/src/omni/tool-result-media.ts diff --git a/packages/cli/src/acp-integration/session/Session.ts b/packages/cli/src/acp-integration/session/Session.ts index 71194357f1b..558dedbf791 100644 --- a/packages/cli/src/acp-integration/session/Session.ts +++ b/packages/cli/src/acp-integration/session/Session.ts @@ -153,6 +153,7 @@ import { normalizeParts, runVisionBridge, bridgeToolResultImages, + processToolResultOmniMedia, shouldRunVisionBridge, formatVisionBridgeNotice, formatFullTurnVisionNotice, @@ -8263,6 +8264,15 @@ export class Session implements SessionContext { }); } + // Omni second normalization trigger point (parity with + // CoreToolScheduler.processToolResultImages, design §8.2): + // inline tool-result media becomes oss:// fileData before the + // vision bridge runs, which then skips the converted parts. + responseParts = await processToolResultOmniMedia( + responseParts, + this.config, + activeToolAbortSignal, + ); const visionBridgeNotices: string[] = []; responseParts = await bridgeToolResultImages({ config: this.config, diff --git a/packages/cli/src/config/config.ts b/packages/cli/src/config/config.ts index f3d31845351..7065d0c7137 100755 --- a/packages/cli/src/config/config.ts +++ b/packages/cli/src/config/config.ts @@ -2193,6 +2193,8 @@ export async function loadCliConfig( : undefined, omniEnabled: settings.omni?.enabled ?? false, omniUploadMaxFileBytes: settings.omni?.upload?.maxFileBytes, + omniMaxEstimatedTokens: settings.omni?.transport?.maxEstimatedTokens, + omniDownloadMaxFileBytes: settings.omni?.download?.maxFileBytes, // CDP tunnel (Plan C, #5626): with the tunnel on, browser automation goes // through the CDP tunnel (far lighter than the OS-level computer-use // driver), so disable computer-use to keep the agent off that heavy path. diff --git a/packages/cli/src/config/settingsSchema.ts b/packages/cli/src/config/settingsSchema.ts index 46921f6751b..4d3963b15ce 100644 --- a/packages/cli/src/config/settingsSchema.ts +++ b/packages/cli/src/config/settingsSchema.ts @@ -3677,6 +3677,67 @@ const SETTINGS_SCHEMA = { }, }, }, + transport: { + type: 'object', + label: 'Omni Transport Guard', + category: 'Experimental', + requiresRestart: true, + default: {}, + description: + 'Transport guard dimensions beyond the byte ceiling for omni ' + + 'media delivery.', + showInDialog: false, + properties: { + maxEstimatedTokens: { + type: 'number', + label: 'Max Estimated Tokens', + category: 'Experimental', + requiresRestart: true, + default: 0, + description: + 'Estimated-token ceiling for a single omni media input, ' + + 'checked before upload using the versioned raw-resource ' + + 'estimator. 0 disables the token guard (estimates are still ' + + 'attached for observability) — the estimation formula is ' + + 'pending confirmation with the model provider; set a ' + + 'positive threshold to enforce fail-closed rejection.', + showInDialog: false, + jsonSchemaOverride: { + type: 'number', + minimum: 0, + default: 0, + }, + }, + }, + }, + download: { + type: 'object', + label: 'Omni Download', + category: 'Experimental', + requiresRestart: true, + default: {}, + description: 'URL media localization limits for omni delivery.', + showInDialog: false, + properties: { + maxFileBytes: { + type: 'number', + label: 'Max Download File Bytes', + category: 'Experimental', + requiresRestart: true, + default: 0, + description: + 'Byte ceiling for downloading URL media inputs. 0 or unset ' + + 'follows omni.upload.maxFileBytes (downloading more than the ' + + 'upload channel can deliver is pointless).', + showInDialog: false, + jsonSchemaOverride: { + type: 'number', + minimum: 0, + default: 0, + }, + }, + }, + }, }, }, } as const satisfies SettingsSchema; diff --git a/packages/cli/src/ui/hooks/atCommandProcessor.ts b/packages/cli/src/ui/hooks/atCommandProcessor.ts index 87cf7f5ae04..8cad91ee800 100644 --- a/packages/cli/src/ui/hooks/atCommandProcessor.ts +++ b/packages/cli/src/ui/hooks/atCommandProcessor.ts @@ -251,6 +251,12 @@ export async function resolveAtCommandQuery({ ref: { id?: string; title?: string }; }> = []; + // URL media references (`@https://…`) collected during the loop and + // localized (downloaded → recognized → promoted into the omni object + // store) after it. Only active when omni delivery is on; otherwise URL + // tokens keep today's fall-through behavior (left as text). + const urlMediaRefs: Array<{ originalAtPath: string; url: string }> = []; + for (const atPathPart of atPathCommandParts) { const originalAtPath = atPathPart.content; // e.g., "@file.txt" or "@" @@ -263,6 +269,23 @@ export async function resolveAtCommandQuery({ const pathName = originalAtPath.substring(1); + // URL media reference (`@https://…`): detected BEFORE every other + // parser — a URL can't be an extension/session/MCP ref, and without + // this branch it would fall through to filesystem resolution where the + // ENOENT skip silently drops it. Only intercepted when omni delivery + // is active; otherwise preserve the legacy text fall-through. + if (/^https?:\/\//i.test(pathName)) { + const omni = await import('@qwen-code/qwen-code-core'); + if (omni.isOmniDeliveryActive(config) && omni.parseHttpUrlRef(pathName)) { + if (!urlMediaRefs.some((r) => r.url === pathName)) { + urlMediaRefs.push({ originalAtPath, url: pathName }); + } + // Keep the URL verbatim in the text sent to the model. + atPathToResolvedSpecMap.set(originalAtPath, pathName); + continue; + } + } + // Extension reference (`@ext:`): detected BEFORE MCP/filesystem // resolution. Only matches when the path starts with `ext:` and the name // corresponds to an active extension. @@ -505,6 +528,86 @@ export async function resolveAtCommandQuery({ ), ); + // URL media localization: download → recognize → promote into the omni + // object store → upload → fileData part. Sequential (media downloads can + // be large; parallel GiB transfers would contend), failure-isolated per + // URL (an error card is shown and the turn continues — mirroring the MCP + // resource block's Promise.allSettled semantics). + const urlMediaParts: Part[] = []; + const urlMediaDisplays: IndividualToolCallDisplay[] = []; + const urlMediaLabels: string[] = []; + if (urlMediaRefs.length > 0) { + const core = await import('@qwen-code/qwen-code-core'); + const os = await import('node:os'); + for (let i = 0; i < urlMediaRefs.length; i++) { + const ref = urlMediaRefs[i]; + const callId = `client-url-media-${userMessageTimestamp}-${i}`; + const hostLabel = (() => { + try { + return new URL(ref.url).hostname; + } catch { + return 'invalid-url'; + } + })(); + let tempDir: string | undefined; + try { + const store = new core.OmniObjectStore(config.storage.getQwenDir()); + const downloadsDir = path.join(store.getOmniRootDir(), 'downloads'); + const downloaded = await core.downloadMediaUrl({ + url: ref.url, + downloadsDir, + maxBytes: core.effectiveMaxDownloadFileBytes(config), + signal, + }); + tempDir = downloaded.partPath; + // Full local-file pipeline on the downloaded bytes: sniff/probe/ + // hash again (the design requires re-recognition from the local + // file — never trust transfer-time observations), guard, store, + // upload. readMediaViaOmniDelivery needs a stable name for + // display; use the URL basename. + const urlBase = path.basename(new URL(ref.url).pathname) || hostLabel; + const delivery = await core.processMediaForOmniDelivery( + downloaded.partPath, + config, + { signal }, + ); + urlMediaParts.push({ + fileData: { + fileUri: delivery.fileUri, + mimeType: delivery.mimeType, + displayName: urlBase, + }, + }); + urlMediaLabels.push(ref.url); + urlMediaDisplays.push({ + callId, + name: 'Fetch Media URL', + description: `Downloaded ${ref.url}`, + status: ToolCallStatus.Success, + resultDisplay: `Localized ${urlBase} (${delivery.recognized.modality}, ${(delivery.recognized.sizeBytes / 1024 / 1024).toFixed(1)}MB) and delivered via omni upload.`, + confirmationDetails: undefined, + }); + } catch (error) { + if (signal.aborted) throw error; + const reason = getErrorMessage(error); + onDebugMessage(`Failed to localize media URL ${ref.url}: ${reason}`); + urlMediaDisplays.push({ + callId, + name: 'Fetch Media URL', + description: `Download ${ref.url}`, + status: ToolCallStatus.Error, + resultDisplay: `Failed to fetch media from ${hostLabel}: ${reason}`, + confirmationDetails: undefined, + }); + } finally { + if (tempDir) { + await fs.rm(tempDir, { force: true }).catch(() => {}); + } + } + } + void os; // reserved for future temp-dir strategies + } + const resourceParts: Part[] = []; const resourceDisplays: IndividualToolCallDisplay[] = []; const resourceLabels: string[] = []; @@ -852,6 +955,7 @@ export async function resolveAtCommandQuery({ ...scopedMentionLabels, ...contentLabelsForDisplay, ...resourceLabels, + ...urlMediaLabels, ]; return { processedQuery: null, @@ -859,6 +963,7 @@ export async function resolveAtCommandQuery({ toolDisplays: [ ...scopedMentionDisplays, ...resourceDisplays, + ...urlMediaDisplays, errorToolCallDisplay, ], filesRead: labelsOnError, @@ -881,11 +986,13 @@ export async function resolveAtCommandQuery({ ...scopedMentionParts, ...fileParts, ...resourceParts, + ...urlMediaParts, ]; const allLabels = [ ...scopedMentionLabels, ...contentLabelsForDisplay, ...resourceLabels, + ...urlMediaLabels, ]; return { @@ -895,6 +1002,7 @@ export async function resolveAtCommandQuery({ ...scopedMentionDisplays, ...fileDisplays, ...resourceDisplays, + ...urlMediaDisplays, ], filesRead: allLabels, recording: { diff --git a/packages/core/src/config/config.ts b/packages/core/src/config/config.ts index 7b3effd5a1b..9d4b67e5dd7 100644 --- a/packages/core/src/config/config.ts +++ b/packages/core/src/config/config.ts @@ -1104,6 +1104,10 @@ export interface ConfigParameters { omniEnabled?: boolean; /** Per-file byte ceiling for omni media uploads (default 1 GiB). */ omniUploadMaxFileBytes?: number; + /** Estimated-token ceiling for omni media (0/unset = guard disabled). */ + omniMaxEstimatedTokens?: number; + /** Byte ceiling for omni URL downloads (unset = follow upload cap). */ + omniDownloadMaxFileBytes?: number; /** Image generation model selected through `/model --image`. */ imageModel?: string; /** @@ -1931,6 +1935,8 @@ export class Config { private readonly artifactOss?: ArtifactOssConfig; private readonly omniEnabled: boolean = false; private readonly omniUploadMaxFileBytes?: number; + private readonly omniMaxEstimatedTokens?: number; + private readonly omniDownloadMaxFileBytes?: number; private workflowsEnabled = false; private readonly skipWorkflowUsageWarning: boolean = false; private readonly computerUseEnabled: boolean = true; @@ -2208,6 +2214,8 @@ export class Config { this.artifactOss = params.artifactOss; this.omniEnabled = params.omniEnabled ?? false; this.omniUploadMaxFileBytes = params.omniUploadMaxFileBytes; + this.omniMaxEstimatedTokens = params.omniMaxEstimatedTokens; + this.omniDownloadMaxFileBytes = params.omniDownloadMaxFileBytes; this.workflowsEnabled = params.workflowsEnabled ?? false; this.skipWorkflowUsageWarning = params.skipWorkflowUsageWarning ?? false; this.computerUseEnabled = params.computerUseEnabled ?? true; @@ -6372,6 +6380,14 @@ export class Config { return this.omniUploadMaxFileBytes; } + getOmniMaxEstimatedTokens(): number | undefined { + return this.omniMaxEstimatedTokens; + } + + getOmniDownloadMaxFileBytes(): number | undefined { + return this.omniDownloadMaxFileBytes; + } + resolveImageGenerationModel( setting: string | undefined, ): ImageGenerationConfig | undefined { diff --git a/packages/core/src/core/coreToolScheduler.ts b/packages/core/src/core/coreToolScheduler.ts index 54f53613c18..6f0940c6bfd 100644 --- a/packages/core/src/core/coreToolScheduler.ts +++ b/packages/core/src/core/coreToolScheduler.ts @@ -173,6 +173,7 @@ import { normalizeParts, } from '../services/visionBridge/image-part-utils.js'; import { bridgeToolResultImages } from '../services/visionBridge/tool-result-vision-bridge.js'; +import { processToolResultOmniMedia } from '../omni/tool-result-media.js'; import { getInvocationContext, runWithInvocationContext, @@ -1387,9 +1388,19 @@ export class CoreToolScheduler { }> { let modelOverride: string | undefined; const notices: string[] = []; + // Omni second normalization trigger point: convert inline tool-result + // media into oss:// fileData BEFORE the vision bridge runs — converted + // parts are invisible to isImagePart, so the bridge skips them. Sibling + // step by design (§8.2): never mixed into bridge logic. Preserves the + // identity-equality contract: unchanged input returns the same array. + const omniProcessed = await processToolResultOmniMedia( + responseParts, + this.config, + signal, + ); const processedParts = await bridgeToolResultImages({ config: this.config, - responseParts, + responseParts: omniProcessed, signal, onFullTurnModel: (model) => { if (!this.onToolResultFullTurnModel?.(model)) return false; diff --git a/packages/core/src/core/openaiContentGenerator/converter.test.ts b/packages/core/src/core/openaiContentGenerator/converter.test.ts index 6807a5cfa9e..51b223bf100 100644 --- a/packages/core/src/core/openaiContentGenerator/converter.test.ts +++ b/packages/core/src/core/openaiContentGenerator/converter.test.ts @@ -2386,6 +2386,88 @@ describe('OpenAIContentConverter', () => { ); }); + it('should convert oss:// audio fileData to input_audio with the bare URL (omni upload delivery)', () => { + const request: GenerateContentParameters = { + model: 'models/test', + contents: [ + { + role: 'user', + parts: [ + { text: 'What is this sound?' }, + { + fileData: { + mimeType: 'audio/mpeg', + fileUri: + 'oss://dashscope-instant/uploads/model/abc/12345678-tone.mp3', + displayName: 'tone.mp3', + }, + }, + ], + }, + ], + }; + + const messages = converter.convertGeminiRequestToOpenAI( + request, + requestContext, + ); + + const userMessage = messages.find((message) => message.role === 'user'); + const contentArray = userMessage?.content as Array<{ + type: string; + input_audio?: { data: string; format: string }; + }>; + const audioPart = contentArray.find((p) => p.type === 'input_audio'); + // Bare oss URL — no data: prefix (unlike the inline branch). + expect(audioPart?.input_audio?.data).toBe( + 'oss://dashscope-instant/uploads/model/abc/12345678-tone.mp3', + ); + expect(audioPart?.input_audio?.format).toBe('mp3'); + }); + + it('should convert flac/ogg/m4a audio fileData instead of textifying', () => { + for (const [mime, format] of [ + ['audio/flac', 'flac'], + ['audio/ogg', 'ogg'], + ['audio/mp4', 'm4a'], + ] as const) { + const request: GenerateContentParameters = { + model: 'models/test', + contents: [ + { + role: 'user', + parts: [ + { + fileData: { + mimeType: mime, + fileUri: 'oss://bucket/key', + displayName: 'clip', + }, + }, + ], + }, + ], + }; + const messages = converter.convertGeminiRequestToOpenAI( + request, + requestContext, + ); + const userMessage = messages.find((m) => m.role === 'user'); + const contentArray = userMessage?.content as Array<{ + type: string; + input_audio?: { data: string; format: string }; + text?: string; + }>; + const audioPart = contentArray.find((p) => p.type === 'input_audio'); + expect(audioPart?.input_audio?.format).toBe(format); + expect( + contentArray.some((p) => + p.text?.includes('Unsupported file media type'), + ), + ).toBe(false); + } + }); + it('should render unsupported inlineData file types as a text block', () => { const request: GenerateContentParameters = { model: 'models/test', @@ -2447,7 +2529,7 @@ describe('OpenAIContentConverter', () => { expect(contentArray[1].text).toContain('archive.zip'); }); - it('should render unsupported fileData types (including audio) as a text block', () => { + it('should render unsupported fileData types as a text block', () => { const request: GenerateContentParameters = { model: 'models/test', contents: [ @@ -2474,9 +2556,9 @@ describe('OpenAIContentConverter', () => { parts: [ { fileData: { - mimeType: 'audio/mpeg', - fileUri: 'https://example.com/audio.mp3', - displayName: 'audio.mp3', + mimeType: 'application/zip', + fileUri: 'https://example.com/archive.zip', + displayName: 'archive.zip', }, }, ], @@ -2504,8 +2586,8 @@ describe('OpenAIContentConverter', () => { expect(contentArray[0].text).toBe('File content'); expect(contentArray[1].type).toBe('text'); expect(contentArray[1].text).toContain('Unsupported file media type'); - expect(contentArray[1].text).toContain('audio/mpeg'); - expect(contentArray[1].text).toContain('audio.mp3'); + expect(contentArray[1].text).toContain('application/zip'); + expect(contentArray[1].text).toContain('archive.zip'); }); it('should create tool message with text-only content when no media parts', () => { diff --git a/packages/core/src/core/openaiContentGenerator/converter.ts b/packages/core/src/core/openaiContentGenerator/converter.ts index 6585d0cc312..5b69f6c24f7 100644 --- a/packages/core/src/core/openaiContentGenerator/converter.ts +++ b/packages/core/src/core/openaiContentGenerator/converter.ts @@ -921,7 +921,9 @@ function createMediaContentPart( type: 'input_audio' as const, input_audio: { data: `data:${mimeType};base64,${part.inlineData.data}`, - format, + // DashScope accepts flac/ogg/m4a beyond the OpenAI SDK's + // wav|mp3 union; the request wire format is identical. + format: format as 'wav' | 'mp3', }, }; } @@ -998,6 +1000,33 @@ function createMediaContentPart( }; } + if (mediaType === 'audio') { + if (!modalities.audio) { + return unsupportedModalityPlaceholder( + 'audio', + filename, + requestContext, + ); + } + const format = getAudioFormat(mimeType); + if (format) { + // Unlike the inline branch (data: URI), the upload channel passes + // the bare URL — DashScope's input_audio.data accepts either, and + // oss:// references are resolved server-side via the + // X-DashScope-OssResourceResolve header (verified live 2026-08-03 + // on qwen3.5-omni-plus). + return { + type: 'input_audio' as const, + input_audio: { + data: fileUri, + // See inline branch: DashScope accepts a wider format set + // than the OpenAI SDK union. + format: format as 'wav' | 'mp3', + }, + }; + } + } + const displayNameStr = part.fileData.displayName ? ` (${part.fileData.displayName})` : ''; @@ -1042,9 +1071,19 @@ function getMediaType(mimeType: string): 'image' | 'audio' | 'video' | 'file' { return 'file'; } -function getAudioFormat(mimeType: string): 'wav' | 'mp3' | null { +/** + * Audio formats the DashScope input_audio channel accepts. Kept in + * lockstep with the omni recognizer's audio sniff set — an upload the + * recognizer accepts must never textify here after paying for transfer. + */ +function getAudioFormat( + mimeType: string, +): 'wav' | 'mp3' | 'flac' | 'ogg' | 'm4a' | null { if (mimeType.includes('wav')) return 'wav'; if (mimeType.includes('mp3') || mimeType.includes('mpeg')) return 'mp3'; + if (mimeType.includes('flac')) return 'flac'; + if (mimeType.includes('ogg')) return 'ogg'; + if (mimeType === 'audio/mp4' || mimeType.includes('m4a')) return 'm4a'; return null; } diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index ddaab28a003..962bf58b41b 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -662,3 +662,26 @@ export { type StartupEventSink, type StartupEventAttrs, } from './utils/startupEventSink.js'; + +// ============================================================================ +// Omni multimodal experiment — upload-based media delivery +// ============================================================================ + +export { + isOmniDeliveryActive, + processMediaForOmniDelivery, + readMediaViaOmniDelivery, + parseHttpUrlRef, + downloadMediaUrl, + effectiveMaxDownloadFileBytes, + recognizeMediaFile, + OmniObjectStore, + OmniDeliveryError, + OmniDownloadError, + OmniTransportGuardError, + type OmniModality, + type OmniMediaDelivery, + type OmniTokenEstimate, + type DownloadedMedia, +} from './omni/index.js'; +export { processToolResultOmniMedia } from './omni/tool-result-media.js'; diff --git a/packages/core/src/omni/download.test.ts b/packages/core/src/omni/download.test.ts new file mode 100644 index 00000000000..70d51fed7e4 --- /dev/null +++ b/packages/core/src/omni/download.test.ts @@ -0,0 +1,169 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { createHash } from 'node:crypto'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { downloadMediaUrl, OmniDownloadError } from './download.js'; + +let downloadsDir: string; + +beforeEach(async () => { + downloadsDir = await fs.mkdtemp(path.join(os.tmpdir(), 'omni-dl-')); +}); + +afterEach(async () => { + await fs.rm(downloadsDir, { recursive: true, force: true }); +}); + +function fetchOk( + body: Buffer | string, + headers: Record = {}, +): typeof fetch { + return vi.fn(async () => new Response(body, { status: 200, headers })); +} + +async function listParts(): Promise { + return (await fs.readdir(downloadsDir)).filter((f) => f.endsWith('.part')); +} + +describe('downloadMediaUrl', () => { + it('streams to a .part file and hashes while writing', async () => { + const bytes = Buffer.from('media-bytes-here'); + const result = await downloadMediaUrl({ + url: 'https://media.example.com/a.mp4', + downloadsDir, + maxBytes: 1_000_000, + fetchFn: fetchOk(bytes, { 'content-type': 'video/mp4' }), + }); + expect(result.sha256).toBe( + createHash('sha256').update(bytes).digest('hex'), + ); + expect(result.sizeBytes).toBe(bytes.length); + expect(result.contentType).toBe('video/mp4'); + await expect(fs.readFile(result.partPath)).resolves.toEqual(bytes); + }); + + it('refuses private/loopback hosts (SSRF)', async () => { + const fetchFn = vi.fn(); + for (const url of [ + 'http://127.0.0.1:8734/x.mp4', + 'http://localhost:8734/x.mp4', + 'http://10.0.0.5/x.mp4', + 'http://internal.lan/x.mp4', + ]) { + await expect( + downloadMediaUrl({ url, downloadsDir, maxBytes: 1000, fetchFn }), + ).rejects.toThrow(/not publicly routable/); + } + expect(fetchFn).not.toHaveBeenCalled(); + expect(await listParts()).toEqual([]); + }); + + it('rejects oversized Content-Length before reading the body', async () => { + await expect( + downloadMediaUrl({ + url: 'https://m.example.com/big.mp4', + downloadsDir, + maxBytes: 10, + fetchFn: fetchOk(Buffer.alloc(100), { 'content-length': '100' }), + }), + ).rejects.toThrow(/Content-Length 100 > 10 bytes/); + expect(await listParts()).toEqual([]); + }); + + it('enforces the byte cap on actual bytes even without Content-Length', async () => { + // Streamed response with no content-length header. + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array(64)); + controller.enqueue(new Uint8Array(64)); + controller.close(); + }, + }); + const fetchFn = vi.fn( + async () => new Response(stream, { status: 200 }), + ) as unknown as typeof fetch; + await expect( + downloadMediaUrl({ + url: 'https://m.example.com/nolen.mp4', + downloadsDir, + maxBytes: 100, + fetchFn, + }), + ).rejects.toThrow(/>100 bytes received/); + expect(await listParts()).toEqual([]); // .part cleaned on failure + }); + + it('surfaces HTTP errors with status and host, cleaning the .part', async () => { + await expect( + downloadMediaUrl({ + url: 'https://m.example.com/nope.mp4', + downloadsDir, + maxBytes: 1000, + fetchFn: vi.fn(async () => new Response('gone', { status: 404 })), + }), + ).rejects.toThrow(/HTTP 404 from m\.example\.com/); + expect(await listParts()).toEqual([]); + }); + + it('follows same-origin redirects and refuses cross-origin ones', async () => { + const bytes = Buffer.from('after-redirect'); + const sameOrigin = vi + .fn() + .mockResolvedValueOnce( + new Response(null, { + status: 302, + headers: { location: 'https://m.example.com/real.mp4' }, + }), + ) + .mockResolvedValueOnce(new Response(bytes, { status: 200 })); + const ok = await downloadMediaUrl({ + url: 'https://m.example.com/a.mp4', + downloadsDir, + maxBytes: 1000, + fetchFn: sameOrigin, + }); + expect(ok.finalUrl).toBe('https://m.example.com/real.mp4'); + expect(ok.sizeBytes).toBe(bytes.length); + + const crossOrigin = vi.fn( + async () => + new Response(null, { + status: 302, + headers: { location: 'https://evil.example.net/x.mp4' }, + }), + ) as unknown as typeof fetch; + await expect( + downloadMediaUrl({ + url: 'https://m.example.com/a.mp4', + downloadsDir, + maxBytes: 1000, + fetchFn: crossOrigin, + }), + ).rejects.toThrow(/Cross-origin redirect refused/); + }); + + it('propagates user aborts and cleans up', async () => { + const controller = new AbortController(); + const fetchFn = vi.fn(async () => { + controller.abort(); + throw Object.assign(new Error('aborted'), { name: 'AbortError' }); + }) as unknown as typeof fetch; + const err = await downloadMediaUrl({ + url: 'https://m.example.com/a.mp4', + downloadsDir, + maxBytes: 1000, + signal: controller.signal, + fetchFn, + }).catch((e: Error) => e); + expect((err as Error).name).toBe('AbortError'); + expect(err).not.toBeInstanceOf(OmniDownloadError); + expect(await listParts()).toEqual([]); + }); +}); diff --git a/packages/core/src/omni/download.ts b/packages/core/src/omni/download.ts new file mode 100644 index 00000000000..3392456daed --- /dev/null +++ b/packages/core/src/omni/download.ts @@ -0,0 +1,220 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { createHash, randomBytes } from 'node:crypto'; +import { createWriteStream } from 'node:fs'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { Readable } from 'node:stream'; +import { pipeline } from 'node:stream/promises'; +import { isPrivateHost, isPermittedRedirect } from '../utils/fetch.js'; +import { createDebugLogger } from '../utils/debugLogger.js'; + +const debugLogger = createDebugLogger('omni:download'); + +/** Thrown for URL localization failures. Messages must stay free of + * credentials; they may include the URL host and HTTP status. */ +export class OmniDownloadError extends Error { + constructor(message: string) { + super(message); + this.name = 'OmniDownloadError'; + } +} + +export interface DownloadedMedia { + /** Path of the fully-written temp file (inside downloads/). Caller is + * responsible for recognition, promotion into objects/, and cleanup. */ + partPath: string; + /** SHA-256 computed while streaming (identity of the downloaded bytes). */ + sha256: string; + /** Total bytes written. */ + sizeBytes: number; + /** Final URL after any permitted redirects. */ + finalUrl: string; + /** Content-Type reported by the server (advisory only — sniff decides). */ + contentType?: string; +} + +const HEADER_TIMEOUT_MS = 30_000; +const IDLE_TIMEOUT_MS = 60_000; +const MAX_REDIRECTS = 5; + +/** Test whether an @-path is a downloadable http(s) URL. */ +export function parseHttpUrlRef(pathName: string): URL | null { + if (!/^https?:\/\//i.test(pathName)) return null; + try { + return new URL(pathName); + } catch { + return null; + } +} + +/** + * Stream a media URL into `downloads/.part`, hashing while writing. + * + * Policy (mirrors utils/fetch.ts fetchWithPolicy, but streaming — that + * helper buffers whole bodies in memory and is unsuitable for GiB media): + * - https or http; private/loopback hosts refused (SSRF); + * - redirects followed only when same-host/protocol/port (isPermittedRedirect), + * bounded by MAX_REDIRECTS; + * - `maxBytes` enforced against BOTH Content-Length and actual bytes written; + * - two-timer model: header timeout (30s, cleared when headers arrive) plus + * an idle watchdog reset per chunk (60s) — a slow large body is fine, a + * stalled one is not; + * - on ANY failure the `.part` file is removed (no half-download survives). + */ +export async function downloadMediaUrl(params: { + url: string; + downloadsDir: string; + maxBytes: number; + signal?: AbortSignal; + fetchFn?: typeof fetch; +}): Promise { + const { url, downloadsDir, maxBytes, signal } = params; + const fetchFn = params.fetchFn ?? fetch; + + if (isPrivateHost(url)) { + throw new OmniDownloadError( + `URL host is not publicly routable (refused for safety): ${new URL(url).hostname}`, + ); + } + + await fs.mkdir(downloadsDir, { recursive: true, mode: 0o700 }); + const partPath = path.join( + downloadsDir, + `${randomBytes(8).toString('hex')}.part`, + ); + + let currentUrl = url; + try { + for (let redirects = 0; ; redirects++) { + const headerAbort = new AbortController(); + const headerTimer = setTimeout( + () => headerAbort.abort(new Error('header timeout')), + HEADER_TIMEOUT_MS, + ); + const combined = AbortSignal.any( + signal ? [signal, headerAbort.signal] : [headerAbort.signal], + ); + let res: Response; + try { + res = await fetchFn(currentUrl, { + redirect: 'manual', + signal: combined, + }); + } catch (err) { + if (signal?.aborted) throw err; + throw new OmniDownloadError( + `Download request failed for ${new URL(currentUrl).hostname}: ${ + err instanceof Error ? err.message : String(err) + }`, + ); + } finally { + clearTimeout(headerTimer); + } + + if (res.status >= 300 && res.status < 400) { + await res.body?.cancel().catch(() => {}); + const location = res.headers.get('location'); + if (!location || redirects >= MAX_REDIRECTS) { + throw new OmniDownloadError( + `Too many or invalid redirects downloading from ${new URL(currentUrl).hostname}`, + ); + } + const redirectUrl = new URL(location, currentUrl).toString(); + if (!isPermittedRedirect(currentUrl, redirectUrl)) { + throw new OmniDownloadError( + `Cross-origin redirect refused: ${new URL(currentUrl).hostname} → ${new URL(redirectUrl).hostname}`, + ); + } + currentUrl = redirectUrl; + continue; + } + + if (!res.ok) { + await res.body?.cancel().catch(() => {}); + throw new OmniDownloadError( + `Download failed: HTTP ${res.status} from ${new URL(currentUrl).hostname}`, + ); + } + + const contentLength = Number(res.headers.get('content-length')); + if (Number.isFinite(contentLength) && contentLength > maxBytes) { + await res.body?.cancel().catch(() => {}); + throw new OmniDownloadError( + `Download exceeds the omni media limit: Content-Length ${contentLength} > ${maxBytes} bytes.`, + ); + } + if (!res.body) { + throw new OmniDownloadError('Download response had no body.'); + } + + // Stream to disk with hash + byte cap + idle watchdog. + const hash = createHash('sha256'); + let written = 0; + const idleAbort = new AbortController(); + let idleTimer = setTimeout( + () => idleAbort.abort(new Error('idle timeout')), + IDLE_TIMEOUT_MS, + ); + const streamSignal = AbortSignal.any( + signal ? [signal, idleAbort.signal] : [idleAbort.signal], + ); + try { + const source = Readable.fromWeb( + res.body as import('node:stream/web').ReadableStream, + { signal: streamSignal }, + ); + const counter = async function* (src: AsyncIterable) { + for await (const chunk of src) { + clearTimeout(idleTimer); + idleTimer = setTimeout( + () => idleAbort.abort(new Error('idle timeout')), + IDLE_TIMEOUT_MS, + ); + written += chunk.length; + if (written > maxBytes) { + throw new OmniDownloadError( + `Download exceeds the omni media limit: >${maxBytes} bytes received.`, + ); + } + hash.update(chunk); + yield chunk; + } + }; + await pipeline( + source, + counter, + createWriteStream(partPath, { mode: 0o600 }), + ); + } catch (err) { + if (signal?.aborted) throw err; + if (err instanceof OmniDownloadError) throw err; + throw new OmniDownloadError( + `Download interrupted from ${new URL(currentUrl).hostname}: ${ + err instanceof Error ? err.message : String(err) + }`, + ); + } finally { + clearTimeout(idleTimer); + } + + debugLogger.debug( + `downloaded ${written} bytes from ${new URL(currentUrl).hostname}`, + ); + return { + partPath, + sha256: hash.digest('hex'), + sizeBytes: written, + finalUrl: currentUrl, + contentType: res.headers.get('content-type') ?? undefined, + }; + } + } catch (err) { + await fs.rm(partPath, { force: true }); + throw err; + } +} diff --git a/packages/core/src/omni/estimation.test.ts b/packages/core/src/omni/estimation.test.ts new file mode 100644 index 00000000000..8e05e1b1862 --- /dev/null +++ b/packages/core/src/omni/estimation.test.ts @@ -0,0 +1,74 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { describe, expect, it } from 'vitest'; +import { estimateRawResourceTokens } from './estimation.js'; +import type { RecognizedMedia } from './recognition.js'; + +function media( + modality: RecognizedMedia['modality'], + metadata: RecognizedMedia['metadata'], +): RecognizedMedia { + return { + modality, + sha256: 'a'.repeat(64), + detectedMimeType: 'x/y', + sizeBytes: 1, + metadata, + }; +} + +describe('estimateRawResourceTokens (raw-resource-v1)', () => { + it('audio: ceil(seconds × 7)', () => { + expect( + estimateRawResourceTokens(media('audio', { durationMs: 5000 })), + ).toEqual({ + estimatedTokenCount: 35, + method: 'raw-resource-v1', + status: 'ok', + }); + // 5.1s → ceil(35.7) = 36 + expect( + estimateRawResourceTokens(media('audio', { durationMs: 5100 })) + .estimatedTokenCount, + ).toBe(36); + }); + + it('image: ceil(w×h / (32×32×2)), frameCount 1', () => { + const est = estimateRawResourceTokens( + media('image', { width: 800, height: 600 }), + ); + expect(est.estimatedTokenCount).toBe(Math.ceil((800 * 600) / 2048)); + expect(est.status).toBe('ok'); + }); + + it('video: frameCount from duration × frameRate', () => { + const est = estimateRawResourceTokens( + media('video', { + width: 852, + height: 480, + durationMs: 10_000, + frameRate: 30, + }), + ); + expect(est.estimatedTokenCount).toBe(Math.ceil((852 * 480 * 300) / 2048)); + expect(est.method).toBe('raw-resource-v1'); + }); + + it('missing required fields → unavailable, never guessed', () => { + expect(estimateRawResourceTokens(media('audio', {})).status).toBe( + 'unavailable', + ); + expect( + estimateRawResourceTokens(media('image', { width: 100 })).status, + ).toBe('unavailable'); + expect( + estimateRawResourceTokens( + media('video', { width: 100, height: 100, durationMs: 1000 }), + ).status, + ).toBe('unavailable'); + }); +}); diff --git a/packages/core/src/omni/estimation.ts b/packages/core/src/omni/estimation.ts new file mode 100644 index 00000000000..831b3606e1a --- /dev/null +++ b/packages/core/src/omni/estimation.ts @@ -0,0 +1,95 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import type { RecognizedMedia } from './recognition.js'; + +/** + * Raw-resource token estimate for an omni media input. + * + * The formula set is versioned via `method` so a revised estimator (e.g. + * one aligned with confirmed server-side accounting) can be added and + * switched without touching call sites. Consumers must treat + * `status: 'unavailable'` as "no estimate" — never guess missing fields. + */ +export interface OmniTokenEstimate { + /** Estimated token count for the raw resource. */ + estimatedTokenCount: number; + /** Estimator identity+version that produced this value. */ + method: 'raw-resource-v1'; + status: 'ok' | 'unavailable'; +} + +const AUDIO_TOKENS_PER_SECOND = 7; +/** Pixels per visual token: 32×32 patch, 2 pixels per patch cell. */ +const VISUAL_PIXELS_PER_TOKEN = 32 * 32 * 2; + +/** + * Estimate tokens from recognized raw-resource metadata (design doc §6.4): + * + * audioTokens = ceil(durationSeconds × 7) + * visualTokens = ceil(width × height × frameCount / (32 × 32 × 2)) + * + * - static image → frameCount 1 + * - video → frameCount from ceil(duration × frameRate); audio track of a + * video is NOT separately estimated in v1 (the visual term dominates and + * the raw formula is already a conservative upper bound) + * - any required field missing → status 'unavailable' + */ +export function estimateRawResourceTokens( + media: RecognizedMedia, +): OmniTokenEstimate { + const unavailable: OmniTokenEstimate = { + estimatedTokenCount: 0, + method: 'raw-resource-v1', + status: 'unavailable', + }; + + switch (media.modality) { + case 'audio': { + const durationMs = media.metadata.durationMs; + if (durationMs === undefined) return unavailable; + return { + estimatedTokenCount: Math.ceil( + (durationMs / 1000) * AUDIO_TOKENS_PER_SECOND, + ), + method: 'raw-resource-v1', + status: 'ok', + }; + } + case 'image': { + const { width, height } = media.metadata; + if (width === undefined || height === undefined) return unavailable; + return { + estimatedTokenCount: Math.ceil( + (width * height) / VISUAL_PIXELS_PER_TOKEN, + ), + method: 'raw-resource-v1', + status: 'ok', + }; + } + case 'video': { + const { width, height, durationMs, frameRate } = media.metadata; + if ( + width === undefined || + height === undefined || + durationMs === undefined || + frameRate === undefined + ) { + return unavailable; + } + const frameCount = Math.ceil((durationMs / 1000) * frameRate); + return { + estimatedTokenCount: Math.ceil( + (width * height * frameCount) / VISUAL_PIXELS_PER_TOKEN, + ), + method: 'raw-resource-v1', + status: 'ok', + }; + } + default: + return unavailable; + } +} diff --git a/packages/core/src/omni/ffmpeg.ts b/packages/core/src/omni/ffmpeg.ts index ec871dcbcf4..9ae682f7f10 100644 --- a/packages/core/src/omni/ffmpeg.ts +++ b/packages/core/src/omni/ffmpeg.ts @@ -121,22 +121,29 @@ export async function assertOmniRuntimeDependencies(): Promise { ); } -/** Basic video metadata extracted via ffprobe. */ -export interface VideoProbeResult { +/** Media metadata extracted via ffprobe (fields populated per modality). */ +export interface MediaProbeResult { /** Container/format name reported by ffprobe (e.g. "mov,mp4,m4a,..."). */ formatName?: string; - /** Duration in milliseconds, when reported. */ + /** Duration in milliseconds (video/audio), when reported. */ durationMs?: number; - /** Width in pixels of the first video stream. */ + /** Width in pixels (video: first video stream; image: the image). */ width?: number; - /** Height in pixels of the first video stream. */ + /** Height in pixels. */ height?: number; - /** Average frame rate (frames per second) of the first video stream. */ + /** Average frame rate (fps) of the first video stream (video only). */ frameRate?: number; - /** Codec name of the first video stream (e.g. "h264"). */ + /** Codec name of the primary stream for the modality. */ codec?: string; + /** Sample rate in Hz of the first audio stream (audio only). */ + sampleRateHz?: number; + /** Channel count of the first audio stream (audio only). */ + channels?: number; } +/** @deprecated S1 name kept for existing imports. */ +export type VideoProbeResult = MediaProbeResult; + /** Parse an ffprobe rational like "30000/1001" (or plain "25") into fps. */ function parseFrameRate(raw: string | undefined): number | undefined { if (!raw) return undefined; @@ -151,16 +158,18 @@ function parseFrameRate(raw: string | undefined): number | undefined { } /** - * Probe a local video file with ffprobe. Throws on non-zero exit or + * Probe a local media file with ffprobe. Throws on non-zero exit or * unparseable output — the omni pipeline treats a failed probe as a * recognition failure (fail closed), not as "probably fine". Error * messages carry the file's basename only (they can reach model-visible - * content). + * content). ffprobe handles images too (single video stream, no duration), + * so the omni path needs no sharp dependency. */ -export async function probeVideoMetadata( +export async function probeMediaMetadata( filePath: string, + modality: 'image' | 'audio' | 'video', signal?: AbortSignal, -): Promise { +): Promise { const { stdout, code, stderr } = await execCommand( 'ffprobe', [ @@ -191,6 +200,8 @@ export async function probeVideoMetadata( height?: number; avg_frame_rate?: string; r_frame_rate?: string; + sample_rate?: string; + channels?: number; }>; }; try { @@ -201,18 +212,51 @@ export async function probeVideoMetadata( ); } const videoStream = parsed.streams?.find((s) => s.codec_type === 'video'); + const audioStream = parsed.streams?.find((s) => s.codec_type === 'audio'); const durationSeconds = Number(parsed.format?.duration); - return { - formatName: parsed.format?.format_name, - durationMs: - Number.isFinite(durationSeconds) && durationSeconds >= 0 - ? Math.round(durationSeconds * 1000) - : undefined, - width: videoStream?.width, - height: videoStream?.height, - frameRate: - parseFrameRate(videoStream?.avg_frame_rate) ?? - parseFrameRate(videoStream?.r_frame_rate), - codec: videoStream?.codec_name, - }; + const durationMs = + Number.isFinite(durationSeconds) && durationSeconds >= 0 + ? Math.round(durationSeconds * 1000) + : undefined; + + const base: MediaProbeResult = { formatName: parsed.format?.format_name }; + switch (modality) { + case 'image': + return { + ...base, + width: videoStream?.width, + height: videoStream?.height, + codec: videoStream?.codec_name, + }; + case 'audio': + return { + ...base, + durationMs, + codec: audioStream?.codec_name, + sampleRateHz: audioStream?.sample_rate + ? Number(audioStream.sample_rate) + : undefined, + channels: audioStream?.channels, + }; + case 'video': + default: + return { + ...base, + durationMs, + width: videoStream?.width, + height: videoStream?.height, + frameRate: + parseFrameRate(videoStream?.avg_frame_rate) ?? + parseFrameRate(videoStream?.r_frame_rate), + codec: videoStream?.codec_name, + }; + } +} + +/** @deprecated S1 wrapper: video probe. */ +export async function probeVideoMetadata( + filePath: string, + signal?: AbortSignal, +): Promise { + return probeMediaMetadata(filePath, 'video', signal); } diff --git a/packages/core/src/omni/guard.test.ts b/packages/core/src/omni/guard.test.ts new file mode 100644 index 00000000000..201db6a7fff --- /dev/null +++ b/packages/core/src/omni/guard.test.ts @@ -0,0 +1,107 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { describe, expect, it, vi } from 'vitest'; +import type { Config } from '../config/config.js'; +import { + assertWithinByteLimit, + assertWithinTokenLimit, + effectiveMaxUploadFileBytes, + DEFAULT_OMNI_MAX_UPLOAD_FILE_BYTES, + OmniTransportGuardError, +} from './guard.js'; +import type { RecognizedMedia } from './recognition.js'; + +function cfg(overrides: { maxBytes?: number; maxTokens?: number }): Config { + return { + getOmniUploadMaxFileBytes: vi.fn().mockReturnValue(overrides.maxBytes), + getOmniMaxEstimatedTokens: vi.fn().mockReturnValue(overrides.maxTokens), + } as unknown as Config; +} + +const AUDIO_5S: RecognizedMedia = { + modality: 'audio', + sha256: 'a'.repeat(64), + detectedMimeType: 'audio/mpeg', + sizeBytes: 40_000, + metadata: { durationMs: 5000 }, +}; + +const VIDEO_8MIN: RecognizedMedia = { + modality: 'video', + sha256: 'b'.repeat(64), + detectedMimeType: 'video/mp4', + sizeBytes: 36_000_000, + metadata: { width: 852, height: 480, durationMs: 506_000, frameRate: 30 }, +}; + +describe('byte guard', () => { + it('uses the default 1 GiB when unset or non-positive', () => { + expect(effectiveMaxUploadFileBytes(cfg({}))).toBe( + DEFAULT_OMNI_MAX_UPLOAD_FILE_BYTES, + ); + expect(effectiveMaxUploadFileBytes(cfg({ maxBytes: 0 }))).toBe( + DEFAULT_OMNI_MAX_UPLOAD_FILE_BYTES, + ); + expect(effectiveMaxUploadFileBytes(cfg({ maxBytes: -5 }))).toBe( + DEFAULT_OMNI_MAX_UPLOAD_FILE_BYTES, + ); + }); + + it('rejects above the configured limit with an explanatory message', () => { + expect(() => + assertWithinByteLimit(cfg({ maxBytes: 1000 }), 2000, 'clip.mp4'), + ).toThrow(/clip\.mp4.*2000 bytes > 1000 bytes.*omni\.upload\.maxFileBytes/); + }); + + it('passes at or below the limit', () => { + expect(() => + assertWithinByteLimit(cfg({ maxBytes: 1000 }), 1000, 'clip.mp4'), + ).not.toThrow(); + }); +}); + +describe('token guard', () => { + it('disabled (unset/0) attaches the estimate but never rejects', () => { + const est = assertWithinTokenLimit(cfg({}), VIDEO_8MIN, 'p3.mp4'); + expect(est.status).toBe('ok'); + expect(est.estimatedTokenCount).toBeGreaterThan(196_608); + expect( + assertWithinTokenLimit(cfg({ maxTokens: 0 }), VIDEO_8MIN, 'p3.mp4') + .status, + ).toBe('ok'); + }); + + it('enabled: rejects above threshold with estimate, threshold, and setting name', () => { + expect(() => + assertWithinTokenLimit(cfg({ maxTokens: 196_608 }), VIDEO_8MIN, 'p3.mp4'), + ).toThrow(OmniTransportGuardError); + expect(() => + assertWithinTokenLimit(cfg({ maxTokens: 196_608 }), VIDEO_8MIN, 'p3.mp4'), + ).toThrow( + /p3\.mp4.*raw-resource-v1.*196608.*omni\.transport\.maxEstimatedTokens/, + ); + }); + + it('enabled: passes small inputs (5s audio ≈ 35 tokens)', () => { + const est = assertWithinTokenLimit( + cfg({ maxTokens: 196_608 }), + AUDIO_5S, + 'tone.mp3', + ); + expect(est.estimatedTokenCount).toBe(35); + }); + + it('unavailable estimates never reject', () => { + const noMeta: RecognizedMedia = { ...AUDIO_5S, metadata: {} }; + const est = assertWithinTokenLimit( + cfg({ maxTokens: 10 }), + noMeta, + 'tone.mp3', + ); + expect(est.status).toBe('unavailable'); + }); +}); diff --git a/packages/core/src/omni/guard.ts b/packages/core/src/omni/guard.ts new file mode 100644 index 00000000000..d1fb9ee2069 --- /dev/null +++ b/packages/core/src/omni/guard.ts @@ -0,0 +1,85 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import type { Config } from '../config/config.js'; +import { + estimateRawResourceTokens, + type OmniTokenEstimate, +} from './estimation.js'; +import type { RecognizedMedia } from './recognition.js'; + +/** Default per-file upload ceiling: 1 GiB (DashScope instant-upload cap). */ +export const DEFAULT_OMNI_MAX_UPLOAD_FILE_BYTES = 1024 * 1024 * 1024; + +/** Thrown when a transport guard rejects an input. Fail closed: callers + * surface the message; there is no silent degradation. Messages must stay + * free of absolute paths (they can reach model-visible content). */ +export class OmniTransportGuardError extends Error { + constructor(message: string) { + super(message); + this.name = 'OmniTransportGuardError'; + } +} + +/** Resolve the effective byte ceiling (undefined/<=0 config → default). */ +export function effectiveMaxUploadFileBytes(config: Config): number { + const configured = config.getOmniUploadMaxFileBytes?.(); + return configured !== undefined && configured > 0 + ? configured + : DEFAULT_OMNI_MAX_UPLOAD_FILE_BYTES; +} + +/** + * Byte-dimension guard. Runs on a cheap stat BEFORE any hashing/probing. + */ +export function assertWithinByteLimit( + config: Config, + sizeBytes: number, + displayName: string, +): void { + const maxBytes = effectiveMaxUploadFileBytes(config); + if (sizeBytes > maxBytes) { + throw new OmniTransportGuardError( + `${displayName} exceeds the omni upload limit: ${sizeBytes} bytes > ` + + `${maxBytes} bytes (omni.upload.maxFileBytes). ` + + `Reduce the file size before retrying.`, + ); + } +} + +/** + * Token-dimension guard. Runs AFTER recognition (needs probe metadata) and + * BEFORE store/upload, so an oversized input costs one probe — not a copy + * and a multi-minute upload. + * + * Threshold semantics (`omni.transport.maxEstimatedTokens`): + * - unset / 0 / negative → guard disabled (the estimation formula is still + * pending confirmation with the model provider; estimates are attached + * for observability but must not reject until a threshold is set); + * - positive → hard fail-closed limit against the versioned estimator. + * + * An `unavailable` estimate never rejects — per design, missing metadata + * must not be guessed, and the transport guard only acts on real numbers. + */ +export function assertWithinTokenLimit( + config: Config, + media: RecognizedMedia, + displayName: string, +): OmniTokenEstimate { + const estimate = estimateRawResourceTokens(media); + const maxTokens = config.getOmniMaxEstimatedTokens?.(); + if (maxTokens === undefined || maxTokens <= 0) return estimate; + if (estimate.status !== 'ok') return estimate; + if (estimate.estimatedTokenCount > maxTokens) { + throw new OmniTransportGuardError( + `${displayName} exceeds the omni estimated-token limit: ` + + `~${estimate.estimatedTokenCount} tokens (${estimate.method}) > ` + + `${maxTokens} (omni.transport.maxEstimatedTokens). ` + + `Reduce duration/resolution or raise the limit.`, + ); + } + return estimate; +} diff --git a/packages/core/src/omni/index.ts b/packages/core/src/omni/index.ts index c6f0b0bee7d..f6771a3975e 100644 --- a/packages/core/src/omni/index.ts +++ b/packages/core/src/omni/index.ts @@ -13,10 +13,17 @@ import { ToolErrorType } from '../tools/tool-error.js'; import { createDebugLogger } from '../utils/debugLogger.js'; import { isAbortError } from '../utils/errors.js'; import { isFfmpegAvailable, isFfprobeAvailable } from './ffmpeg.js'; +import type { OmniTokenEstimate } from './estimation.js'; import { - extensionForVideoMime, - recognizeVideoFile, - type RecognizedVideo, + assertWithinByteLimit, + assertWithinTokenLimit, + effectiveMaxUploadFileBytes, +} from './guard.js'; +import { + extensionForMime, + recognizeMediaFile, + type OmniModality, + type RecognizedMedia, } from './recognition.js'; import { OmniObjectStore } from './storage.js'; import { DashScopeUploader } from './upload.js'; @@ -30,16 +37,31 @@ export { export { OmniObjectStore } from './storage.js'; export { DashScopeUploader, OSS_URL_PREFIX } from './upload.js'; export { + recognizeMediaFile, recognizeVideoFile, + sniffMediaType, sniffVideoMimeType, hashFileSha256, + type OmniModality, + type RecognizedMedia, } from './recognition.js'; +export { + estimateRawResourceTokens, + type OmniTokenEstimate, +} from './estimation.js'; +export { + OmniTransportGuardError, + DEFAULT_OMNI_MAX_UPLOAD_FILE_BYTES, +} from './guard.js'; +export { + downloadMediaUrl, + parseHttpUrlRef, + OmniDownloadError, + type DownloadedMedia, +} from './download.js'; const debugLogger = createDebugLogger('omni'); -/** Default per-file upload ceiling: 1 GiB (DashScope instant-upload cap). */ -export const DEFAULT_OMNI_MAX_UPLOAD_FILE_BYTES = 1024 * 1024 * 1024; - /** * Placeholder the model-config resolver assigns under Qwen OAuth; the real * token is swapped in per-request by QwenContentGenerator and never lands @@ -48,8 +70,8 @@ export const DEFAULT_OMNI_MAX_UPLOAD_FILE_BYTES = 1024 * 1024 * 1024; */ const QWEN_OAUTH_PLACEHOLDER_API_KEY = 'QWEN_OAUTH_DYNAMIC_TOKEN'; -/** Result of the S1 video delivery pipeline. */ -export interface OmniVideoDelivery { +/** Result of the omni media delivery pipeline. */ +export interface OmniMediaDelivery { /** `oss://…` URL to place in fileData.fileUri. */ fileUri: string; /** Authoritative (sniffed) MIME type for the Part. */ @@ -57,11 +79,16 @@ export interface OmniVideoDelivery { /** Content hash — identity of the stored object. */ sha256: string; /** Recognition output, for logs/display. */ - recognized: RecognizedVideo; + recognized: RecognizedMedia; + /** Raw-resource token estimate (attached even when the guard is off). */ + tokenEstimate: OmniTokenEstimate; /** Whether the object store already held this content. */ deduped: boolean; } +/** @deprecated S1 name kept for existing imports. */ +export type OmniVideoDelivery = OmniMediaDelivery; + /** Thrown for omni pipeline failures. The pipeline fails closed: callers * surface the message instead of silently falling back to inline base64. * Messages must not contain absolute paths or raw upstream response @@ -74,7 +101,7 @@ export class OmniDeliveryError extends Error { } /** - * Gate for the S1 video delivery path. All conditions must hold: + * Gate for the omni delivery path. All conditions must hold: * * 1. omni enabled (settings or QWEN_CODE_ENABLE_OMNI=1); * 2. trusted workspace (the pipeline writes .qwen/omni/ and uploads @@ -87,10 +114,10 @@ export class OmniDeliveryError extends Error { * 5. a DashScope-compatible provider. * * Any failed condition falls back to the pre-omni inline behavior. - * Video modality is checked by the caller (fileUtils) alongside the + * Modality support is checked by the caller (fileUtils) alongside the * existing modality gate. */ -export function isOmniVideoDeliveryActive(config: Config): boolean { +export function isOmniDeliveryActive(config: Config): boolean { // Optional calls so stub Configs in tests (and embedders constructing // partial configs) don't need the omni accessors to process files. if (!config.isOmniEnabled?.()) return false; @@ -106,7 +133,7 @@ export function isOmniVideoDeliveryActive(config: Config): boolean { cgc.apiKey === QWEN_OAUTH_PLACEHOLDER_API_KEY ) { debugLogger.debug( - 'omni delivery inactive: no static API key usable for the uploads endpoint (Qwen OAuth is not supported in S1)', + 'omni delivery inactive: no static API key usable for the uploads endpoint (Qwen OAuth is not supported)', ); return false; } @@ -119,19 +146,26 @@ export function isOmniVideoDeliveryActive(config: Config): boolean { return DashScopeOpenAICompatibleProvider.isDashScopeProvider(cgc); } +/** @deprecated S1 name kept for existing imports. */ +export const isOmniVideoDeliveryActive = isOmniDeliveryActive; + /** - * S1 pipeline: recognize → promote into the content-addressed store → - * upload via the DashScope temporary channel → return the oss:// URL. + * Omni pipeline: recognize → transport guard → promote into the + * content-addressed store → upload via the DashScope temporary channel → + * return the oss:// URL plus the token estimate. * - * No caching in S1: every invocation re-uploads (S3 adds the - * (sha256, model) upload cache). Throws OmniDeliveryError on any failure; - * user aborts propagate as the original abort error. + * All modalities are uploaded AS-IS — no resizing, no transcoding. + * Degradation is the job of S4 policies (which must disclose); the default + * path never silently alters content. No caching yet (S3 adds the + * (sha256, model) upload cache). Throws OmniDeliveryError / + * OmniTransportGuardError on failure; user aborts propagate untouched. */ -export async function processVideoForOmniDelivery( +export async function processMediaForOmniDelivery( filePath: string, config: Config, - signal?: AbortSignal, -): Promise { + options?: { expectedModality?: OmniModality; signal?: AbortSignal }, +): Promise { + const { expectedModality, signal } = options ?? {}; const displayName = path.basename(filePath); // Defense in depth: startup validation already asserted this, but the @@ -142,48 +176,43 @@ export async function processVideoForOmniDelivery( ]); if (!ffmpeg || !ffprobe) { throw new OmniDeliveryError( - 'ffmpeg/ffprobe not available; omni video delivery requires both on PATH.', + 'ffmpeg/ffprobe not available; omni media delivery requires both on PATH.', ); } - // Enforce the byte ceiling from a cheap stat BEFORE hashing/probing — - // a 60GB capture must not stream through SHA-256 only to be rejected. - const configuredMax = config.getOmniUploadMaxFileBytes?.(); - const maxBytes = - configuredMax !== undefined && configuredMax > 0 - ? configuredMax - : DEFAULT_OMNI_MAX_UPLOAD_FILE_BYTES; + // Byte guard from a cheap stat BEFORE hashing/probing — a 60GB capture + // must not stream through SHA-256 only to be rejected. const stat = await fs.stat(filePath).catch((err) => { throw new OmniDeliveryError( - `Cannot stat video file ${displayName}: ${ + `Cannot stat media file ${displayName}: ${ err instanceof Error ? err.message : String(err) }`, { cause: err }, ); }); - if (stat.size > maxBytes) { - throw new OmniDeliveryError( - `Video exceeds the omni upload limit: ${stat.size} bytes > ` + - `${maxBytes} bytes (omni.upload.maxFileBytes). ` + - `Reduce the file size before retrying.`, - ); - } + assertWithinByteLimit(config, stat.size, displayName); - let recognized: RecognizedVideo; + let recognized: RecognizedMedia; try { - recognized = await recognizeVideoFile(filePath, signal); + recognized = await recognizeMediaFile(filePath, { + expectedModality, + signal, + }); } catch (err) { if (signal?.aborted) throw err; throw new OmniDeliveryError( - `Video recognition failed for ${displayName}: ${ + `Media recognition failed for ${displayName}: ${ err instanceof Error ? err.message : String(err) }`, { cause: err }, ); } + // Token guard AFTER probe (needs metadata), BEFORE copy/upload. + const tokenEstimate = assertWithinTokenLimit(config, recognized, displayName); + const store = new OmniObjectStore(config.storage.getQwenDir()); - const extension = extensionForVideoMime(recognized.detectedMimeType); + const extension = extensionForMime(recognized.detectedMimeType); let objectPath: string; let deduped: boolean; try { @@ -198,7 +227,7 @@ export async function processVideoForOmniDelivery( } catch (err) { if (signal?.aborted) throw err; throw new OmniDeliveryError( - `Failed to store video in the omni object store: ${ + `Failed to store media in the omni object store: ${ err instanceof Error ? err.message : String(err) }`, { cause: err }, @@ -227,72 +256,143 @@ export async function processVideoForOmniDelivery( } debugLogger.debug( - `omni video delivered: sha256=${recognized.sha256.slice(0, 12)}… ` + - `size=${recognized.sizeBytes} deduped=${deduped} uri=${fileUri}`, + `omni ${recognized.modality} delivered: sha256=${recognized.sha256.slice(0, 12)}… ` + + `size=${recognized.sizeBytes} est=${tokenEstimate.estimatedTokenCount}(${tokenEstimate.status}) ` + + `deduped=${deduped} uri=${fileUri}`, ); return { fileUri, mimeType: recognized.detectedMimeType, sha256: recognized.sha256, recognized, + tokenEstimate, deduped, }; } +/** @deprecated S1 wrapper: video-only pipeline. */ +export async function processVideoForOmniDelivery( + filePath: string, + config: Config, + signal?: AbortSignal, +): Promise { + return processMediaForOmniDelivery(filePath, config, { + expectedModality: 'video', + signal, + }); +} + /** Read-result shape consumed by fileUtils.processSingleFileContent for * media Parts (structurally mirrors ProcessedFileReadResult's media case * without importing fileUtils, which would create a cycle). */ -export interface OmniVideoReadResult { +export interface OmniMediaReadResult { llmContent: | string + | Array< + | { text: string } + | { + fileData: { + fileUri: string; + mimeType: string; + displayName: string; + }; + } + > | { fileData: { fileUri: string; mimeType: string; displayName: string } }; returnDisplay: string; error?: string; errorType?: ToolErrorType; + tokenEstimate?: OmniTokenEstimate; } +/** @deprecated S1 name kept for existing imports. */ +export type OmniVideoReadResult = OmniMediaReadResult; + /** - * fileUtils-facing wrapper: run the S1 delivery pipeline and shape the + * fileUtils-facing wrapper: run the delivery pipeline and shape the * outcome as a file-read result. Fails closed on delivery errors (no * inline fallback); rethrows user aborts so the caller's abort handling - * applies. + * applies. For images, a text part with dimensions and a zoom hint is + * emitted alongside the fileData part (zoom_image reads the original from + * disk and stays functional under upload delivery). */ -export async function readVideoViaOmniDelivery(params: { +export async function readMediaViaOmniDelivery(params: { filePath: string; config: Config; displayName: string; relativePathForDisplay: string; + expectedModality: OmniModality; signal?: AbortSignal; -}): Promise { - const { filePath, config, displayName, relativePathForDisplay, signal } = - params; +}): Promise { + const { + filePath, + config, + displayName, + relativePathForDisplay, + expectedModality, + signal, + } = params; try { - const delivery = await processVideoForOmniDelivery( - filePath, - config, + const delivery = await processMediaForOmniDelivery(filePath, config, { + expectedModality, signal, - ); - return { - llmContent: { - fileData: { - fileUri: delivery.fileUri, - mimeType: delivery.mimeType, - displayName, - }, + }); + const fileDataPart = { + fileData: { + fileUri: delivery.fileUri, + mimeType: delivery.mimeType, + displayName, }, - returnDisplay: `Read video file (omni upload): ${relativePathForDisplay}`, + }; + const { width, height } = delivery.recognized.metadata; + const llmContent = + delivery.recognized.modality === 'image' && + width !== undefined && + height !== undefined + ? [ + { + text: + `Image ${displayName}: full resolution ${width}x${height} px. ` + + `Use zoom_image for a closer look at details.`, + }, + fileDataPart, + ] + : fileDataPart; + return { + llmContent, + returnDisplay: `Read ${delivery.recognized.modality} file (omni upload): ${relativePathForDisplay}`, + tokenEstimate: delivery.tokenEstimate, }; } catch (err) { if (isAbortError(err) || signal?.aborted) throw err; // Fail closed: no silent fallback to inline base64 — a fallback would // resurrect the 10MB cap surprise and mislead the user into thinking - // the model saw the original video. + // the model saw the original media. const message = err instanceof Error ? err.message : String(err); return { - llmContent: `[Omni video delivery failed for ${displayName}: ${message}]`, - returnDisplay: `Failed to deliver video via omni upload: ${relativePathForDisplay}`, - error: `Omni video delivery failed: ${filePath}: ${message}`, + llmContent: `[Omni media delivery failed for ${displayName}: ${message}]`, + returnDisplay: `Failed to deliver media via omni upload: ${relativePathForDisplay}`, + error: `Omni media delivery failed: ${filePath}: ${message}`, errorType: ToolErrorType.READ_CONTENT_FAILURE, }; } } + +/** @deprecated S1 wrapper. */ +export async function readVideoViaOmniDelivery(params: { + filePath: string; + config: Config; + displayName: string; + relativePathForDisplay: string; + signal?: AbortSignal; +}): Promise { + return readMediaViaOmniDelivery({ ...params, expectedModality: 'video' }); +} + +/** Effective download byte ceiling — aligned with the upload channel cap + * (downloading more than can be delivered is pointless). */ +export function effectiveMaxDownloadFileBytes(config: Config): number { + const configured = config.getOmniDownloadMaxFileBytes?.(); + if (configured !== undefined && configured > 0) return configured; + return effectiveMaxUploadFileBytes(config); +} diff --git a/packages/core/src/omni/recognition.test.ts b/packages/core/src/omni/recognition.test.ts index d96e045ca67..255d999fa48 100644 --- a/packages/core/src/omni/recognition.test.ts +++ b/packages/core/src/omni/recognition.test.ts @@ -91,3 +91,74 @@ describe('hashFileSha256', () => { } }); }); + +describe('sniffMediaType (S2 modalities)', async () => { + const { sniffMediaType } = await import('./recognition.js'); + + it('detects images: png/jpeg/webp/gif', () => { + expect( + sniffMediaType( + Buffer.concat([Buffer.from([0x89, 0x50, 0x4e, 0x47]), Buffer.alloc(8)]), + ), + ).toMatchObject({ mimeType: 'image/png', modality: 'image' }); + expect(sniffMediaType(Buffer.from([0xff, 0xd8, 0xff, 0xe0]))).toMatchObject( + { mimeType: 'image/jpeg', modality: 'image' }, + ); + expect( + sniffMediaType( + Buffer.concat([ + Buffer.from('RIFF', 'latin1'), + Buffer.alloc(4), + Buffer.from('WEBP', 'latin1'), + ]), + ), + ).toMatchObject({ mimeType: 'image/webp', modality: 'image' }); + expect(sniffMediaType(Buffer.from('GIF89a....'))).toMatchObject({ + mimeType: 'image/gif', + modality: 'image', + }); + }); + + it('detects audio: mp3(id3/framesync)/wav/flac/ogg/m4a', () => { + expect(sniffMediaType(Buffer.from('ID3\x04\x00'))).toMatchObject({ + mimeType: 'audio/mpeg', + modality: 'audio', + }); + expect(sniffMediaType(Buffer.from([0xff, 0xfb, 0x90, 0x00]))).toMatchObject( + { mimeType: 'audio/mpeg', modality: 'audio' }, + ); + expect( + sniffMediaType( + Buffer.concat([ + Buffer.from('RIFF', 'latin1'), + Buffer.alloc(4), + Buffer.from('WAVE', 'latin1'), + ]), + ), + ).toMatchObject({ mimeType: 'audio/wav', modality: 'audio' }); + expect(sniffMediaType(Buffer.from('fLaC....'))).toMatchObject({ + mimeType: 'audio/flac', + modality: 'audio', + }); + expect(sniffMediaType(Buffer.from('OggS....'))).toMatchObject({ + mimeType: 'audio/ogg', + modality: 'audio', + }); + const m4a = Buffer.concat([ + Buffer.from([0, 0, 0, 0x18]), + Buffer.from('ftypM4A ', 'latin1'), + Buffer.alloc(4), + ]); + expect(sniffMediaType(m4a)).toMatchObject({ + mimeType: 'audio/mp4', + modality: 'audio', + }); + }); + + it('rejects non-media content', () => { + expect(sniffMediaType(Buffer.from('#!/bin/sh\necho hi'))).toBeNull(); + expect(sniffMediaType(Buffer.from(''))).toBeNull(); + expect(sniffMediaType(Buffer.from('%PDF-1.7'))).toBeNull(); + expect(sniffMediaType(Buffer.alloc(0))).toBeNull(); + }); +}); diff --git a/packages/core/src/omni/recognition.ts b/packages/core/src/omni/recognition.ts index 188a234b68a..16738f6c3cb 100644 --- a/packages/core/src/omni/recognition.ts +++ b/packages/core/src/omni/recognition.ts @@ -9,73 +9,153 @@ import { createReadStream } from 'node:fs'; import fs from 'node:fs/promises'; import path from 'node:path'; import { pipeline } from 'node:stream/promises'; -import { probeVideoMetadata, type VideoProbeResult } from './ffmpeg.js'; +import { probeMediaMetadata, type MediaProbeResult } from './ffmpeg.js'; -/** Result of minimal S1 recognition for a local video file. */ -export interface RecognizedVideo { +/** Media modality handled by the omni pipeline. */ +export type OmniModality = 'image' | 'audio' | 'video'; + +/** Result of content recognition for a local media file. */ +export interface RecognizedMedia { + /** Modality derived from the sniffed container type. */ + modality: OmniModality; /** Hex-encoded SHA-256 of the full file content. */ sha256: string; /** Authoritative MIME type detected from content (sniff), not extension. */ detectedMimeType: string; /** File size in bytes. */ sizeBytes: number; - /** Basic ffprobe metadata. */ - metadata: VideoProbeResult; + /** ffprobe/header metadata (fields populated per modality). */ + metadata: MediaProbeResult; +} + +/** @deprecated S1 name — video-only alias kept for existing imports. */ +export type RecognizedVideo = RecognizedMedia; + +interface SniffedType { + mimeType: string; + modality: OmniModality; + extension: string; +} + +function bmffBrandType(brand: string): SniffedType { + if (brand.startsWith('qt')) { + return { + mimeType: 'video/quicktime', + modality: 'video', + extension: '.mov', + }; + } + // M4A/M4B audio brands inside ISO BMFF. + if (brand.startsWith('M4A') || brand.startsWith('M4B')) { + return { mimeType: 'audio/mp4', modality: 'audio', extension: '.m4a' }; + } + return { mimeType: 'video/mp4', modality: 'video', extension: '.mp4' }; } /** - * Content-sniffed video container detection (magic bytes). Returns the - * detected MIME type, or null when the header does not look like a video - * container we recognize. Covers the common containers for S1: MP4/MOV - * (ISO BMFF "ftyp"), WebM/Matroska (EBML), and AVI (RIFF). + * Content-sniffed media type detection (magic bytes). Returns null when the + * header does not match a supported container. Covers: + * video — MP4/MOV (ISO BMFF), WebM/Matroska (EBML), AVI (RIFF); + * image — JPEG, PNG, WebP (RIFF), GIF; + * audio — MP3 (ID3 or frame sync), WAV (RIFF), FLAC, OGG, M4A (BMFF brand). */ -export function sniffVideoMimeType(header: Buffer): string | null { +export function sniffMediaType(header: Buffer): SniffedType | null { if (header.length >= 12) { - // ISO BMFF: bytes 4..8 are "ftyp"; the major brand distinguishes - // QuickTime ("qt ") from the MP4 family. if (header.subarray(4, 8).toString('latin1') === 'ftyp') { - const brand = header.subarray(8, 12).toString('latin1'); - return brand.startsWith('qt') ? 'video/quicktime' : 'video/mp4'; + return bmffBrandType(header.subarray(8, 12).toString('latin1')); + } + const riff = header.subarray(0, 4).toString('latin1') === 'RIFF'; + if (riff) { + const kind = header.subarray(8, 12).toString('latin1'); + if (kind === 'AVI ') { + return { + mimeType: 'video/x-msvideo', + modality: 'video', + extension: '.avi', + }; + } + if (kind === 'WAVE') { + return { mimeType: 'audio/wav', modality: 'audio', extension: '.wav' }; + } + if (kind === 'WEBP') { + return { + mimeType: 'image/webp', + modality: 'image', + extension: '.webp', + }; + } + return null; + } + } + if (header.length >= 8) { + // PNG + if (header.readUInt32BE(0) === 0x89504e47) { + return { mimeType: 'image/png', modality: 'image', extension: '.png' }; } } if (header.length >= 4) { - // EBML magic for WebM/Matroska. Distinguishing the two requires parsing - // the DocType element; default to video/webm which DashScope accepts - // for both in practice, and matroska readers tolerate. + // EBML (WebM/Matroska) if (header.readUInt32BE(0) === 0x1a45dfa3) { - return 'video/webm'; + return { mimeType: 'video/webm', modality: 'video', extension: '.webm' }; + } + // FLAC + if (header.subarray(0, 4).toString('latin1') === 'fLaC') { + return { mimeType: 'audio/flac', modality: 'audio', extension: '.flac' }; + } + // OGG + if (header.subarray(0, 4).toString('latin1') === 'OggS') { + return { mimeType: 'audio/ogg', modality: 'audio', extension: '.ogg' }; + } + // GIF87a / GIF89a + if (header.subarray(0, 3).toString('latin1') === 'GIF') { + return { mimeType: 'image/gif', modality: 'image', extension: '.gif' }; } } - if (header.length >= 12) { - // RIFF....AVI LIST - if ( - header.subarray(0, 4).toString('latin1') === 'RIFF' && - header.subarray(8, 12).toString('latin1') === 'AVI ' - ) { - return 'video/x-msvideo'; + if (header.length >= 3) { + // JPEG + if (header[0] === 0xff && header[1] === 0xd8 && header[2] === 0xff) { + return { mimeType: 'image/jpeg', modality: 'image', extension: '.jpg' }; + } + // MP3: ID3 tag or bare MPEG frame sync (0xFFEx/0xFFFx). + if (header.subarray(0, 3).toString('latin1') === 'ID3') { + return { mimeType: 'audio/mpeg', modality: 'audio', extension: '.mp3' }; + } + if (header[0] === 0xff && (header[1]! & 0xe0) === 0xe0) { + return { mimeType: 'audio/mpeg', modality: 'audio', extension: '.mp3' }; } } return null; } -/** Map a detected video MIME type to the canonical file extension used for - * content-addressed object names. Covers exactly the MIME types - * `sniffVideoMimeType` can emit. */ -export function extensionForVideoMime(mimeType: string): string { - switch (mimeType) { - case 'video/mp4': - return '.mp4'; - case 'video/quicktime': - return '.mov'; - case 'video/webm': - return '.webm'; - case 'video/x-msvideo': - return '.avi'; - default: - return '.bin'; - } +/** S1-compat: video-only sniff. */ +export function sniffVideoMimeType(header: Buffer): string | null { + const sniffed = sniffMediaType(header); + return sniffed?.modality === 'video' ? sniffed.mimeType : null; +} + +/** Map a detected MIME type to the canonical object-store extension. */ +export function extensionForMime(mimeType: string): string { + const sniffTable: Record = { + 'video/mp4': '.mp4', + 'video/quicktime': '.mov', + 'video/webm': '.webm', + 'video/x-msvideo': '.avi', + 'image/jpeg': '.jpg', + 'image/png': '.png', + 'image/webp': '.webp', + 'image/gif': '.gif', + 'audio/mpeg': '.mp3', + 'audio/wav': '.wav', + 'audio/flac': '.flac', + 'audio/ogg': '.ogg', + 'audio/mp4': '.m4a', + }; + return sniffTable[mimeType] ?? '.bin'; } +/** @deprecated S1 name kept for existing imports. */ +export const extensionForVideoMime = extensionForMime; + /** Stream a file through SHA-256 without loading it into memory. */ export async function hashFileSha256( filePath: string, @@ -90,16 +170,20 @@ export async function hashFileSha256( const SNIFF_BYTES = 4096; /** - * Minimal S1 recognition for a local video file: content sniff (magic - * bytes), streaming SHA-256, and ffprobe metadata. Throws when the file - * does not sniff as a video container or when ffprobe fails — the omni - * pipeline fails closed on unrecognizable input. Error messages carry the - * file's basename only (they can reach model-visible content). + * Recognize a local media file: content sniff (magic bytes), streaming + * SHA-256, and ffprobe metadata. Throws when the content does not sniff as + * a supported media container or when probing fails — the omni pipeline + * fails closed on unrecognizable input. Error messages carry the file's + * basename only (they can reach model-visible content). + * + * When `expectedModality` is given, a successful sniff of a DIFFERENT + * modality is rejected (e.g. an .mp3 that is actually an mp4 container). */ -export async function recognizeVideoFile( +export async function recognizeMediaFile( filePath: string, - signal?: AbortSignal, -): Promise { + options?: { expectedModality?: OmniModality; signal?: AbortSignal }, +): Promise { + const { expectedModality, signal } = options ?? {}; const handle = await fs.open(filePath, 'r'); let header: Buffer; let sizeBytes: number; @@ -113,17 +197,36 @@ export async function recognizeVideoFile( await handle.close(); } - const detectedMimeType = sniffVideoMimeType(header); - if (!detectedMimeType) { + const sniffed = sniffMediaType(header); + if (!sniffed) { throw new Error( - `File content does not match a supported video container (mp4/mov/webm/mkv/avi): ${path.basename(filePath)}`, + `File content does not match a supported media container: ${path.basename(filePath)}`, + ); + } + if (expectedModality && sniffed.modality !== expectedModality) { + throw new Error( + `File content sniffs as ${sniffed.modality} (${sniffed.mimeType}) but was referenced as ${expectedModality}: ${path.basename(filePath)}`, ); } const [sha256, metadata] = await Promise.all([ hashFileSha256(filePath, signal), - probeVideoMetadata(filePath, signal), + probeMediaMetadata(filePath, sniffed.modality, signal), ]); - return { sha256, detectedMimeType, sizeBytes, metadata }; + return { + modality: sniffed.modality, + sha256, + detectedMimeType: sniffed.mimeType, + sizeBytes, + metadata, + }; +} + +/** @deprecated S1 wrapper: video-only recognition. */ +export async function recognizeVideoFile( + filePath: string, + signal?: AbortSignal, +): Promise { + return recognizeMediaFile(filePath, { expectedModality: 'video', signal }); } diff --git a/packages/core/src/omni/tool-result-media.ts b/packages/core/src/omni/tool-result-media.ts new file mode 100644 index 00000000000..eab3ea37929 --- /dev/null +++ b/packages/core/src/omni/tool-result-media.ts @@ -0,0 +1,123 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { randomBytes } from 'node:crypto'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import type { Part } from '@google/genai'; +import type { Config } from '../config/config.js'; +import { createDebugLogger } from '../utils/debugLogger.js'; +import { isOmniDeliveryActive, processMediaForOmniDelivery } from './index.js'; +import { OmniObjectStore } from './storage.js'; +import { sniffMediaType } from './recognition.js'; + +const debugLogger = createDebugLogger('omni:tool-result'); + +/** + * Second normalization trigger point (design §5.2/§8.2): tool-result media + * flows through the same recognize → guard → store → upload pipeline as + * user input, converting inline base64 Parts into oss:// fileData Parts. + * + * Invoked from BOTH physical funnels — CoreToolScheduler's terminal sites + * and ACP Session.runTool — as a sibling of the vision-bridge processing + * (never mixed into it: converted fileData parts are invisible to + * isImagePart, so the bridge correctly skips them). + * + * Contract mirrors processToolResultImages: + * - returns the ORIGINAL array identity when nothing changed (callers use + * `response !== convertedResponse` to decide whether to recompute + * content-length accounting); + * - failure of any single part leaves that part inline (tool results were + * produced locally and already fit in memory — degrading to the S1-era + * inline behavior is safe here, unlike user-input delivery where inline + * silently violates the size contract; the failure is logged); + * - user aborts propagate. + */ +export async function processToolResultOmniMedia( + responseParts: Part[], + config: Config, + signal: AbortSignal, +): Promise { + if (!isOmniDeliveryActive(config)) return responseParts; + + const modalities = config.getContentGeneratorConfig?.()?.modalities ?? {}; + let changed = false; + + const convertPart = async (part: Part): Promise => { + const inline = part.inlineData; + if (!inline?.data || !inline.mimeType) return part; + const top = inline.mimeType.split('/')[0]; + if (top !== 'image' && top !== 'audio' && top !== 'video') return part; + if (!modalities[top as 'image' | 'audio' | 'video']) return part; + + // Sniff the decoded bytes before touching disk — non-media or + // unsupported containers stay inline untouched. + const bytes = Buffer.from(inline.data, 'base64'); + if (!sniffMediaType(bytes.subarray(0, 4096))) return part; + + const store = new OmniObjectStore(config.storage.getQwenDir()); + const stagingDir = path.join(store.getOmniRootDir(), 'downloads'); + await fs.mkdir(stagingDir, { recursive: true, mode: 0o700 }); + const tempPath = path.join( + stagingDir, + `${randomBytes(8).toString('hex')}.part`, + ); + try { + await fs.writeFile(tempPath, bytes, { mode: 0o600 }); + const delivery = await processMediaForOmniDelivery(tempPath, config, { + signal, + }); + changed = true; + return { + fileData: { + fileUri: delivery.fileUri, + mimeType: delivery.mimeType, + displayName: inline.displayName ?? `tool-media.${top}`, + }, + }; + } catch (err) { + if (signal.aborted) throw err; + debugLogger.debug( + `tool-result media upload failed, keeping inline: ${ + err instanceof Error ? err.message : String(err) + }`, + ); + return part; + } finally { + await fs.rm(tempPath, { force: true }).catch(() => {}); + } + }; + + const result: Part[] = []; + for (const part of responseParts) { + const nested = part.functionResponse?.parts; + if (Array.isArray(nested) && nested.length > 0) { + const convertedNested: Part[] = []; + let nestedChanged = false; + for (const nestedPart of nested as Part[]) { + const converted = await convertPart(nestedPart); + if (converted !== nestedPart) nestedChanged = true; + convertedNested.push(converted); + } + if (nestedChanged) { + result.push({ + ...part, + functionResponse: { + ...part.functionResponse, + parts: convertedNested, + }, + } as Part); + changed = true; + } else { + result.push(part); + } + continue; + } + result.push(await convertPart(part)); + } + + return changed ? result : responseParts; +} diff --git a/packages/core/src/utils/fileUtils.ts b/packages/core/src/utils/fileUtils.ts index 41c5727269a..3b3e93a5c83 100644 --- a/packages/core/src/utils/fileUtils.ts +++ b/packages/core/src/utils/fileUtils.ts @@ -925,6 +925,13 @@ export interface ProcessedFileReadResult { pdfVisionBridgeCandidate?: PDFVisionBridgeCandidate; /** User-only disclosure attached after a prepared PDF candidate runs. */ pdfVisionBridgeNotice?: string; + /** + * Raw-resource token estimate for omni-delivered media (the single + * storage location for the estimate — design doc §6.4). Absent for + * non-omni reads; `status: 'unavailable'` when required metadata was + * missing. + */ + tokenEstimate?: import('../omni/estimation.js').OmniTokenEstimate; } export interface PDFVisionBridgeFallback { @@ -1253,15 +1260,24 @@ export async function processSingleFileContent( errorType: ToolErrorType.FILE_TOO_LARGE, }; } - // Omni experiment: when active, local video files are delivered via the - // DashScope upload channel instead of inline base64, with a 1 GiB - // per-file ceiling replacing the 10MB inline cap below. Dynamic import + // Omni experiment: when active, local media files (image/audio/video) + // are delivered via the DashScope upload channel instead of inline + // base64, with a 1 GiB per-file ceiling replacing the 10MB inline cap + // below. Content is uploaded AS-IS (no resize/transcode — degradation + // is the job of omni policies, which must disclose). Dynamic import // keeps the omni module (which reaches into the provider layer) out of - // fileUtils' static dependency graph. + // fileUtils' static dependency graph. Gated per-modality on the same + // `modalities` config the converter uses, so the omni path never + // swallows a file the vision bridge would otherwise transcribe + // (bridge only activates when modalities.image is false). let omniModule: typeof import('../omni/index.js') | undefined; - if (fileType === 'video') { + if ( + (fileType === 'video' && modalities.video) || + (fileType === 'audio' && modalities.audio) || + (fileType === 'image' && modalities.image) + ) { const omni = await import('../omni/index.js'); - if (omni.isOmniVideoDeliveryActive(config)) { + if (omni.isOmniDeliveryActive(config)) { omniModule = omni; } } @@ -1460,6 +1476,19 @@ export async function processSingleFileContent( }; } case 'image': { + if (omniModule) { + // Omni: upload the ORIGINAL bytes — no local resize (that is a + // lossy transform reserved for omni policies with disclosure). + // renderImageOverview is skipped entirely on this path. + return await omniModule.readMediaViaOmniDelivery({ + filePath, + config, + displayName, + relativePathForDisplay, + expectedModality: 'image', + signal, + }); + } if (shouldRenderImageOverview) { try { const view = await renderImageOverview( @@ -1534,15 +1563,16 @@ export async function processSingleFileContent( } case 'audio': case 'video': { - if (fileType === 'video' && omniModule) { + if (omniModule) { // Delegated to the omni module: fail-closed delivery via the // DashScope upload channel; user aborts are rethrown and land in // this function's outer abort handling. - return await omniModule.readVideoViaOmniDelivery({ + return await omniModule.readMediaViaOmniDelivery({ filePath, config, displayName, relativePathForDisplay, + expectedModality: fileType, signal, }); } diff --git a/packages/vscode-ide-companion/schemas/settings.schema.json b/packages/vscode-ide-companion/schemas/settings.schema.json index d25389ca7a9..170c12b6ceb 100644 --- a/packages/vscode-ide-companion/schemas/settings.schema.json +++ b/packages/vscode-ide-companion/schemas/settings.schema.json @@ -3279,6 +3279,30 @@ "description": "Per-file byte ceiling for omni media uploads. Defaults to 1 GiB, the DashScope temporary-upload per-file cap. Inputs above the limit fail closed with an explanatory error." } } + }, + "transport": { + "description": "Transport guard dimensions beyond the byte ceiling for omni media delivery.", + "type": "object", + "properties": { + "maxEstimatedTokens": { + "type": "number", + "minimum": 0, + "default": 0, + "description": "Estimated-token ceiling for a single omni media input, checked before upload using the versioned raw-resource estimator. 0 disables the token guard (estimates are still attached for observability) — the estimation formula is pending confirmation with the model provider; set a positive threshold to enforce fail-closed rejection." + } + } + }, + "download": { + "description": "URL media localization limits for omni delivery.", + "type": "object", + "properties": { + "maxFileBytes": { + "type": "number", + "minimum": 0, + "default": 0, + "description": "Byte ceiling for downloading URL media inputs. 0 or unset follows omni.upload.maxFileBytes (downloading more than the upload channel can deliver is pointless)." + } + } } } }, From 14409cb5b609581c66450f4f02eb699d84c1c74a Mon Sep 17 00:00:00 2001 From: LaZzyMan Date: Mon, 3 Aug 2026 19:14:29 +0800 Subject: [PATCH 02/14] fix(cli): deliver URL-only @ media instead of dropping it after upload MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The no-file-paths early return in resolveAtCommandQuery did not count urlMediaRefs, so a prompt whose only @-reference was a URL ran the full download → recognize → store → upload pipeline and then discarded both the delivered fileData parts and their display cards. Found by S2 E2E verification (the model improvised around the missing image via its own tools, masking the drop). Part of the omni-experiment S2 slice (#8184). --- packages/cli/src/ui/hooks/atCommandProcessor.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/cli/src/ui/hooks/atCommandProcessor.ts b/packages/cli/src/ui/hooks/atCommandProcessor.ts index 8cad91ee800..090f2ef8719 100644 --- a/packages/cli/src/ui/hooks/atCommandProcessor.ts +++ b/packages/cli/src/ui/hooks/atCommandProcessor.ts @@ -668,7 +668,11 @@ export async function resolveAtCommandQuery({ mcpResourceRefs.length === 0 && mcpServerMentions.length === 0 && extensionMentions.length === 0 && - sessionMentions.length === 0 + sessionMentions.length === 0 && + // URL media refs count as content too: without this, a URL-only prompt + // would run the whole download→upload pipeline and then discard the + // delivered parts (and their display cards) via this early return. + urlMediaRefs.length === 0 ) { onDebugMessage('No valid file paths found in @ commands to read.'); if (initialQueryText === '@' && query.trim() === '@') { From 303c773981752707339489077ebe680a28f51343 Mon Sep 17 00:00:00 2001 From: LaZzyMan Date: Tue, 4 Aug 2026 13:37:01 +0800 Subject: [PATCH 03/14] fix(omni): apply S2 adversarial review findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Security lens: - tool-result media funnel: per-result upload budget (8 parts / 128 MiB aggregate — a malicious tool result must not fan out unbounded uploads from the user's account); excess parts stay inline - modality gate on tool-result parts now keyed to the SNIFFED modality, not the declared MIME type (a part declared audio whose bytes are a video container no longer bypasses a video-disabled config) - fs error messages are path-scrubbed before reaching model-visible error text (basenames survive, absolute paths do not) Correctness lens: - non-sniffable media formats (bmp/tiff/heic images, exotic containers) fall back to the baseline inline path via a cheap content pre-sniff instead of failing closed — fail-closed remains for genuine pipeline failures, not for formats the recognizer simply does not support - user cancellation during URL media download ends the turn gracefully instead of rejecting through resolveAtCommandQuery (which has no throw contract) into an unhandled-rejection banner Part of the omni-experiment S2 slice (#8184). --- .../cli/src/ui/hooks/atCommandProcessor.ts | 27 +++- packages/core/src/omni/index.ts | 24 +-- packages/core/src/omni/recognition.ts | 28 ++++ .../core/src/omni/tool-result-media.test.ts | 137 ++++++++++++++++++ packages/core/src/omni/tool-result-media.ts | 30 +++- packages/core/src/utils/fileUtils.ts | 12 +- 6 files changed, 238 insertions(+), 20 deletions(-) create mode 100644 packages/core/src/omni/tool-result-media.test.ts diff --git a/packages/cli/src/ui/hooks/atCommandProcessor.ts b/packages/cli/src/ui/hooks/atCommandProcessor.ts index 090f2ef8719..97b1b30cbee 100644 --- a/packages/cli/src/ui/hooks/atCommandProcessor.ts +++ b/packages/cli/src/ui/hooks/atCommandProcessor.ts @@ -538,7 +538,6 @@ export async function resolveAtCommandQuery({ const urlMediaLabels: string[] = []; if (urlMediaRefs.length > 0) { const core = await import('@qwen-code/qwen-code-core'); - const os = await import('node:os'); for (let i = 0; i < urlMediaRefs.length; i++) { const ref = urlMediaRefs[i]; const callId = `client-url-media-${userMessageTimestamp}-${i}`; @@ -549,7 +548,7 @@ export async function resolveAtCommandQuery({ return 'invalid-url'; } })(); - let tempDir: string | undefined; + let tempPartPath: string | undefined; try { const store = new core.OmniObjectStore(config.storage.getQwenDir()); const downloadsDir = path.join(store.getOmniRootDir(), 'downloads'); @@ -559,7 +558,7 @@ export async function resolveAtCommandQuery({ maxBytes: core.effectiveMaxDownloadFileBytes(config), signal, }); - tempDir = downloaded.partPath; + tempPartPath = downloaded.partPath; // Full local-file pipeline on the downloaded bytes: sniff/probe/ // hash again (the design requires re-recognition from the local // file — never trust transfer-time observations), guard, store, @@ -588,7 +587,22 @@ export async function resolveAtCommandQuery({ confirmationDetails: undefined, }); } catch (error) { - if (signal.aborted) throw error; + if (signal.aborted) { + // User cancelled mid-download: end the turn quietly instead of + // throwing — resolveAtCommandQuery has no throw contract, and a + // rejection here would surface as an unhandled-rejection banner + // (cancelOngoingRequest already handles the UI reset). + if (tempPartPath) { + await fs.rm(tempPartPath, { force: true }).catch(() => {}); + } + return { + processedQuery: null, + shouldProceed: false, + toolDisplays: [...urlMediaDisplays], + filesRead: urlMediaLabels, + // No chat-recording entry for a user-cancelled resolution. + }; + } const reason = getErrorMessage(error); onDebugMessage(`Failed to localize media URL ${ref.url}: ${reason}`); urlMediaDisplays.push({ @@ -600,12 +614,11 @@ export async function resolveAtCommandQuery({ confirmationDetails: undefined, }); } finally { - if (tempDir) { - await fs.rm(tempDir, { force: true }).catch(() => {}); + if (tempPartPath) { + await fs.rm(tempPartPath, { force: true }).catch(() => {}); } } } - void os; // reserved for future temp-dir strategies } const resourceParts: Part[] = []; diff --git a/packages/core/src/omni/index.ts b/packages/core/src/omni/index.ts index f6771a3975e..da283955a3b 100644 --- a/packages/core/src/omni/index.ts +++ b/packages/core/src/omni/index.ts @@ -40,6 +40,7 @@ export { recognizeMediaFile, recognizeVideoFile, sniffMediaType, + sniffFileModality, sniffVideoMimeType, hashFileSha256, type OmniModality, @@ -62,6 +63,17 @@ export { const debugLogger = createDebugLogger('omni'); +/** + * Scrub absolute POSIX path segments out of an error message, keeping the + * basename — fs errors embed full paths (`ENOENT: … stat '/Users/x/…'`) + * and several wraps below flow into model-visible llmContent, which must + * never carry real paths. + */ +function sanitizeErrorMessage(err: unknown): string { + const msg = err instanceof Error ? err.message : String(err); + return msg.replace(/(?:\/[\w~. -]+)+\/([\w. -]+)/g, '$1'); +} + /** * Placeholder the model-config resolver assigns under Qwen OAuth; the real * token is swapped in per-request by QwenContentGenerator and never lands @@ -184,9 +196,7 @@ export async function processMediaForOmniDelivery( // must not stream through SHA-256 only to be rejected. const stat = await fs.stat(filePath).catch((err) => { throw new OmniDeliveryError( - `Cannot stat media file ${displayName}: ${ - err instanceof Error ? err.message : String(err) - }`, + `Cannot stat media file ${displayName}: ${sanitizeErrorMessage(err)}`, { cause: err }, ); }); @@ -201,9 +211,7 @@ export async function processMediaForOmniDelivery( } catch (err) { if (signal?.aborted) throw err; throw new OmniDeliveryError( - `Media recognition failed for ${displayName}: ${ - err instanceof Error ? err.message : String(err) - }`, + `Media recognition failed for ${displayName}: ${sanitizeErrorMessage(err)}`, { cause: err }, ); } @@ -227,9 +235,7 @@ export async function processMediaForOmniDelivery( } catch (err) { if (signal?.aborted) throw err; throw new OmniDeliveryError( - `Failed to store media in the omni object store: ${ - err instanceof Error ? err.message : String(err) - }`, + `Failed to store media in the omni object store: ${sanitizeErrorMessage(err)}`, { cause: err }, ); } diff --git a/packages/core/src/omni/recognition.ts b/packages/core/src/omni/recognition.ts index 16738f6c3cb..17782ce2a51 100644 --- a/packages/core/src/omni/recognition.ts +++ b/packages/core/src/omni/recognition.ts @@ -169,6 +169,34 @@ export async function hashFileSha256( /** Number of header bytes read for content sniffing. */ const SNIFF_BYTES = 4096; +/** + * Cheap pre-gate: sniff a file's header and report the detected modality, + * or null when the content is not a supported media container. Used by + * fileUtils to decide omni-vs-legacy BEFORE committing to the fail-closed + * pipeline — a legitimate .bmp/.tiff (no sniff support) must fall back to + * the baseline inline path, not hard-fail. + */ +export async function sniffFileModality( + filePath: string, +): Promise { + let handle; + try { + handle = await fs.open(filePath, 'r'); + } catch { + return null; + } + try { + const stat = await handle.stat(); + const buf = Buffer.alloc(Math.min(SNIFF_BYTES, stat.size)); + await handle.read(buf, 0, buf.length, 0); + return sniffMediaType(buf)?.modality ?? null; + } catch { + return null; + } finally { + await handle.close(); + } +} + /** * Recognize a local media file: content sniff (magic bytes), streaming * SHA-256, and ffprobe metadata. Throws when the content does not sniff as diff --git a/packages/core/src/omni/tool-result-media.test.ts b/packages/core/src/omni/tool-result-media.test.ts new file mode 100644 index 00000000000..06ab457fed5 --- /dev/null +++ b/packages/core/src/omni/tool-result-media.test.ts @@ -0,0 +1,137 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import type { Part } from '@google/genai'; +import type { Config } from '../config/config.js'; + +const deliverMock = vi.hoisted(() => vi.fn()); +const gateMock = vi.hoisted(() => vi.fn()); +vi.mock('./index.js', () => ({ + isOmniDeliveryActive: gateMock, + processMediaForOmniDelivery: deliverMock, +})); + +import { processToolResultOmniMedia } from './tool-result-media.js'; + +// A minimal real PNG header so sniffMediaType accepts the bytes as image. +const PNG_BYTES = Buffer.concat([ + Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), + Buffer.alloc(64), +]); +// ISO BMFF video header (ftypisom) — sniffs as video/mp4. +const MP4_BYTES = Buffer.concat([ + Buffer.from([0, 0, 0, 0x18]), + Buffer.from('ftypisom', 'latin1'), + Buffer.alloc(64), +]); + +function inlinePart(mimeType: string, bytes: Buffer): Part { + return { inlineData: { mimeType, data: bytes.toString('base64') } }; +} + +function cfg(modalities: Record): Config { + return { + isOmniEnabled: () => true, + getContentGeneratorConfig: () => ({ modalities }), + storage: { getQwenDir: () => '/tmp/omni-trm-test-qwen' }, + } as unknown as Config; +} + +beforeEach(() => { + gateMock.mockReturnValue(true); + deliverMock.mockReset(); + deliverMock.mockResolvedValue({ + fileUri: 'oss://bucket/key', + mimeType: 'image/png', + sha256: 'a'.repeat(64), + recognized: { modality: 'image' }, + tokenEstimate: { + estimatedTokenCount: 1, + method: 'raw-resource-v1', + status: 'ok', + }, + deduped: false, + }); +}); + +describe('processToolResultOmniMedia', () => { + const signal = new AbortController().signal; + + it('returns the original array identity when nothing changes', async () => { + const parts: Part[] = [{ text: 'no media' }]; + const result = await processToolResultOmniMedia(parts, cfg({}), signal); + expect(result).toBe(parts); + }); + + it('converts qualifying inline media to fileData', async () => { + const parts = [inlinePart('image/png', PNG_BYTES)]; + const result = await processToolResultOmniMedia( + parts, + cfg({ image: true }), + signal, + ); + expect(result).not.toBe(parts); + expect(result[0]!.fileData?.fileUri).toBe('oss://bucket/key'); + }); + + it('gates on the SNIFFED modality, not the declared MIME type', async () => { + // Declared audio (enabled), actual bytes are an MP4 video container, + // and video modality is DISABLED — must stay inline, no upload. + const parts = [inlinePart('audio/wav', MP4_BYTES)]; + const result = await processToolResultOmniMedia( + parts, + cfg({ audio: true, video: false }), + signal, + ); + expect(result).toBe(parts); + expect(deliverMock).not.toHaveBeenCalled(); + }); + + it('enforces the per-result upload-count budget (excess stays inline)', async () => { + const parts = Array.from({ length: 12 }, () => + inlinePart('image/png', PNG_BYTES), + ); + const result = await processToolResultOmniMedia( + parts, + cfg({ image: true }), + signal, + ); + expect(deliverMock).toHaveBeenCalledTimes(8); + const uploaded = result.filter((p) => p.fileData).length; + const keptInline = result.filter((p) => p.inlineData).length; + expect(uploaded).toBe(8); + expect(keptInline).toBe(4); + }); + + it('keeps parts inline when a single delivery fails, without failing the batch', async () => { + deliverMock + .mockRejectedValueOnce(new Error('upload exploded')) + .mockResolvedValueOnce({ + fileUri: 'oss://bucket/key2', + mimeType: 'image/png', + sha256: 'b'.repeat(64), + recognized: { modality: 'image' }, + tokenEstimate: { + estimatedTokenCount: 1, + method: 'raw-resource-v1', + status: 'ok', + }, + deduped: false, + }); + const parts = [ + inlinePart('image/png', PNG_BYTES), + inlinePart('image/png', PNG_BYTES), + ]; + const result = await processToolResultOmniMedia( + parts, + cfg({ image: true }), + signal, + ); + expect(result[0]!.inlineData).toBeDefined(); + expect(result[1]!.fileData?.fileUri).toBe('oss://bucket/key2'); + }); +}); diff --git a/packages/core/src/omni/tool-result-media.ts b/packages/core/src/omni/tool-result-media.ts index eab3ea37929..3f15244131f 100644 --- a/packages/core/src/omni/tool-result-media.ts +++ b/packages/core/src/omni/tool-result-media.ts @@ -16,6 +16,11 @@ import { sniffMediaType } from './recognition.js'; const debugLogger = createDebugLogger('omni:tool-result'); +/** Upload-count budget per tool result (excess parts stay inline). */ +const MAX_UPLOADS_PER_TOOL_RESULT = 8; +/** Aggregate upload-byte budget per tool result. */ +const MAX_UPLOAD_BYTES_PER_TOOL_RESULT = 128 * 1024 * 1024; + /** * Second normalization trigger point (design §5.2/§8.2): tool-result media * flows through the same recognize → guard → store → upload pipeline as @@ -45,18 +50,34 @@ export async function processToolResultOmniMedia( const modalities = config.getContentGeneratorConfig?.()?.modalities ?? {}; let changed = false; + // Per-tool-result upload budget: a malicious/compromised tool must not + // be able to fan out an unbounded number of uploads (cost/quota burn, + // multi-minute stalls) from a single result. Parts over budget stay + // inline (safe: they were produced locally and already fit in memory). + let uploadsRemaining = MAX_UPLOADS_PER_TOOL_RESULT; + let uploadBytesRemaining = MAX_UPLOAD_BYTES_PER_TOOL_RESULT; const convertPart = async (part: Part): Promise => { const inline = part.inlineData; if (!inline?.data || !inline.mimeType) return part; const top = inline.mimeType.split('/')[0]; if (top !== 'image' && top !== 'audio' && top !== 'video') return part; - if (!modalities[top as 'image' | 'audio' | 'video']) return part; // Sniff the decoded bytes before touching disk — non-media or - // unsupported containers stay inline untouched. + // unsupported containers stay inline untouched. The SNIFFED modality + // is the authoritative gate: a part declared audio/* whose bytes are + // actually a video container must not slip past a video-disabled + // config on the strength of its declared MIME type. const bytes = Buffer.from(inline.data, 'base64'); - if (!sniffMediaType(bytes.subarray(0, 4096))) return part; + const sniffed = sniffMediaType(bytes.subarray(0, 4096)); + if (!sniffed) return part; + if (!modalities[sniffed.modality]) return part; + if (uploadsRemaining <= 0 || bytes.length > uploadBytesRemaining) { + debugLogger.debug( + `tool-result media budget exhausted; keeping part inline (${bytes.length} bytes)`, + ); + return part; + } const store = new OmniObjectStore(config.storage.getQwenDir()); const stagingDir = path.join(store.getOmniRootDir(), 'downloads'); @@ -68,9 +89,12 @@ export async function processToolResultOmniMedia( try { await fs.writeFile(tempPath, bytes, { mode: 0o600 }); const delivery = await processMediaForOmniDelivery(tempPath, config, { + expectedModality: sniffed.modality, signal, }); changed = true; + uploadsRemaining--; + uploadBytesRemaining -= bytes.length; return { fileData: { fileUri: delivery.fileUri, diff --git a/packages/core/src/utils/fileUtils.ts b/packages/core/src/utils/fileUtils.ts index 3b3e93a5c83..0685ea6ee57 100644 --- a/packages/core/src/utils/fileUtils.ts +++ b/packages/core/src/utils/fileUtils.ts @@ -1278,7 +1278,17 @@ export async function processSingleFileContent( ) { const omni = await import('../omni/index.js'); if (omni.isOmniDeliveryActive(config)) { - omniModule = omni; + // Cheap content pre-sniff decides omni-vs-legacy BEFORE committing + // to the fail-closed pipeline: formats the recognizer does not + // support (bmp/tiff/heic images, exotic containers) must keep the + // baseline inline behavior instead of hard-failing, and a file + // whose bytes sniff as a DIFFERENT modality than its extension + // suggests also falls back (the legacy path sends it inline under + // its extension-derived type, exactly as before omni). + const sniffedModality = await omni.sniffFileModality(filePath); + if (sniffedModality === fileType) { + omniModule = omni; + } } } From bfba5fe7060c32a1d8b830c80ce264e249ca4d68 Mon Sep 17 00:00:00 2001 From: LaZzyMan Date: Tue, 4 Aug 2026 15:14:13 +0800 Subject: [PATCH 04/14] fix(omni): keep the omni graph out of the ACP static bundle closure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The serve fast-path bundle-closure CI check failed: iconv-lite encoding tables (553KB) became statically reachable from the acpAgent chunk. Root cause: the CLI-side dynamic imports of the FULL core barrel (`import('@qwen-code/qwen-code-core')` in atCommandProcessor) request the whole namespace object, defeating tree-shaking of the barrel — which statically re-exports sync-file-encoding → iconvHelper → iconv-lite tables — and esbuild folds a dynamic import of an already-statically- imported module into a static chunk edge. Fixes: - add a `./omni` subpath export to @qwen-code/qwen-code-core (pattern: existing ./transcriptRecords) and point every CLI-side dynamic import at it, so only the omni module graph is requested; - re-export processToolResultOmniMedia from omni/index (circular-safe: both modules only bind functions) so the subpath serves the tool-result funnel; - coreToolScheduler and ACP Session load the funnel via dynamic import (mirroring fileUtils' existing pattern for the omni module). Verified: `npm run check:serve-fast-path-bundle` passes; bisected against base in a clean worktree to isolate the offending edge. Part of the omni-experiment S2 slice (#8184). --- .../cli/src/acp-integration/session/Session.ts | 18 ++++++++++++------ .../cli/src/ui/hooks/atCommandProcessor.ts | 4 ++-- packages/core/package.json | 4 ++++ packages/core/src/core/coreToolScheduler.ts | 8 +++++++- packages/core/src/omni/index.ts | 3 +++ 5 files changed, 28 insertions(+), 9 deletions(-) diff --git a/packages/cli/src/acp-integration/session/Session.ts b/packages/cli/src/acp-integration/session/Session.ts index 558dedbf791..a34de78dceb 100644 --- a/packages/cli/src/acp-integration/session/Session.ts +++ b/packages/cli/src/acp-integration/session/Session.ts @@ -153,7 +153,6 @@ import { normalizeParts, runVisionBridge, bridgeToolResultImages, - processToolResultOmniMedia, shouldRunVisionBridge, formatVisionBridgeNotice, formatFullTurnVisionNotice, @@ -8268,11 +8267,18 @@ export class Session implements SessionContext { // CoreToolScheduler.processToolResultImages, design §8.2): // inline tool-result media becomes oss:// fileData before the // vision bridge runs, which then skips the converted parts. - responseParts = await processToolResultOmniMedia( - responseParts, - this.config, - activeToolAbortSignal, - ); + { + // Dynamic import: keeps omni out of the ACP static closure + // (serve fast-path bundle-closure CI check). + const { processToolResultOmniMedia } = await import( + '@qwen-code/qwen-code-core/omni' + ); + responseParts = await processToolResultOmniMedia( + responseParts, + this.config, + activeToolAbortSignal, + ); + } const visionBridgeNotices: string[] = []; responseParts = await bridgeToolResultImages({ config: this.config, diff --git a/packages/cli/src/ui/hooks/atCommandProcessor.ts b/packages/cli/src/ui/hooks/atCommandProcessor.ts index 97b1b30cbee..14c66be7953 100644 --- a/packages/cli/src/ui/hooks/atCommandProcessor.ts +++ b/packages/cli/src/ui/hooks/atCommandProcessor.ts @@ -275,7 +275,7 @@ export async function resolveAtCommandQuery({ // ENOENT skip silently drops it. Only intercepted when omni delivery // is active; otherwise preserve the legacy text fall-through. if (/^https?:\/\//i.test(pathName)) { - const omni = await import('@qwen-code/qwen-code-core'); + const omni = await import('@qwen-code/qwen-code-core/omni'); if (omni.isOmniDeliveryActive(config) && omni.parseHttpUrlRef(pathName)) { if (!urlMediaRefs.some((r) => r.url === pathName)) { urlMediaRefs.push({ originalAtPath, url: pathName }); @@ -537,7 +537,7 @@ export async function resolveAtCommandQuery({ const urlMediaDisplays: IndividualToolCallDisplay[] = []; const urlMediaLabels: string[] = []; if (urlMediaRefs.length > 0) { - const core = await import('@qwen-code/qwen-code-core'); + const core = await import('@qwen-code/qwen-code-core/omni'); for (let i = 0; i < urlMediaRefs.length; i++) { const ref = urlMediaRefs[i]; const callId = `client-url-media-${userMessageTimestamp}-${i}`; diff --git a/packages/core/package.json b/packages/core/package.json index dffe771c14e..0fedf744890 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -13,6 +13,10 @@ "types": "./dist/index.d.ts", "import": "./dist/index.js" }, + "./omni": { + "types": "./dist/src/omni/index.d.ts", + "import": "./dist/src/omni/index.js" + }, "./transcriptRecords": { "types": "./dist/src/utils/transcript-records.d.ts", "import": "./dist/src/utils/transcript-records.js" diff --git a/packages/core/src/core/coreToolScheduler.ts b/packages/core/src/core/coreToolScheduler.ts index 6f0940c6bfd..c0028afd221 100644 --- a/packages/core/src/core/coreToolScheduler.ts +++ b/packages/core/src/core/coreToolScheduler.ts @@ -173,7 +173,6 @@ import { normalizeParts, } from '../services/visionBridge/image-part-utils.js'; import { bridgeToolResultImages } from '../services/visionBridge/tool-result-vision-bridge.js'; -import { processToolResultOmniMedia } from '../omni/tool-result-media.js'; import { getInvocationContext, runWithInvocationContext, @@ -1393,6 +1392,13 @@ export class CoreToolScheduler { // parts are invisible to isImagePart, so the bridge skips them. Sibling // step by design (§8.2): never mixed into bridge logic. Preserves the // identity-equality contract: unchanged input returns the same array. + // Dynamic import keeps the omni module graph out of the ACP/serve + // fast-path static bundle closure (mirrors fileUtils' pattern; the + // serve bundle-closure CI check forbids iconv-scale chunks on the + // acpAgent static path). + const { processToolResultOmniMedia } = await import( + '../omni/tool-result-media.js' + ); const omniProcessed = await processToolResultOmniMedia( responseParts, this.config, diff --git a/packages/core/src/omni/index.ts b/packages/core/src/omni/index.ts index da283955a3b..a22a5ae2f54 100644 --- a/packages/core/src/omni/index.ts +++ b/packages/core/src/omni/index.ts @@ -60,6 +60,9 @@ export { OmniDownloadError, type DownloadedMedia, } from './download.js'; +// Circular-safe (both modules only bind functions): lets the ./omni +// subpath entry serve the tool-result funnel without the big barrel. +export { processToolResultOmniMedia } from './tool-result-media.js'; const debugLogger = createDebugLogger('omni'); From 5c70a36bd4fa37369dc24dab24b147b7ab38f842 Mon Sep 17 00:00:00 2001 From: LaZzyMan Date: Tue, 4 Aug 2026 15:56:52 +0800 Subject: [PATCH 05/14] fix(build): register the core ./omni subpath in test resolver configs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous fix added a `./omni` subpath export to @qwen-code/qwen-code-core, but vitest/tsc in this monorepo resolve the core package through explicit aliases to TS sources rather than the package exports map — so every suite loading atCommandProcessor or the ACP Session failed with "Failed to resolve import '@qwen-code/qwen-code-core/omni'". Add the subpath alias alongside the existing transcriptRecords/goalWire precedents in cli/acp-bridge/sdk-typescript vitest configs and the cli/acp-bridge tsconfig paths. Part of the omni-experiment S2 slice (#8184). --- packages/acp-bridge/tsconfig.json | 1 + packages/acp-bridge/vitest.config.ts | 4 ++++ packages/cli/tsconfig.json | 1 + packages/cli/vitest.config.ts | 4 ++++ packages/sdk-typescript/vitest.config.ts | 4 ++++ 5 files changed, 14 insertions(+) diff --git a/packages/acp-bridge/tsconfig.json b/packages/acp-bridge/tsconfig.json index ce1a0e43a84..38b6a3fc0b0 100644 --- a/packages/acp-bridge/tsconfig.json +++ b/packages/acp-bridge/tsconfig.json @@ -12,6 +12,7 @@ "@qwen-code/qwen-code-core/transcriptRecords": [ "../core/src/utils/transcript-records.ts" ], + "@qwen-code/qwen-code-core/omni": ["../core/src/omni/index.ts"], "@qwen-code/qwen-code-core/*": ["../core/src/*"] } }, diff --git a/packages/acp-bridge/vitest.config.ts b/packages/acp-bridge/vitest.config.ts index 48e2356ea40..ac66bfb7778 100644 --- a/packages/acp-bridge/vitest.config.ts +++ b/packages/acp-bridge/vitest.config.ts @@ -18,6 +18,10 @@ export default defineConfig({ __dirname, '../core/src/utils/transcript-records.ts', ), + '@qwen-code/qwen-code-core/omni': path.resolve( + __dirname, + '../core/src/omni/index.ts', + ), }, }, test: { diff --git a/packages/cli/tsconfig.json b/packages/cli/tsconfig.json index aaa5b7d3e10..eb63283de66 100644 --- a/packages/cli/tsconfig.json +++ b/packages/cli/tsconfig.json @@ -11,6 +11,7 @@ "@qwen-code/qwen-code-core/transcriptRecords": [ "../core/src/utils/transcript-records.ts" ], + "@qwen-code/qwen-code-core/omni": ["../core/src/omni/index.ts"], "@qwen-code/qwen-code-core/*": ["../core/src/*"], "@qwen-code/acp-bridge": ["../acp-bridge/src/index.ts"], "@qwen-code/acp-bridge/transcriptReplay": [ diff --git a/packages/cli/vitest.config.ts b/packages/cli/vitest.config.ts index d6cfd6cefd6..2adb9ea9c84 100644 --- a/packages/cli/vitest.config.ts +++ b/packages/cli/vitest.config.ts @@ -19,6 +19,10 @@ export default defineConfig({ __dirname, '../core/src/utils/transcript-records.ts', ), + '@qwen-code/qwen-code-core/omni': path.resolve( + __dirname, + '../core/src/omni/index.ts', + ), '@qwen-code/qwen-code-core': path.resolve(__dirname, '../core/index.ts'), // cli's daemon-status-provider.test.ts imports `FakeAgent` / // `makeChannel` from acp-bridge's package-private diff --git a/packages/sdk-typescript/vitest.config.ts b/packages/sdk-typescript/vitest.config.ts index 7812bdb9e0d..2390d6f6003 100644 --- a/packages/sdk-typescript/vitest.config.ts +++ b/packages/sdk-typescript/vitest.config.ts @@ -46,6 +46,10 @@ export default defineConfig({ __dirname, '../core/src/utils/transcript-records.ts', ), + '@qwen-code/qwen-code-core/omni': path.resolve( + __dirname, + '../core/src/omni/index.ts', + ), '@qwen-code/acp-bridge/transcriptReplay': path.resolve( __dirname, '../acp-bridge/src/transcript-replay.ts', From f048f0d2081ef76fa6868fa0d07ff5cdd342b098 Mon Sep 17 00:00:00 2001 From: LaZzyMan Date: Tue, 4 Aug 2026 17:26:46 +0800 Subject: [PATCH 06/14] test: isolate runtime-path suites from the runner's QWEN_RUNTIME_DIR MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The self-hosted CI runners export QWEN_RUNTIME_DIR for their own qwen job tooling. Storage.getRuntimeBaseDir gives that env var top priority, so logger.test.ts (HOME-derived path expectations) and the ACP Session runtime-pinning tests (Storage.setRuntimeBaseDir-based expectations) fail whenever a PR lands on such a runner — independent of the change under test. Reproduced locally with QWEN_RUNTIME_DIR=/tmp/fake-spool. Clear the variable inside the affected suites (restored after each test), making them hermetic on any runner. --- .../cli/src/acp-integration/session/Session.test.ts | 4 ++++ packages/core/src/core/logger.test.ts | 11 +++++++++++ 2 files changed, 15 insertions(+) diff --git a/packages/cli/src/acp-integration/session/Session.test.ts b/packages/cli/src/acp-integration/session/Session.test.ts index aefd1b475bb..8b391659e53 100644 --- a/packages/cli/src/acp-integration/session/Session.test.ts +++ b/packages/cli/src/acp-integration/session/Session.test.ts @@ -461,6 +461,9 @@ describe('Session', () => { } beforeEach(() => { + // Self-hosted CI runners export QWEN_RUNTIME_DIR; it outranks + // Storage.setRuntimeBaseDir, breaking the runtime-pinning assertions. + vi.stubEnv('QWEN_RUNTIME_DIR', ''); startToolSpanSpy.mockClear(); addToolArgumentsAttributesSpy.mockClear(); addToolCallResultAttributesSpy.mockClear(); @@ -662,6 +665,7 @@ describe('Session', () => { }); afterEach(() => { + vi.unstubAllEnvs(); // Reset global runtime base dir state to prevent state leakage between tests core.Storage.setRuntimeBaseDir(null); // Clear session reference to allow garbage collection diff --git a/packages/core/src/core/logger.test.ts b/packages/core/src/core/logger.test.ts index 03b8f49fa78..519f49376ff 100644 --- a/packages/core/src/core/logger.test.ts +++ b/packages/core/src/core/logger.test.ts @@ -39,6 +39,7 @@ const hash = getProjectHash(projectDir); const TEST_HOME_DIR = path.join(os.tmpdir(), 'qwen-core-logger-home'); let originalHome: string | undefined; +let originalRuntimeDir: string | undefined; let testGeminiDir: string; let testLogFilePath: string; let testCheckpointFilePath: string; @@ -121,6 +122,11 @@ describe('Logger', () => { vi.setSystemTime(new Date('2025-01-01T12:00:00.000Z')); originalHome = process.env['HOME']; process.env['HOME'] = TEST_HOME_DIR; + // Self-hosted CI runners export QWEN_RUNTIME_DIR for their own qwen + // tooling; it outranks HOME-derived paths in Storage, so these + // path-expectation tests must run with it cleared. + originalRuntimeDir = process.env['QWEN_RUNTIME_DIR']; + delete process.env['QWEN_RUNTIME_DIR']; setTestPaths(); // Clean up before the test await cleanupLogAndCheckpointFiles(); @@ -143,6 +149,11 @@ describe('Logger', () => { } else { process.env['HOME'] = originalHome; } + if (originalRuntimeDir === undefined) { + delete process.env['QWEN_RUNTIME_DIR']; + } else { + process.env['QWEN_RUNTIME_DIR'] = originalRuntimeDir; + } }); afterAll(async () => { From 88d7e34005082459ea7575d9a93a460f2cf3608a Mon Sep 17 00:00:00 2001 From: LaZzyMan Date: Tue, 4 Aug 2026 19:39:40 +0800 Subject: [PATCH 07/14] fix(omni): gate the omni dynamic import behind isOmniEnabled MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `await import()` of the omni graph inside processSingleFileContent and resolveAtCommandQuery performs a real filesystem read under vitest (the SSR transform writes and reads the module under /tmp/RP_*/ssr/), so mock-fs based suites — pathReader's image cases — failed with ENOENT on a path they never mocked. Gate both call sites on the cheap synchronous config.isOmniEnabled() before reaching for the import. Sessions without omni configured (every existing test suite, and every user who has not opted in) now never load the module, which also avoids the wasted module load on the common path. Delivery behavior when omni IS enabled is unchanged: isOmniDeliveryActive still performs the full five-condition gate check afterwards. --- packages/cli/src/ui/hooks/atCommandProcessor.ts | 2 +- packages/core/src/utils/fileUtils.ts | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/cli/src/ui/hooks/atCommandProcessor.ts b/packages/cli/src/ui/hooks/atCommandProcessor.ts index 14c66be7953..77b5b31744c 100644 --- a/packages/cli/src/ui/hooks/atCommandProcessor.ts +++ b/packages/cli/src/ui/hooks/atCommandProcessor.ts @@ -274,7 +274,7 @@ export async function resolveAtCommandQuery({ // this branch it would fall through to filesystem resolution where the // ENOENT skip silently drops it. Only intercepted when omni delivery // is active; otherwise preserve the legacy text fall-through. - if (/^https?:\/\//i.test(pathName)) { + if (/^https?:\/\//i.test(pathName) && config.isOmniEnabled?.()) { const omni = await import('@qwen-code/qwen-code-core/omni'); if (omni.isOmniDeliveryActive(config) && omni.parseHttpUrlRef(pathName)) { if (!urlMediaRefs.some((r) => r.url === pathName)) { diff --git a/packages/core/src/utils/fileUtils.ts b/packages/core/src/utils/fileUtils.ts index 0685ea6ee57..aa2d9c30b88 100644 --- a/packages/core/src/utils/fileUtils.ts +++ b/packages/core/src/utils/fileUtils.ts @@ -1272,9 +1272,13 @@ export async function processSingleFileContent( // (bridge only activates when modalities.image is false). let omniModule: typeof import('../omni/index.js') | undefined; if ( - (fileType === 'video' && modalities.video) || - (fileType === 'audio' && modalities.audio) || - (fileType === 'image' && modalities.image) + ((fileType === 'video' && modalities.video) || + (fileType === 'audio' && modalities.audio) || + (fileType === 'image' && modalities.image)) && + // Cheap gate BEFORE the dynamic import: the import itself touches + // the filesystem (vitest SSR transforms), which breaks mock-fs + // suites and wastes a module load for every non-omni user. + config.isOmniEnabled?.() ) { const omni = await import('../omni/index.js'); if (omni.isOmniDeliveryActive(config)) { From fa3528317c31b4e8501193370a5672b695dec536 Mon Sep 17 00:00:00 2001 From: LaZzyMan Date: Wed, 5 Aug 2026 10:57:29 +0800 Subject: [PATCH 08/14] fix(omni): resolve DNS before each media fetch and close review gaps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SSRF (review-blocking): downloadMediaUrl's gate called only isPrivateHost, which classifies hostname text and IP literals but never resolves. A syntactically public name whose A/AAAA record points at 127.0.0.1, 169.254.169.254 or an RFC1918 address was therefore accepted, and because this path uploads the fetched bytes to DashScope, an internal endpoint's response would be exfiltrated. Add findNonPublicAddress() to utils/fetch.ts: it resolves the hostname via dns.lookup({all: true}) and rejects if ANY returned address is non-public, since the connect may pick any of them. IP literals short-circuit without a lookup. Extract isPrivateAddress() as the shared bare-address classifier so isPrivateIp keeps one implementation. download.ts calls it inside the redirect loop, immediately before each connect, so every accepted hop is re-validated — a permitted same-host redirect can still re-resolve to a private address between hops. Two deliberate choices, both recorded in the doc comment: the helper is NOT wired into fetchWithPolicy, because WebFetch intentionally supports intranet FQDNs resolving to private addresses, so applying it there would be a silent breaking change; and DNS failure returns null rather than failing closed, because the connect that follows reports the real error and failing closed would turn a resolver blip into something the user cannot act on. The residual check-then-connect race is documented too: fetch resolves again, so closing it fully needs an agent that pins the connect to a vetted address, which undici does not expose portably. Also fixed: sniffMediaType classified any file starting with 0xFF 0xFE (the UTF-16 LE BOM) as audio/mpeg, because 0xFE passes the frame-sync mask. Real MPEG frames never use 0xFE — the layer bits would be 11, reserved — so excluding it costs no genuine detection and keeps the "null for non-media" contract honest for callers without a secondary modality gate. Tests for the previously uncovered branches reviewers flagged: - download: public-looking host resolving to 127.0.0.1/169.254.169.254/ 10.1.2.3; mixed A records where only one is private; per-hop re-check asserting the redirect is refused before the second fetch; DNS failure; the ?? fetch fallback via a globalThis spy; and the production DNS path asserting dns.lookup receives {all: true, verbatim: true} - readMediaViaOmniDelivery: image gets the resolution + zoom_image hint parts, audio gets a bare fileData, failure yields an error result and never inline base64 - probeMediaMetadata: the audio branch reads the audio stream (proven with a file carrying both h264 and aac), the image branch reports no duration - sniffFileModality: recognized headers, non-media, and an absent path - atCommandProcessor URL pipeline: a URL with omni active becomes a fileData part, the same URL with omni disabled falls through as text without downloading, and a URL-only prompt does not hit the no-content early return --- .../src/ui/hooks/atCommandProcessor.test.ts | 110 +++++++++++ packages/core/src/omni/download.test.ts | 133 +++++++++++++ packages/core/src/omni/download.ts | 28 ++- packages/core/src/omni/ffmpeg.test.ts | 60 ++++++ packages/core/src/omni/index.test.ts | 185 +++++++++++++++++- packages/core/src/omni/recognition.test.ts | 63 ++++++ packages/core/src/omni/recognition.ts | 10 +- packages/core/src/utils/fetch.ts | 82 +++++++- 8 files changed, 661 insertions(+), 10 deletions(-) diff --git a/packages/cli/src/ui/hooks/atCommandProcessor.test.ts b/packages/cli/src/ui/hooks/atCommandProcessor.test.ts index 4d7d034d4d3..0685b0fa93f 100644 --- a/packages/cli/src/ui/hooks/atCommandProcessor.test.ts +++ b/packages/cli/src/ui/hooks/atCommandProcessor.test.ts @@ -1692,4 +1692,114 @@ describe('handleAtCommand', () => { expect(JSON.stringify(result.processedQuery)).toContain('COLON BODY'); }); }); + + describe('URL media references (@https://…)', () => { + // The omni graph is imported dynamically inside the URL branch, so a + // module-registry mock is what intercepts it. Every export the branch + // touches is stubbed; the pipeline is exercised end to end from + // handleAtCommand down to the fileData part. + const omniMocks = vi.hoisted(() => ({ + isOmniDeliveryActive: vi.fn(), + parseHttpUrlRef: vi.fn(), + downloadMediaUrl: vi.fn(), + processMediaForOmniDelivery: vi.fn(), + effectiveMaxDownloadFileBytes: vi.fn(), + })); + + vi.mock('@qwen-code/qwen-code-core/omni', () => ({ + ...omniMocks, + OmniObjectStore: class { + getOmniRootDir() { + return path.join(os.tmpdir(), 'omni-at-test'); + } + }, + })); + + function omniConfig(enabled: boolean): Config { + return { + ...mockConfig, + isOmniEnabled: () => enabled, + storage: { getQwenDir: () => path.join(os.tmpdir(), 'omni-at-test') }, + } as unknown as Config; + } + + beforeEach(async () => { + omniMocks.isOmniDeliveryActive.mockReturnValue(true); + omniMocks.parseHttpUrlRef.mockImplementation((url: string) => ({ url })); + omniMocks.effectiveMaxDownloadFileBytes.mockReturnValue(1024 * 1024); + const partPath = path.join(testRootDir, 'downloaded.mp4'); + await createTestFile(partPath, 'fake media bytes'); + omniMocks.downloadMediaUrl.mockResolvedValue({ partPath }); + omniMocks.processMediaForOmniDelivery.mockResolvedValue({ + fileUri: 'oss://bucket/clip.mp4', + mimeType: 'video/mp4', + recognized: { modality: 'video', sizeBytes: 2 * 1024 * 1024 }, + }); + }); + + it('localizes a media URL into a fileData part when omni is active', async () => { + const result = await handleAtCommand({ + query: 'summarize @https://example.com/clip.mp4 please', + config: omniConfig(true), + onDebugMessage: mockOnDebugMessage, + messageId: 700, + signal: abortController.signal, + }); + + expect(result.shouldProceed).toBe(true); + expect(omniMocks.downloadMediaUrl).toHaveBeenCalledWith( + expect.objectContaining({ url: 'https://example.com/clip.mp4' }), + ); + const parts = result.processedQuery as Array>; + expect(parts).toContainEqual({ + fileData: { + fileUri: 'oss://bucket/clip.mp4', + mimeType: 'video/mp4', + displayName: 'clip.mp4', + }, + }); + // The URL stays verbatim in the prompt text so the model can correlate + // it with the delivered part. + expect(parts[0]!['text']).toContain('https://example.com/clip.mp4'); + expect(result.toolDisplays![0]).toMatchObject({ + name: 'Fetch Media URL', + status: ToolCallStatus.Success, + }); + }); + + it('falls through to text (no download) when omni is disabled', async () => { + const query = 'summarize @https://example.com/clip.mp4 please'; + const result = await handleAtCommand({ + query, + config: omniConfig(false), + onDebugMessage: mockOnDebugMessage, + messageId: 701, + signal: abortController.signal, + }); + + // Legacy behavior: the URL is left as text, nothing is fetched. + expect(omniMocks.downloadMediaUrl).not.toHaveBeenCalled(); + expect(result.shouldProceed).toBe(true); + expect(result.processedQuery).toEqual([{ text: query }]); + }); + + it('does not hit the no-content early return for a URL-only prompt', async () => { + // urlMediaRefs must count as content; otherwise the pipeline runs and + // its parts/cards are discarded by the early return. + const result = await handleAtCommand({ + query: '@https://example.com/clip.mp4', + config: omniConfig(true), + onDebugMessage: mockOnDebugMessage, + messageId: 702, + signal: abortController.signal, + }); + + expect(result.shouldProceed).toBe(true); + const parts = result.processedQuery as Array>; + expect(parts.some((p) => 'fileData' in p)).toBe(true); + expect(mockOnDebugMessage).not.toHaveBeenCalledWith( + 'No valid file paths found in @ commands to read.', + ); + }); + }); }); diff --git a/packages/core/src/omni/download.test.ts b/packages/core/src/omni/download.test.ts index 70d51fed7e4..5d424b849e8 100644 --- a/packages/core/src/omni/download.test.ts +++ b/packages/core/src/omni/download.test.ts @@ -11,6 +11,15 @@ import os from 'node:os'; import path from 'node:path'; import { downloadMediaUrl, OmniDownloadError } from './download.js'; +// Keep the suite hermetic: the SSRF gate resolves hostnames, and tests that +// do not inject a resolver must not depend on (or wait for) real DNS. +const dnsLookupMock = vi.hoisted(() => + vi.fn(async () => [{ address: '93.184.216.34', family: 4 }]), +); +vi.mock('node:dns/promises', () => ({ + default: { lookup: dnsLookupMock }, +})); + let downloadsDir: string; beforeEach(async () => { @@ -65,6 +74,130 @@ describe('downloadMediaUrl', () => { expect(await listParts()).toEqual([]); }); + it('refuses a public-looking host that resolves to a private address', async () => { + const fetchFn = vi.fn(); + // DNS rebinding / split horizon: the name is syntactically public, so + // the text-level gate passes; only resolution reveals the target. + for (const address of ['127.0.0.1', '169.254.169.254', '10.1.2.3']) { + await expect( + downloadMediaUrl({ + url: 'https://media.example.com/clip.mp4', + downloadsDir, + maxBytes: 1_000_000, + fetchFn, + resolver: async () => [address], + }), + ).rejects.toThrow(/not publicly routable.*resolves to/); + } + expect(fetchFn).not.toHaveBeenCalled(); + expect(await listParts()).toEqual([]); + }); + + it('refuses when ANY resolved address is private (mixed A records)', async () => { + const fetchFn = vi.fn(); + await expect( + downloadMediaUrl({ + url: 'https://media.example.com/clip.mp4', + downloadsDir, + maxBytes: 1_000_000, + fetchFn, + // The connect may pick either address, so one bad entry is fatal. + resolver: async () => ['93.184.216.34', '127.0.0.1'], + }), + ).rejects.toThrow(/resolves to 127\.0\.0\.1/); + expect(fetchFn).not.toHaveBeenCalled(); + }); + + it('allows a host resolving to public addresses', async () => { + const result = await downloadMediaUrl({ + url: 'https://media.example.com/a.mp4', + downloadsDir, + maxBytes: 1_000_000, + fetchFn: fetchOk(Buffer.from('ok-bytes')), + resolver: async () => ['93.184.216.34', '2606:4700:4700::1111'], + }); + expect(result.sizeBytes).toBe(8); + }); + + it('re-checks resolution at every redirect hop', async () => { + // Hop 1 resolves public; the same-host redirect target resolves private + // (rebinding between hops) and must be refused before the second fetch. + const fetchFn = vi.fn( + async () => + new Response(null, { + status: 302, + headers: { location: 'https://media.example.com/real.mp4' }, + }), + ); + let call = 0; + await expect( + downloadMediaUrl({ + url: 'https://media.example.com/a.mp4', + downloadsDir, + maxBytes: 1_000_000, + fetchFn, + resolver: async () => (++call === 1 ? ['93.184.216.34'] : ['10.0.0.9']), + }), + ).rejects.toThrow(/resolves to 10\.0\.0\.9/); + expect(fetchFn).toHaveBeenCalledTimes(1); + expect(await listParts()).toEqual([]); + }); + + it('proceeds when DNS resolution fails (the connect reports it instead)', async () => { + // Failing closed here would turn a resolver blip into an unactionable + // error; the fetch that follows surfaces the real failure. + const result = await downloadMediaUrl({ + url: 'https://media.example.com/a.mp4', + downloadsDir, + maxBytes: 1_000_000, + fetchFn: fetchOk(Buffer.from('bytes')), + resolver: async () => { + throw new Error('EAI_AGAIN'); + }, + }); + expect(result.sizeBytes).toBe(5); + }); + + it('defaults to the global fetch when no fetchFn is injected', async () => { + // Guards the `?? fetch` fallback: every other test injects a mock, so + // deleting that fallback would otherwise leave the suite green. + const globalSpy = vi + .spyOn(globalThis, 'fetch') + .mockResolvedValue(new Response(Buffer.from('via-global'))); + try { + const result = await downloadMediaUrl({ + url: 'https://media.example.com/a.mp4', + downloadsDir, + maxBytes: 1_000_000, + resolver: async () => ['93.184.216.34'], + }); + expect(globalSpy).toHaveBeenCalledTimes(1); + expect(result.sizeBytes).toBe(10); + } finally { + globalSpy.mockRestore(); + } + }); + + it('resolves via dns.lookup when no resolver is injected', async () => { + // The `resolver` param is a test seam; production must go through DNS. + // Injecting a resolver in every other test would leave that wiring + // (and the {all: true} option the multi-address check depends on) + // completely unexercised. + dnsLookupMock.mockClear(); + await expect( + downloadMediaUrl({ + url: 'https://media.example.com/a.mp4', + downloadsDir, + maxBytes: 1_000_000, + fetchFn: fetchOk(Buffer.from('ok')), + }), + ).resolves.toMatchObject({ sizeBytes: 2 }); + expect(dnsLookupMock).toHaveBeenCalledWith('media.example.com', { + all: true, + verbatim: true, + }); + }); + it('rejects oversized Content-Length before reading the body', async () => { await expect( downloadMediaUrl({ diff --git a/packages/core/src/omni/download.ts b/packages/core/src/omni/download.ts index 3392456daed..dc197d3853e 100644 --- a/packages/core/src/omni/download.ts +++ b/packages/core/src/omni/download.ts @@ -10,7 +10,12 @@ import fs from 'node:fs/promises'; import path from 'node:path'; import { Readable } from 'node:stream'; import { pipeline } from 'node:stream/promises'; -import { isPrivateHost, isPermittedRedirect } from '../utils/fetch.js'; +import { + isPrivateHost, + isPermittedRedirect, + findNonPublicAddress, + type HostResolver, +} from '../utils/fetch.js'; import { createDebugLogger } from '../utils/debugLogger.js'; const debugLogger = createDebugLogger('omni:download'); @@ -57,7 +62,11 @@ export function parseHttpUrlRef(pathName: string): URL | null { * * Policy (mirrors utils/fetch.ts fetchWithPolicy, but streaming — that * helper buffers whole bodies in memory and is unsuitable for GiB media): - * - https or http; private/loopback hosts refused (SSRF); + * - https or http; private/loopback hosts refused (SSRF), by hostname text + * AND by resolved address — a public-looking name whose A record points at + * 127.0.0.1 or 169.254.169.254 is refused too, re-checked at every + * redirect hop (the bytes are uploaded to a third party, so DNS rebinding + * here would exfiltrate internal data); * - redirects followed only when same-host/protocol/port (isPermittedRedirect), * bounded by MAX_REDIRECTS; * - `maxBytes` enforced against BOTH Content-Length and actual bytes written; @@ -72,8 +81,10 @@ export async function downloadMediaUrl(params: { maxBytes: number; signal?: AbortSignal; fetchFn?: typeof fetch; + /** Test seam for the SSRF address check; production resolves via DNS. */ + resolver?: HostResolver; }): Promise { - const { url, downloadsDir, maxBytes, signal } = params; + const { url, downloadsDir, maxBytes, signal, resolver } = params; const fetchFn = params.fetchFn ?? fetch; if (isPrivateHost(url)) { @@ -91,6 +102,17 @@ export async function downloadMediaUrl(params: { let currentUrl = url; try { for (let redirects = 0; ; redirects++) { + // Resolve immediately before each connect, every hop: the text-level + // gate above cannot see where a public-looking name actually points, + // and a permitted same-host redirect can still re-resolve to a + // private address between hops. + const nonPublic = await findNonPublicAddress(currentUrl, resolver); + if (nonPublic) { + throw new OmniDownloadError( + `URL host is not publicly routable (refused for safety): ` + + `${new URL(currentUrl).hostname} resolves to ${nonPublic}`, + ); + } const headerAbort = new AbortController(); const headerTimer = setTimeout( () => headerAbort.abort(new Error('header timeout')), diff --git a/packages/core/src/omni/ffmpeg.test.ts b/packages/core/src/omni/ffmpeg.test.ts index 9903b9e16f7..847ad7a5325 100644 --- a/packages/core/src/omni/ffmpeg.test.ts +++ b/packages/core/src/omni/ffmpeg.test.ts @@ -15,6 +15,7 @@ import { assertOmniRuntimeDependencies, isFfmpegAvailable, isFfprobeAvailable, + probeMediaMetadata, probeVideoMetadata, resetFfmpegCachesForTests, } from './ffmpeg.js'; @@ -180,3 +181,62 @@ describe('probeVideoMetadata', () => { ); }); }); + +describe('probeMediaMetadata per-modality branches', () => { + it("reads the AUDIO stream (not video) for modality 'audio'", async () => { + // A file carrying both streams proves the audio branch selects the + // audio stream: reading videoStream?.codec_name here would yield 'h264' + // and drop sampleRate/channels entirely. + mockExecResult(() => ({ + stdout: JSON.stringify({ + format: { format_name: 'mov,mp4', duration: '12.5' }, + streams: [ + { codec_type: 'video', codec_name: 'h264', width: 640, height: 480 }, + { + codec_type: 'audio', + codec_name: 'aac', + sample_rate: '44100', + channels: 2, + }, + ], + }), + })); + await expect(probeMediaMetadata('/a.m4a', 'audio')).resolves.toEqual({ + formatName: 'mov,mp4', + durationMs: 12_500, + codec: 'aac', + sampleRateHz: 44_100, + channels: 2, + }); + }); + + it("reads only dimensions for modality 'image' (no duration)", async () => { + mockExecResult(() => ({ + stdout: JSON.stringify({ + format: { format_name: 'png_pipe', duration: '0.04' }, + streams: [ + { codec_type: 'video', codec_name: 'png', width: 1920, height: 1080 }, + ], + }), + })); + // Images must not report a duration even when ffprobe invents one. + await expect(probeMediaMetadata('/i.png', 'image')).resolves.toEqual({ + formatName: 'png_pipe', + width: 1920, + height: 1080, + codec: 'png', + }); + }); + + it("audio with no audio stream yields undefined codec, not the video's", async () => { + mockExecResult(() => ({ + stdout: JSON.stringify({ + format: { format_name: 'mp4', duration: '3' }, + streams: [{ codec_type: 'video', codec_name: 'h264' }], + }), + })); + const result = await probeMediaMetadata('/silent.mp4', 'audio'); + expect(result.codec).toBeUndefined(); + expect(result.sampleRateHz).toBeUndefined(); + }); +}); diff --git a/packages/core/src/omni/index.test.ts b/packages/core/src/omni/index.test.ts index 9ecd97f37d5..69aa66e7198 100644 --- a/packages/core/src/omni/index.test.ts +++ b/packages/core/src/omni/index.test.ts @@ -4,7 +4,10 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { afterEach, describe, expect, it, vi } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; import type { Config } from '../config/config.js'; import { AuthType } from '../core/contentGenerator.js'; import { isOmniVideoDeliveryActive } from './index.js'; @@ -122,3 +125,183 @@ describe('isOmniVideoDeliveryActive', () => { ).toBe(false); }); }); + +describe('readMediaViaOmniDelivery result shape', () => { + // Mock the leaf dependencies so the real pipeline runs end to end; the + // branch under test (image gets a resolution hint part, everything else + // gets a bare fileData) is otherwise never exercised. + function deliveryConfig(): Config { + return { + isOmniEnabled: vi.fn().mockReturnValue(true), + isTrustedFolder: vi.fn().mockReturnValue(true), + getContentGeneratorConfig: vi.fn().mockReturnValue(DASHSCOPE_CGC), + getModel: vi.fn().mockReturnValue('qwen3.5-omni-plus'), + getOmniUploadMaxFileBytes: vi.fn().mockReturnValue(0), + getOmniMaxEstimatedTokens: vi.fn().mockReturnValue(0), + storage: { getQwenDir: () => '/tmp/omni-test-qwen' }, + } as unknown as Config; + } + + function mockRecognized( + modality: 'image' | 'audio' | 'video', + metadata: Record, + ) { + return { + modality, + detectedMimeType: + modality === 'image' + ? 'image/png' + : modality === 'audio' + ? 'audio/mpeg' + : 'video/mp4', + sha256: 'a'.repeat(64), + sizeBytes: 1234, + metadata, + }; + } + + // The static `import ... from './index.js'` at the top of this file loads + // the real graph before any doMock runs, so the registry has to be dropped + // for the per-test mocks below to be the ones index.js binds to. + let tmpDir: string; + beforeEach(async () => { + vi.resetModules(); + tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'omni-index-')); + }); + + afterEach(async () => { + vi.resetAllMocks(); + vi.doUnmock('./ffmpeg.js'); + vi.doUnmock('./recognition.js'); + vi.doUnmock('./storage.js'); + vi.doUnmock('./upload.js'); + await fs.rm(tmpDir, { recursive: true, force: true }); + }); + + /** The byte guard stats the real path before anything is mocked, so the + * input has to exist on disk; its bytes are irrelevant because + * recognizeMediaFile is stubbed. */ + async function realFile(name: string): Promise { + const filePath = path.join(tmpDir, name); + await fs.writeFile(filePath, 'not really media'); + return filePath; + } + + it('adds a resolution + zoom_image hint part for images', async () => { + vi.doMock('./ffmpeg.js', () => ({ + isFfmpegAvailable: vi.fn().mockResolvedValue(true), + isFfprobeAvailable: vi.fn().mockResolvedValue(true), + })); + vi.doMock('./recognition.js', () => ({ + recognizeMediaFile: vi + .fn() + .mockResolvedValue( + mockRecognized('image', { width: 1920, height: 1080 }), + ), + extensionForMime: vi.fn().mockReturnValue('.png'), + })); + vi.doMock('./storage.js', () => ({ + OmniObjectStore: class { + async putFile() { + return { objectPath: '/tmp/obj.png', deduped: false }; + } + }, + })); + vi.doMock('./upload.js', () => ({ + DashScopeUploader: class { + async uploadFile() { + return 'oss://bucket/key'; + } + }, + OSS_URL_PREFIX: 'oss://', + })); + const { readMediaViaOmniDelivery } = await import('./index.js'); + + const result = await readMediaViaOmniDelivery({ + filePath: await realFile('pic.png'), + config: deliveryConfig(), + displayName: 'pic.png', + relativePathForDisplay: 'pic.png', + expectedModality: 'image', + }); + + expect(Array.isArray(result.llmContent)).toBe(true); + const parts = result.llmContent as Array>; + expect(parts).toHaveLength(2); + expect(parts[0]!['text']).toContain('1920x1080'); + expect(parts[0]!['text']).toContain('zoom_image'); + expect(parts[1]).toEqual({ + fileData: { + fileUri: 'oss://bucket/key', + mimeType: 'image/png', + displayName: 'pic.png', + }, + }); + }); + + it('returns a bare fileData part for audio (no zoom hint)', async () => { + vi.doMock('./ffmpeg.js', () => ({ + isFfmpegAvailable: vi.fn().mockResolvedValue(true), + isFfprobeAvailable: vi.fn().mockResolvedValue(true), + })); + vi.doMock('./recognition.js', () => ({ + recognizeMediaFile: vi + .fn() + .mockResolvedValue(mockRecognized('audio', { durationMs: 60_000 })), + extensionForMime: vi.fn().mockReturnValue('.mp3'), + })); + vi.doMock('./storage.js', () => ({ + OmniObjectStore: class { + async putFile() { + return { objectPath: '/tmp/obj.mp3', deduped: false }; + } + }, + })); + vi.doMock('./upload.js', () => ({ + DashScopeUploader: class { + async uploadFile() { + return 'oss://bucket/audio'; + } + }, + OSS_URL_PREFIX: 'oss://', + })); + const { readMediaViaOmniDelivery } = await import('./index.js'); + + const result = await readMediaViaOmniDelivery({ + filePath: await realFile('song.mp3'), + config: deliveryConfig(), + displayName: 'song.mp3', + relativePathForDisplay: 'song.mp3', + expectedModality: 'audio', + }); + + expect(Array.isArray(result.llmContent)).toBe(false); + expect(result.llmContent).toEqual({ + fileData: { + fileUri: 'oss://bucket/audio', + mimeType: 'audio/mpeg', + displayName: 'song.mp3', + }, + }); + expect(result.returnDisplay).toContain('audio'); + }); + + it('fails closed with an error result (never inline base64) on failure', async () => { + vi.doMock('./ffmpeg.js', () => ({ + isFfmpegAvailable: vi.fn().mockResolvedValue(false), + isFfprobeAvailable: vi.fn().mockResolvedValue(false), + })); + const { readMediaViaOmniDelivery } = await import('./index.js'); + + const result = await readMediaViaOmniDelivery({ + filePath: '/tmp/clip.mp4', + config: deliveryConfig(), + displayName: 'clip.mp4', + relativePathForDisplay: 'clip.mp4', + expectedModality: 'video', + }); + + expect(result.error).toMatch(/Omni media delivery failed/); + expect(result.llmContent).toContain('ffmpeg/ffprobe not available'); + }); +}); diff --git a/packages/core/src/omni/recognition.test.ts b/packages/core/src/omni/recognition.test.ts index 255d999fa48..04e25d67ef0 100644 --- a/packages/core/src/omni/recognition.test.ts +++ b/packages/core/src/omni/recognition.test.ts @@ -12,6 +12,7 @@ import path from 'node:path'; import { extensionForVideoMime, hashFileSha256, + sniffFileModality, sniffVideoMimeType, } from './recognition.js'; @@ -161,4 +162,66 @@ describe('sniffMediaType (S2 modalities)', async () => { expect(sniffMediaType(Buffer.from('%PDF-1.7'))).toBeNull(); expect(sniffMediaType(Buffer.alloc(0))).toBeNull(); }); + + it('does not mistake the UTF-16 LE BOM for an MPEG frame sync', () => { + // 0xFF 0xFE passes the naive sync mask (0xFE & 0xE0 === 0xE0), but a + // real MPEG frame never uses 0xFE (reserved layer bits). Callers + // without a secondary modality gate must not see audio/mpeg here. + const utf16le = Buffer.concat([ + Buffer.from([0xff, 0xfe]), + Buffer.from('h\0e\0l\0l\0o\0', 'latin1'), + ]); + expect(sniffMediaType(utf16le)).toBeNull(); + // Genuine frame syncs still detect. + expect(sniffMediaType(Buffer.from([0xff, 0xfb, 0x90]))).toMatchObject({ + mimeType: 'audio/mpeg', + modality: 'audio', + }); + expect(sniffMediaType(Buffer.from([0xff, 0xf3, 0x00]))).toMatchObject({ + modality: 'audio', + }); + }); +}); + +describe('sniffFileModality', () => { + it('reports the modality of a recognized media header', async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'omni-sniff-')); + try { + const video = path.join(dir, 'clip.mp4'); + await fs.writeFile(video, mp4Header('isom')); + await expect(sniffFileModality(video)).resolves.toBe('video'); + + const audio = path.join(dir, 'song.mp3'); + await fs.writeFile(audio, Buffer.from('ID3\0\0\0', 'latin1')); + await expect(sniffFileModality(audio)).resolves.toBe('audio'); + + const image = path.join(dir, 'pic.jpg'); + await fs.writeFile(image, Buffer.from([0xff, 0xd8, 0xff, 0xe0])); + await expect(sniffFileModality(image)).resolves.toBe('image'); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } + }); + + it('returns null for non-media content (legacy path keeps it)', async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'omni-sniff-')); + try { + const text = path.join(dir, 'notes.txt'); + await fs.writeFile(text, 'just some text, definitely not media'); + await expect(sniffFileModality(text)).resolves.toBeNull(); + // Empty file: stat.size 0 → zero-length read, must not throw. + const empty = path.join(dir, 'empty.bin'); + await fs.writeFile(empty, ''); + await expect(sniffFileModality(empty)).resolves.toBeNull(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } + }); + + it('returns null (never throws) for an unreadable path', async () => { + // The pre-gate must degrade to "not omni", not break the read. + await expect( + sniffFileModality(path.join(os.tmpdir(), 'omni-absent-xyz.mp4')), + ).resolves.toBeNull(); + }); }); diff --git a/packages/core/src/omni/recognition.ts b/packages/core/src/omni/recognition.ts index 17782ce2a51..c2b3ce57017 100644 --- a/packages/core/src/omni/recognition.ts +++ b/packages/core/src/omni/recognition.ts @@ -120,7 +120,15 @@ export function sniffMediaType(header: Buffer): SniffedType | null { if (header.subarray(0, 3).toString('latin1') === 'ID3') { return { mimeType: 'audio/mpeg', modality: 'audio', extension: '.mp3' }; } - if (header[0] === 0xff && (header[1]! & 0xe0) === 0xe0) { + // 0xFF 0xFE is the UTF-16 LE BOM and also passes the sync mask + // (0xFE & 0xE0 === 0xE0); excluding it keeps the "null for non-media" + // contract honest for callers without a secondary modality gate. Real + // MPEG frames never use 0xFE: bits 4-3 (layer) would be 11, reserved. + if ( + header[0] === 0xff && + header[1] !== 0xfe && + (header[1]! & 0xe0) === 0xe0 + ) { return { mimeType: 'audio/mpeg', modality: 'audio', extension: '.mp3' }; } } diff --git a/packages/core/src/utils/fetch.ts b/packages/core/src/utils/fetch.ts index 22de2a3f1d5..5907abd0ec7 100644 --- a/packages/core/src/utils/fetch.ts +++ b/packages/core/src/utils/fetch.ts @@ -7,6 +7,8 @@ import { getErrorMessage } from './errors.js'; import { delay } from './retry.js'; import { isTlsVerificationDisabled } from './runtimeFetchOptions.js'; +import dns from 'node:dns/promises'; +import net from 'node:net'; import { URL } from 'node:url'; const PRIVATE_IP_RANGES = [ @@ -101,13 +103,21 @@ function mappedIpv4(hostname: string): string | undefined { return `${hi >> 8}.${hi & 0xff}.${lo >> 8}.${lo & 0xff}`; } +/** + * Classify a bare IP address (not a URL) against the private ranges. Used + * both for URL hostnames and for the addresses a hostname resolves to. + */ +export function isPrivateAddress(address: string): boolean { + // The URL API brackets IPv6 hostnames ([::1]); strip them so the IPv6 + // ranges above can actually match. + const bare = address.replace(/^\[|\]$/g, ''); + const target = mappedIpv4(bare) ?? bare; + return PRIVATE_IP_RANGES.some((range) => range.test(target)); +} + export function isPrivateIp(url: string): boolean { try { - // The URL API brackets IPv6 hostnames ([::1]); strip them so the IPv6 - // ranges above can actually match. - const hostname = new URL(url).hostname.replace(/^\[|\]$/g, ''); - const target = mappedIpv4(hostname) ?? hostname; - return PRIVATE_IP_RANGES.some((range) => range.test(target)); + return isPrivateAddress(new URL(url).hostname); } catch (_e) { return false; } @@ -140,6 +150,68 @@ export function isPrivateHost(url: string): boolean { const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]); +/** Injectable resolver seam (tests supply a fake; production uses DNS). */ +export type HostResolver = (hostname: string) => Promise; + +/** + * Reject a URL whose host is not publicly routable, resolving hostnames + * through DNS rather than trusting their spelling. + * + * `isPrivateHost` classifies hostname *text* and IP literals, so a + * public-looking name (`media.example.com`) passes even when its A/AAAA + * record points at 127.0.0.1, 169.254.169.254 (cloud metadata) or an + * RFC1918 address — the DNS-rebinding and split-horizon cases. Callers that + * hand the fetched bytes to a third party must resolve first and reject if + * ANY returned address is non-public (all of them, since the connect may + * pick any). + * + * Deliberately NOT wired into `fetchWithPolicy`: WebFetch intentionally + * supports intranet FQDNs that resolve to private addresses. This gate is + * for paths where the response leaves the machine. + * + * Returns the offending address, or null when every address is public. + * DNS failures return null — the connect that follows will fail anyway, and + * failing closed here would turn a transient resolver blip into an error + * the user cannot act on. + * + * A check-then-connect race remains: `fetch` resolves the name again, so a + * record that flips between the two lookups is not caught. Closing it fully + * needs a custom agent that pins the connect to a vetted address, which + * undici does not expose portably. Calling this immediately before each + * connect — and again on every redirect hop — shrinks the window to the + * width of one syscall pair and blocks the practical attack (a low-TTL + * record answering public once, then private). + */ +export async function findNonPublicAddress( + url: string, + resolver?: HostResolver, +): Promise { + let hostname: string; + try { + hostname = new URL(url).hostname; + } catch { + return null; + } + // IP literals need no resolution — isPrivateAddress is the whole check. + const bare = hostname.replace(/^\[|\]$/g, ''); + if (net.isIP(bare) !== 0) { + return isPrivateAddress(bare) ? bare : null; + } + let addresses: string[]; + try { + const resolve = + resolver ?? + (async (h: string) => + (await dns.lookup(h, { all: true, verbatim: true })).map( + (entry) => entry.address, + )); + addresses = await resolve(hostname); + } catch { + return null; + } + return addresses.find((address) => isPrivateAddress(address)) ?? null; +} + /** * A redirect is followed silently only when it stays on the same host * (allowing a leading "www." to be added or removed), same protocol and From f63a76405ff9273f41e7e61a2c3e184ee6f00fa1 Mon Sep 17 00:00:00 2001 From: LaZzyMan Date: Wed, 5 Aug 2026 16:10:14 +0800 Subject: [PATCH 09/14] fix(omni): pin URL-media connections to the vetted address, fail closed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Validating the resolved address and then letting fetch re-resolve is check-then-connect: a low-TTL record answering public once and private the second time would still be connected to, and the fetched bytes are uploaded to a third party. Each hop now resolves via resolveNetworkTarget('public') — which vets every returned address and returns a lookup pinned to the vetted one — and installs that lookup as the request dispatcher, so the name is never resolved a second time. - fail closed: unresolvable/unvettable hosts and targets without a pinned lookup are refused instead of fetched unpinned - re-validated and re-pinned at every redirect hop; the previous hop's agent is replaced and closed - fetch defaults to undici's own fetch so dispatcher and fetch come from one undici version (mixing majors rejects the agent) - Node-only: Bun accepts dispatcher and silently ignores it (verified on 1.3.11), so non-Node runtimes are refused with a clear message - https and credential-free URLs enforced with named errors - utils/fetch.ts: remove findNonPublicAddress/HostResolver (a weaker duplicate of resolveNetworkTarget) and the doc paragraph wrongly claiming undici cannot pin connections --- packages/core/src/omni/download.test.ts | 332 ++++++++++++++++++++---- packages/core/src/omni/download.ts | 157 +++++++++-- packages/core/src/utils/fetch.ts | 74 +----- 3 files changed, 429 insertions(+), 134 deletions(-) diff --git a/packages/core/src/omni/download.test.ts b/packages/core/src/omni/download.test.ts index 5d424b849e8..2327e7ddf1d 100644 --- a/packages/core/src/omni/download.test.ts +++ b/packages/core/src/omni/download.test.ts @@ -7,18 +7,66 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { createHash } from 'node:crypto'; import fs from 'node:fs/promises'; +import net from 'node:net'; +import type { AddressInfo, LookupFunction } from 'node:net'; import os from 'node:os'; import path from 'node:path'; import { downloadMediaUrl, OmniDownloadError } from './download.js'; -// Keep the suite hermetic: the SSRF gate resolves hostnames, and tests that -// do not inject a resolver must not depend on (or wait for) real DNS. +// Keep the suite hermetic: the SSRF gate resolves hostnames, and tests that do +// not inject a fake target must not depend on (or wait for) real DNS. const dnsLookupMock = vi.hoisted(() => vi.fn(async () => [{ address: '93.184.216.34', family: 4 }]), ); -vi.mock('node:dns/promises', () => ({ - default: { lookup: dnsLookupMock }, -})); +vi.mock('node:dns', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, promises: { ...actual.promises, lookup: dnsLookupMock } }; +}); + +/** + * A pinned target of the shape resolveNetworkTarget returns: `lookup` answers + * with `address` regardless of the hostname asked for, which is what binds the + * connection. Tests that only need the gate's verdict never reach it. + */ +function pinnedTarget(url: string, address = '93.184.216.34') { + const lookup: LookupFunction = (_hostname, options, callback) => { + if (options.all) { + callback(null, [{ address, family: 4 }]); + } else { + callback(null, address, 4); + } + }; + return { url: new URL(url), lookup }; +} + +/** + * Ask a request's dispatcher where it is pinned, by invoking the `connect.lookup` + * it was built with. Asserting on the resolved address (rather than merely that + * a dispatcher exists) is what makes the pinning tests fail if the lookup is + * missing or points somewhere else. + */ +async function pinnedAddressOf(init: RequestInit | undefined): Promise { + const dispatcher = (init as { dispatcher?: unknown } | undefined)?.dispatcher; + if (!dispatcher) return 'NO-DISPATCHER'; + const lookup = dispatcher as { + // undici stores constructor options on a symbol-keyed internal; read the + // connect options back off it rather than reaching into private fields. + [key: symbol]: unknown; + }; + const options = Object.getOwnPropertySymbols(lookup) + .map((s) => lookup[s]) + .find( + (v): v is { connect?: { lookup?: LookupFunction } } => + typeof v === 'object' && v !== null && 'connect' in v, + ); + const fn = options?.connect?.lookup; + if (!fn) return 'NO-LOOKUP'; + return new Promise((resolve) => { + fn('media.example.com', { all: false }, (err, address) => + resolve(err ? `ERR:${err.message}` : String(address)), + ); + }); +} let downloadsDir: string; @@ -76,18 +124,18 @@ describe('downloadMediaUrl', () => { it('refuses a public-looking host that resolves to a private address', async () => { const fetchFn = vi.fn(); - // DNS rebinding / split horizon: the name is syntactically public, so - // the text-level gate passes; only resolution reveals the target. + // DNS rebinding / split horizon: the name is syntactically public, so the + // text-level gate passes; only resolution reveals the target. for (const address of ['127.0.0.1', '169.254.169.254', '10.1.2.3']) { + dnsLookupMock.mockResolvedValueOnce([{ address, family: 4 }]); await expect( downloadMediaUrl({ url: 'https://media.example.com/clip.mp4', downloadsDir, maxBytes: 1_000_000, fetchFn, - resolver: async () => [address], }), - ).rejects.toThrow(/not publicly routable.*resolves to/); + ).rejects.toThrow(/refused for safety/); } expect(fetchFn).not.toHaveBeenCalled(); expect(await listParts()).toEqual([]); @@ -95,16 +143,68 @@ describe('downloadMediaUrl', () => { it('refuses when ANY resolved address is private (mixed A records)', async () => { const fetchFn = vi.fn(); + // The connect may pick either address, so one bad entry is fatal. + dnsLookupMock.mockResolvedValueOnce([ + { address: '93.184.216.34', family: 4 }, + { address: '127.0.0.1', family: 4 }, + ]); await expect( downloadMediaUrl({ url: 'https://media.example.com/clip.mp4', downloadsDir, maxBytes: 1_000_000, fetchFn, - // The connect may pick either address, so one bad entry is fatal. - resolver: async () => ['93.184.216.34', '127.0.0.1'], }), - ).rejects.toThrow(/resolves to 127\.0\.0\.1/); + ).rejects.toThrow(/refused for safety/); + expect(fetchFn).not.toHaveBeenCalled(); + }); + + it('fails closed when the host cannot be resolved', async () => { + // Reversal of the old fail-open behavior: bytes from an unverifiable host + // are uploaded to a third party, so "cannot verify" must mean "refuse", + // not "connect and let the socket report it". + const fetchFn = vi.fn(); + dnsLookupMock.mockRejectedValueOnce( + Object.assign(new Error('EAI_AGAIN'), { code: 'EAI_AGAIN' }), + ); + await expect( + downloadMediaUrl({ + url: 'https://media.example.com/a.mp4', + downloadsDir, + maxBytes: 1_000_000, + fetchFn, + }), + ).rejects.toThrow(/could not be verified/); + expect(fetchFn).not.toHaveBeenCalled(); + expect(await listParts()).toEqual([]); + }); + + it('refuses a target that cannot be pinned to a vetted address', async () => { + // If the resolve step yields no pinned lookup, the connection cannot be + // bound, so claiming rebinding protection would be false. Refuse instead. + const fetchFn = vi.fn(); + await expect( + downloadMediaUrl({ + url: 'https://media.example.com/a.mp4', + downloadsDir, + maxBytes: 1_000_000, + fetchFn, + resolveTarget: async (u) => ({ url: new URL(u) }), + }), + ).rejects.toThrow(/cannot be safely bound/); + expect(fetchFn).not.toHaveBeenCalled(); + }); + + it('refuses plaintext http URLs', async () => { + const fetchFn = vi.fn(); + await expect( + downloadMediaUrl({ + url: 'http://media.example.com/a.mp4', + downloadsDir, + maxBytes: 1_000_000, + fetchFn, + }), + ).rejects.toThrow(/must be https/); expect(fetchFn).not.toHaveBeenCalled(); }); @@ -114,12 +214,48 @@ describe('downloadMediaUrl', () => { downloadsDir, maxBytes: 1_000_000, fetchFn: fetchOk(Buffer.from('ok-bytes')), - resolver: async () => ['93.184.216.34', '2606:4700:4700::1111'], + resolveTarget: async (u) => pinnedTarget(u), }); expect(result.sizeBytes).toBe(8); }); - it('re-checks resolution at every redirect hop', async () => { + it('pins the connection to the vetted address (real socket)', async () => { + // The regression test for the check-then-connect hole: a preflight-only + // gate lets `fetch` re-resolve the name, so this asserts the socket is + // actually opened to the address the gate approved — through the real + // undici agent, with no fetchFn injected. + // + // `pinned.invalid` has no DNS record (RFC 6761 guarantees .invalid never + // resolves), so an inbound connection can ONLY have come from the pin. The + // TLS handshake then fails against this plain TCP listener, which is fine: + // the assertion is where the connect went, not that it completed. Dropping + // the dispatcher would make this ENOTFOUND with zero connections. + const connections: string[] = []; + const server = net.createServer((socket) => { + connections.push(socket.remoteAddress ?? ''); + socket.destroy(); + }); + await new Promise((resolve) => + server.listen(0, '127.0.0.1', () => resolve()), + ); + const { port } = server.address() as AddressInfo; + try { + await expect( + downloadMediaUrl({ + url: `https://pinned.invalid:${port}/clip.mp4`, + downloadsDir, + maxBytes: 1_000_000, + resolveTarget: async (u) => pinnedTarget(u, '127.0.0.1'), + }), + ).rejects.toThrow(OmniDownloadError); + expect(connections).toHaveLength(1); + expect(await listParts()).toEqual([]); + } finally { + server.close(); + } + }); + + it('re-resolves and re-pins at every redirect hop', async () => { // Hop 1 resolves public; the same-host redirect target resolves private // (rebinding between hops) and must be refused before the second fetch. const fetchFn = vi.fn( @@ -136,53 +272,113 @@ describe('downloadMediaUrl', () => { downloadsDir, maxBytes: 1_000_000, fetchFn, - resolver: async () => (++call === 1 ? ['93.184.216.34'] : ['10.0.0.9']), + resolveTarget: async (u) => { + if (++call === 1) return pinnedTarget(u); + throw new Error('resolved to a blocked address: media.example.com'); + }, }), - ).rejects.toThrow(/resolves to 10\.0\.0\.9/); + ).rejects.toThrow(/refused for safety/); expect(fetchFn).toHaveBeenCalledTimes(1); expect(await listParts()).toEqual([]); }); - it('proceeds when DNS resolution fails (the connect reports it instead)', async () => { - // Failing closed here would turn a resolver blip into an unactionable - // error; the fetch that follows surfaces the real failure. - const result = await downloadMediaUrl({ + it('cannot be downgraded or credentialed via a redirect hop', async () => { + // The https and credentials gates run once, before the loop, so they rely + // on isPermittedRedirect refusing a hop that changes protocol or adds + // credentials. If that ever loosened, a hop could bypass both checks — + // assert the refusal here rather than trusting the invariant. + for (const location of [ + 'http://media.example.com/a.mp4', + 'https://alice:s3cret@media.example.com/a.mp4', + ]) { + const fetchFn = vi.fn( + async () => new Response(null, { status: 302, headers: { location } }), + ); + await expect( + downloadMediaUrl({ + url: 'https://media.example.com/a.mp4', + downloadsDir, + maxBytes: 1_000_000, + fetchFn, + resolveTarget: async (u) => pinnedTarget(u), + }), + ).rejects.toThrow(/Cross-origin redirect refused/); + } + expect(await listParts()).toEqual([]); + }); + + it('pins each hop to its own vetted address', async () => { + // Replaces a weaker assertion that only required `dispatcher` to be + // non-null — which would pass with an Agent carrying no lookup, or one + // pinned to the wrong address. Instead, interrogate each dispatcher by + // asking its lookup where it points, per hop. (Agent close/lifecycle is + // deliberately NOT asserted here: spying on the shared + // `Agent.prototype.close` poisons every other test in the process.) + const pins: string[] = []; + const fetchFn = vi + .fn() + .mockImplementationOnce(async (_u, init) => { + pins.push(await pinnedAddressOf(init)); + return new Response(null, { + status: 302, + headers: { location: 'https://media.example.com/real.mp4' }, + }); + }) + .mockImplementationOnce(async (_u, init) => { + pins.push(await pinnedAddressOf(init)); + return new Response(Buffer.from('ok')); + }); + let hop = 0; + await downloadMediaUrl({ url: 'https://media.example.com/a.mp4', downloadsDir, maxBytes: 1_000_000, - fetchFn: fetchOk(Buffer.from('bytes')), - resolver: async () => { - throw new Error('EAI_AGAIN'); - }, + fetchFn, + // Hop 1 and hop 2 vet to *different* addresses, so a stale agent + // reused across hops would show up as the wrong pin below. + resolveTarget: async (u) => + pinnedTarget(u, ++hop === 1 ? '93.184.216.34' : '93.184.216.35'), }); - expect(result.sizeBytes).toBe(5); + expect(pins).toEqual(['93.184.216.34', '93.184.216.35']); }); - it('defaults to the global fetch when no fetchFn is injected', async () => { - // Guards the `?? fetch` fallback: every other test injects a mock, so - // deleting that fallback would otherwise leave the suite green. - const globalSpy = vi - .spyOn(globalThis, 'fetch') - .mockResolvedValue(new Response(Buffer.from('via-global'))); + it('does not route through the global fetch (undici version-skew safety)', async () => { + // The dispatcher comes from the bundled undici, so fetch must come from the + // same version — Node's built-in fetch may be a different major whose + // handler-interface check rejects the Agent (`invalid onError method`, see + // runtimeFetchOptions.ts). Reverting to the global fetch would reintroduce + // that skew, so assert it is never called while the pin still lands. + const connections: string[] = []; + const server = net.createServer((socket) => { + connections.push(socket.remoteAddress ?? ''); + socket.destroy(); + }); + await new Promise((resolve) => + server.listen(0, '127.0.0.1', () => resolve()), + ); + const { port } = server.address() as AddressInfo; + const globalSpy = vi.spyOn(globalThis, 'fetch'); try { - const result = await downloadMediaUrl({ - url: 'https://media.example.com/a.mp4', - downloadsDir, - maxBytes: 1_000_000, - resolver: async () => ['93.184.216.34'], - }); - expect(globalSpy).toHaveBeenCalledTimes(1); - expect(result.sizeBytes).toBe(10); + await expect( + downloadMediaUrl({ + url: `https://skew.invalid:${port}/clip.mp4`, + downloadsDir, + maxBytes: 1_000_000, + resolveTarget: async (u) => pinnedTarget(u, '127.0.0.1'), + }), + ).rejects.toThrow(OmniDownloadError); + expect(globalSpy).not.toHaveBeenCalled(); + expect(connections).toHaveLength(1); } finally { globalSpy.mockRestore(); + server.close(); } }); - it('resolves via dns.lookup when no resolver is injected', async () => { - // The `resolver` param is a test seam; production must go through DNS. - // Injecting a resolver in every other test would leave that wiring - // (and the {all: true} option the multi-address check depends on) - // completely unexercised. + it('resolves via dns.lookup when no target resolver is injected', async () => { + // `resolveTarget` is a test seam; production must go through DNS. Injecting + // it everywhere would leave that wiring — and the {all: true} option the + // multi-address check depends on — completely unexercised. dnsLookupMock.mockClear(); await expect( downloadMediaUrl({ @@ -198,6 +394,54 @@ describe('downloadMediaUrl', () => { }); }); + it('refuses URLs that embed credentials, without echoing them', async () => { + // resolveNetworkTarget would also reject these, but with a message naming + // "extension network requests". Assert both the refusal and that neither + // the user nor the password reaches the error text — it is rendered in the + // UI and written to the debug log. + const fetchFn = vi.fn(); + const err = await downloadMediaUrl({ + url: 'https://alice:s3cret@media.example.com/a.mp4', + downloadsDir, + maxBytes: 1_000_000, + fetchFn, + resolveTarget: async (u) => pinnedTarget(u), + }).catch((e: Error) => e); + expect(err).toBeInstanceOf(OmniDownloadError); + expect((err as Error).message).toMatch(/must not embed credentials/); + expect((err as Error).message).not.toMatch(/s3cret|alice/); + expect(fetchFn).not.toHaveBeenCalled(); + expect(await listParts()).toEqual([]); + }); + + it('refuses to download at all on runtimes that cannot pin the connection', async () => { + // Bun accepts `dispatcher` and silently ignores it, so the pinned lookup is + // never consulted and the request connects wherever it re-resolves to — + // verified against Bun 1.3.11, for both the global fetch and undici's own. + // The gate is an allowlist (`!== 'node'`), so an unrecognized runtime is + // refused too rather than fetched unpinned. + const versions = process.versions as Record; + const previous = versions['bun']; + const fetchFn = vi.fn(); + versions['bun'] = '1.3.11'; + try { + await expect( + downloadMediaUrl({ + url: 'https://media.example.com/a.mp4', + downloadsDir, + maxBytes: 1_000_000, + fetchFn, + resolveTarget: async (u) => pinnedTarget(u), + }), + ).rejects.toThrow(/refused for safety on bun/); + } finally { + if (previous === undefined) delete versions['bun']; + else versions['bun'] = previous; + } + expect(fetchFn).not.toHaveBeenCalled(); + expect(await listParts()).toEqual([]); + }); + it('rejects oversized Content-Length before reading the body', async () => { await expect( downloadMediaUrl({ diff --git a/packages/core/src/omni/download.ts b/packages/core/src/omni/download.ts index dc197d3853e..7a03f300301 100644 --- a/packages/core/src/omni/download.ts +++ b/packages/core/src/omni/download.ts @@ -10,12 +10,12 @@ import fs from 'node:fs/promises'; import path from 'node:path'; import { Readable } from 'node:stream'; import { pipeline } from 'node:stream/promises'; +import { isPrivateHost, isPermittedRedirect } from '../utils/fetch.js'; import { - isPrivateHost, - isPermittedRedirect, - findNonPublicAddress, - type HostResolver, -} from '../utils/fetch.js'; + resolveNetworkTarget, + type ResolvedNetworkTarget, +} from '../extension/network-policy.js'; +import { loadUndici, detectRuntime } from '../utils/runtimeFetchOptions.js'; import { createDebugLogger } from '../utils/debugLogger.js'; const debugLogger = createDebugLogger('omni:download'); @@ -47,7 +47,14 @@ const HEADER_TIMEOUT_MS = 30_000; const IDLE_TIMEOUT_MS = 60_000; const MAX_REDIRECTS = 5; -/** Test whether an @-path is a downloadable http(s) URL. */ +/** + * Test whether an @-path is a downloadable http(s) URL. + * + * Deliberately still matches `http://` even though `downloadMediaUrl` refuses + * plaintext: recognizing it yields an explicit "must be https" error, whereas + * failing to recognize it would let the ref fall through to filesystem + * resolution, where the ENOENT skip drops it with no message at all. + */ export function parseHttpUrlRef(pathName: string): URL | null { if (!/^https?:\/\//i.test(pathName)) return null; try { @@ -62,11 +69,21 @@ export function parseHttpUrlRef(pathName: string): URL | null { * * Policy (mirrors utils/fetch.ts fetchWithPolicy, but streaming — that * helper buffers whole bodies in memory and is unsuitable for GiB media): - * - https or http; private/loopback hosts refused (SSRF), by hostname text - * AND by resolved address — a public-looking name whose A record points at - * 127.0.0.1 or 169.254.169.254 is refused too, re-checked at every - * redirect hop (the bytes are uploaded to a third party, so DNS rebinding - * here would exfiltrate internal data); + * - https only (plaintext is refused: media swapped in flight would be + * uploaded to a third party under the user's account); private/loopback + * hosts refused (SSRF), by hostname text + * AND by resolved address — and the vetted address is then PINNED to the + * connection via an undici agent whose `connect.lookup` returns only that + * address, so the client cannot re-resolve the name after validation. A + * preflight check alone is check-then-connect: `fetch` resolves again, and a + * low-TTL record answering public once then private would still be + * connected to. Validation failure (including a name that will not resolve) + * fails closed — these bytes are uploaded to a third party, so an + * unverifiable host must not be fetched at all; + * - re-validated and re-pinned at every redirect hop (a permitted same-host + * redirect can still re-resolve elsewhere between hops); + * - Node only: refused outright on runtimes that accept the pinning agent and + * ignore it (Bun), since there the pin is unenforceable; * - redirects followed only when same-host/protocol/port (isPermittedRedirect), * bounded by MAX_REDIRECTS; * - `maxBytes` enforced against BOTH Content-Length and actual bytes written; @@ -81,11 +98,19 @@ export async function downloadMediaUrl(params: { maxBytes: number; signal?: AbortSignal; fetchFn?: typeof fetch; - /** Test seam for the SSRF address check; production resolves via DNS. */ - resolver?: HostResolver; + /** + * Test seam for the SSRF resolve-and-pin step; production resolves via DNS + * and pins the answer (see resolveNetworkTarget). Tests inject a fake so + * they neither touch real DNS nor need a listening socket. + */ + resolveTarget?: (url: string) => Promise; }): Promise { - const { url, downloadsDir, maxBytes, signal, resolver } = params; - const fetchFn = params.fetchFn ?? fetch; + const { url, downloadsDir, maxBytes, signal } = params; + // `'public'` is what turns on resolution + address vetting + pinning; the + // helper is a no-op passthrough for any other policy. + const resolveTarget = + params.resolveTarget ?? + ((target: string) => resolveNetworkTarget(target, 'public', signal)); if (isPrivateHost(url)) { throw new OmniDownloadError( @@ -93,6 +118,50 @@ export async function downloadMediaUrl(params: { ); } + // https only, checked here so the message names the real reason: the + // resolve-and-pin step below refuses plaintext (its error would just say + // "could not be verified"). Media fetched over http can be swapped in + // flight, and these bytes are uploaded to a third party under the user's + // account — a URL whose contents cannot be attributed to the named host has + // no business being delivered to the model. + const parsed = new URL(url); + if (parsed.protocol !== 'https:') { + throw new OmniDownloadError( + `URL media must be https (refused for safety): ` + + `${parsed.protocol}//${parsed.hostname}`, + ); + } + + // Credentials in the URL are refused explicitly. `resolveNetworkTarget` + // would also reject them, but its message says "extension network requests" + // (that helper's original caller), which misattributes the reason here. The + // check is deliberately silent about the values themselves: OmniDownloadError + // messages reach the UI and the debug log. + if (parsed.username || parsed.password) { + throw new OmniDownloadError( + `URL media must not embed credentials (refused for safety): ` + + `${parsed.hostname}`, + ); + } + + // Refuse outright on runtimes that cannot bind the connection. Bun accepts + // `dispatcher` and silently ignores it (its `fetch` is a native + // implementation, and it shims the `undici` module too, so neither the + // global nor undici's own `fetch` routes through the agent) — the request + // would resolve the name itself and connect wherever it landed, with the + // pinned lookup never called. Since this path uploads the fetched bytes to a + // third party, an unenforceable pin must be a refusal rather than a claim we + // cannot keep. The same runtime fact is merely a missed optimization for + // utils/apiPreconnect.ts, which skips non-Node runtimes for this reason. + const runtime = detectRuntime(); + if (runtime !== 'node') { + throw new OmniDownloadError( + `URL media requires the Node runtime to verify the connection target ` + + `(refused for safety on ${runtime}): use a local file path instead of ` + + `a URL, or run under Node.`, + ); + } + await fs.mkdir(downloadsDir, { recursive: true, mode: 0o700 }); const partPath = path.join( downloadsDir, @@ -100,19 +169,54 @@ export async function downloadMediaUrl(params: { ); let currentUrl = url; + // Kept outside the loop: the pinned agent must stay open for the whole + // streaming read, and each redirect hop replaces it with one pinned to that + // hop's freshly vetted address. + let dispatcher: import('undici').Dispatcher | undefined; try { for (let redirects = 0; ; redirects++) { - // Resolve immediately before each connect, every hop: the text-level + // Resolve, vet, and PIN before each connect, every hop. The text-level // gate above cannot see where a public-looking name actually points, - // and a permitted same-host redirect can still re-resolve to a - // private address between hops. - const nonPublic = await findNonPublicAddress(currentUrl, resolver); - if (nonPublic) { + // and validating without pinning would only be check-then-connect — + // `fetch` would resolve the name a second time and could still be + // handed a private address. + let target: ResolvedNetworkTarget; + try { + target = await resolveTarget(currentUrl); + } catch (err) { + if (signal?.aborted) throw err; + // Fail closed: unlike the general web-fetch path, these bytes are + // uploaded to a third party, so "could not verify" must mean "do not + // fetch" rather than "connect and hope". + throw new OmniDownloadError( + `URL host is not publicly routable or could not be verified ` + + `(refused for safety): ${new URL(currentUrl).hostname}: ` + + `${err instanceof Error ? err.message : String(err)}`, + ); + } + if (!target.lookup) { + // No pinned lookup means the connection cannot be bound to the vetted + // address, so DNS-rebinding protection would be a claim we cannot + // keep. Refuse instead of fetching unpinned. throw new OmniDownloadError( - `URL host is not publicly routable (refused for safety): ` + - `${new URL(currentUrl).hostname} resolves to ${nonPublic}`, + `URL host cannot be safely bound to a verified address ` + + `(refused for safety): ${new URL(currentUrl).hostname}`, ); } + const undici = await loadUndici(); + // Default to undici's OWN fetch, not the global one. The dispatcher below + // comes from the bundled undici, and Node's built-in fetch may ship a + // different major version whose handler-interface check rejects it + // (`invalid onError method`) — see runtimeFetchOptions.ts, which pins + // undiciFetch for exactly this reason. Here the stakes are higher than a + // failed request: if the dispatcher were ever dropped rather than + // rejected, the pin would silently not apply, so fetch and Agent must + // come from one version. + const fetchFn = params.fetchFn ?? (undici.fetch as typeof fetch); + // Replace (not accumulate) the previous hop's agent. + const previousDispatcher = dispatcher; + dispatcher = new undici.Agent({ connect: { lookup: target.lookup } }); + await previousDispatcher?.close().catch(() => {}); const headerAbort = new AbortController(); const headerTimer = setTimeout( () => headerAbort.abort(new Error('header timeout')), @@ -126,7 +230,10 @@ export async function downloadMediaUrl(params: { res = await fetchFn(currentUrl, { redirect: 'manual', signal: combined, - }); + // Not in the DOM RequestInit type; undici reads it (same cast as + // services/image-generation-service.ts). + dispatcher, + } as RequestInit); } catch (err) { if (signal?.aborted) throw err; throw new OmniDownloadError( @@ -238,5 +345,9 @@ export async function downloadMediaUrl(params: { } catch (err) { await fs.rm(partPath, { force: true }); throw err; + } finally { + // The agent owns a live connection pool; the streaming read above needs it + // open until the body is fully consumed, so it can only be closed here. + await dispatcher?.close().catch(() => {}); } } diff --git a/packages/core/src/utils/fetch.ts b/packages/core/src/utils/fetch.ts index 5907abd0ec7..e814ecb0719 100644 --- a/packages/core/src/utils/fetch.ts +++ b/packages/core/src/utils/fetch.ts @@ -7,8 +7,6 @@ import { getErrorMessage } from './errors.js'; import { delay } from './retry.js'; import { isTlsVerificationDisabled } from './runtimeFetchOptions.js'; -import dns from 'node:dns/promises'; -import net from 'node:net'; import { URL } from 'node:url'; const PRIVATE_IP_RANGES = [ @@ -104,10 +102,14 @@ function mappedIpv4(hostname: string): string | undefined { } /** - * Classify a bare IP address (not a URL) against the private ranges. Used - * both for URL hostnames and for the addresses a hostname resolves to. + * Classify a bare IP address (not a URL) against the private ranges. + * + * Hostname *text* only — a public-looking name whose A record points at a + * private address is not caught here. Callers that hand fetched bytes to a + * third party must resolve and pin the connection instead; see + * `resolveNetworkTarget` in extension/network-policy.ts. */ -export function isPrivateAddress(address: string): boolean { +function isPrivateAddress(address: string): boolean { // The URL API brackets IPv6 hostnames ([::1]); strip them so the IPv6 // ranges above can actually match. const bare = address.replace(/^\[|\]$/g, ''); @@ -150,68 +152,6 @@ export function isPrivateHost(url: string): boolean { const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]); -/** Injectable resolver seam (tests supply a fake; production uses DNS). */ -export type HostResolver = (hostname: string) => Promise; - -/** - * Reject a URL whose host is not publicly routable, resolving hostnames - * through DNS rather than trusting their spelling. - * - * `isPrivateHost` classifies hostname *text* and IP literals, so a - * public-looking name (`media.example.com`) passes even when its A/AAAA - * record points at 127.0.0.1, 169.254.169.254 (cloud metadata) or an - * RFC1918 address — the DNS-rebinding and split-horizon cases. Callers that - * hand the fetched bytes to a third party must resolve first and reject if - * ANY returned address is non-public (all of them, since the connect may - * pick any). - * - * Deliberately NOT wired into `fetchWithPolicy`: WebFetch intentionally - * supports intranet FQDNs that resolve to private addresses. This gate is - * for paths where the response leaves the machine. - * - * Returns the offending address, or null when every address is public. - * DNS failures return null — the connect that follows will fail anyway, and - * failing closed here would turn a transient resolver blip into an error - * the user cannot act on. - * - * A check-then-connect race remains: `fetch` resolves the name again, so a - * record that flips between the two lookups is not caught. Closing it fully - * needs a custom agent that pins the connect to a vetted address, which - * undici does not expose portably. Calling this immediately before each - * connect — and again on every redirect hop — shrinks the window to the - * width of one syscall pair and blocks the practical attack (a low-TTL - * record answering public once, then private). - */ -export async function findNonPublicAddress( - url: string, - resolver?: HostResolver, -): Promise { - let hostname: string; - try { - hostname = new URL(url).hostname; - } catch { - return null; - } - // IP literals need no resolution — isPrivateAddress is the whole check. - const bare = hostname.replace(/^\[|\]$/g, ''); - if (net.isIP(bare) !== 0) { - return isPrivateAddress(bare) ? bare : null; - } - let addresses: string[]; - try { - const resolve = - resolver ?? - (async (h: string) => - (await dns.lookup(h, { all: true, verbatim: true })).map( - (entry) => entry.address, - )); - addresses = await resolve(hostname); - } catch { - return null; - } - return addresses.find((address) => isPrivateAddress(address)) ?? null; -} - /** * A redirect is followed silently only when it stays on the same host * (allowing a leading "www." to be added or removed), same protocol and From ba9ff593d2a0b24ac8d85b63a66643121d10abd2 Mon Sep 17 00:00:00 2001 From: LaZzyMan Date: Wed, 5 Aug 2026 16:10:30 +0800 Subject: [PATCH 10/14] fix(cli): stop the @-path parser truncating URLs at query strings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The @-path terminator set treats '?', '&', ',' and ';' as sentence punctuation, which is right for filesystem paths but wrong for URL refs: a presigned OSS/S3 link carries its signature in the query string, so '@https://…/a.mkv?x-oss-signature=…' was silently cut at the '?' and the download failed with HTTP 403 naming only the host. URL refs now run to the first character RFC 3986 does not permit unencoded (covering whitespace and unspaced CJK prose), with trailing sentence punctuation trimmed afterwards so it can never cut a URL short mid-query. Filesystem path parsing is unchanged. --- .../src/ui/hooks/atCommandProcessor.test.ts | 42 +++++++++++++++++++ .../cli/src/ui/hooks/atCommandProcessor.ts | 32 +++++++++++++- 2 files changed, 72 insertions(+), 2 deletions(-) diff --git a/packages/cli/src/ui/hooks/atCommandProcessor.test.ts b/packages/cli/src/ui/hooks/atCommandProcessor.test.ts index 0685b0fa93f..c3b8e05a56d 100644 --- a/packages/cli/src/ui/hooks/atCommandProcessor.test.ts +++ b/packages/cli/src/ui/hooks/atCommandProcessor.test.ts @@ -32,6 +32,48 @@ describe('extractAtPathCommands', () => { expect(extractAtPathCommands('@foo')).toEqual(['foo']); expect(extractAtPathCommands('@foo @bar')).toEqual(['foo', 'bar']); }); + + // The punctuation terminators exist to stop a filesystem path at a sentence + // boundary, but `?`/`&`/`,`/`;` are structural in a URL. A presigned + // OSS/S3 link carries its signature in the query string, so truncating at + // `?` produced a request that failed with HTTP 403 — and the truncation was + // invisible in the error, which named only the host. + it('keeps the whole query string of a URL ref', () => { + const presigned = + 'https://bucket.oss-cn-wulanchabu.aliyuncs.com/dir/movie.mkv' + + '?x-oss-credential=LTAI%2F20260805%2Fcn-wulanchabu%2Foss%2Faliyun_v4_request' + + '&x-oss-date=20260805T064817Z&x-oss-expires=32400' + + '&x-oss-signature-version=OSS4-HMAC-SHA256&x-oss-signature=f1454dcb'; + expect(extractAtPathCommands(`@${presigned} 视频里有什么`)).toEqual([ + presigned, + ]); + }); + + it('does not let URL punctuation leak into filesystem path parsing', () => { + // The same characters must still terminate a plain path. + expect(extractAtPathCommands('Look at @src/a.ts, then @src/b.ts;')).toEqual( + ['src/a.ts', 'src/b.ts'], + ); + }); + + it('strips trailing sentence punctuation from a URL ref', () => { + const url = 'https://example.com/clip.mp4'; + // ASCII and CJK terminators, and the query-string case where `?` is + // structural but the final `。` is prose. + expect(extractAtPathCommands(`@${url}. Explain it`)).toEqual([url]); + expect(extractAtPathCommands(`@${url}。分析一下`)).toEqual([url]); + expect(extractAtPathCommands(`@${url}?v=2, 这是什么`)).toEqual([ + `${url}?v=2`, + ]); + }); + + it('keeps whitespace as the delimiter between a URL ref and the prompt', () => { + const url = 'https://example.com/a.mp4?token=x&sig=y'; + expect(extractAtPathCommands(`@${url} @src/notes.md`)).toEqual([ + url, + 'src/notes.md', + ]); + }); }); describe('handleAtCommand', () => { diff --git a/packages/cli/src/ui/hooks/atCommandProcessor.ts b/packages/cli/src/ui/hooks/atCommandProcessor.ts index 77b5b31744c..6ff9259ef3a 100644 --- a/packages/cli/src/ui/hooks/atCommandProcessor.ts +++ b/packages/cli/src/ui/hooks/atCommandProcessor.ts @@ -121,14 +121,31 @@ function parseAllAtCommands(query: string): AtCommandPart[] { } // Parse @path + // + // A URL ref (`@https://…`) is delimited differently from a filesystem + // path. The punctuation terminators below exist to stop a path at a + // sentence boundary ("see file.txt, then…"), but `?`, `&`, `,` and `;` + // are structural in a URL — a presigned OSS/S3 link carries its signature + // in the query string, so terminating at `?` would silently drop it and + // the request would fail with HTTP 403. + // + // Instead a URL runs to the first character RFC 3986 does not permit + // unencoded. That covers whitespace and also CJK prose written with no + // space ("@https://host/a.mp4。分析一下"), which a whitespace-only rule + // would swallow into the URL. + const isUrlRef = /^https?:\/\//i.test(query.slice(atIndex + 1)); + // unreserved / gen-delims / sub-delims / pct-encoding, per RFC 3986. + const NON_URL_CHAR = /[^A-Za-z0-9\-._~:/?#[\]@!$&'()*+,;=%]/; let pathEndIndex = atIndex + 1; let inEscape = false; while (pathEndIndex < query.length) { - const char = query[pathEndIndex]; + const char = query[pathEndIndex]!; if (inEscape) { inEscape = false; } else if (char === '\\') { inEscape = true; + } else if (isUrlRef) { + if (NON_URL_CHAR.test(char)) break; } else if (/[,\s;!?()[\]{}]/.test(char)) { // Path ends at first whitespace or punctuation not escaped break; @@ -143,7 +160,18 @@ function parseAllAtCommands(query: string): AtCommandPart[] { } pathEndIndex++; } - const rawAtPath = query.substring(atIndex, pathEndIndex); + let rawAtPath = query.substring(atIndex, pathEndIndex); + if (isUrlRef) { + // `.`/`,`/`;`/`!`/`?` are legal URL characters but are usually prose + // when they land at the very end ("@https://host/a.mp4. Explain it"). + // Trimmed after the scan rather than treated as terminators, so they + // can never cut a URL short mid-query. Brackets and parens are left + // alone: they appear balanced inside real URLs, and a wrapped URL still + // yields a named error rather than a silently mangled request. + const trimmed = rawAtPath.replace(/[.,;:!?]+$/, ''); + pathEndIndex = atIndex + trimmed.length; + rawAtPath = trimmed; + } // unescapePath expects the @ symbol to be present, and will handle it. const atPath = unescapePath(rawAtPath); parts.push({ type: 'atPath', content: atPath }); From e7154f6001f8c730ad3033ca82dd743f81f6c969 Mon Sep 17 00:00:00 2001 From: LaZzyMan Date: Wed, 5 Aug 2026 17:52:45 +0800 Subject: [PATCH 11/14] fix(cli): highlight the full URL ref in the input box The input-box highlighter had its own @-ref charset that excluded '%', '?', '&' and '=', so a presigned URL was painted file-colored only up to the first percent-escape and looked truncated even after the parser was fixed to consume it whole. URL refs now use the same RFC 3986 charset and trailing-punctuation trim as parseAllAtCommands, keeping the painted span identical to what the parser consumes. --- packages/cli/src/ui/utils/highlight.test.ts | 22 +++++++++++++++++++ packages/cli/src/ui/utils/highlight.ts | 24 +++++++++++++++++++-- 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/packages/cli/src/ui/utils/highlight.test.ts b/packages/cli/src/ui/utils/highlight.test.ts index 75a9789df72..f03f4c2d693 100644 --- a/packages/cli/src/ui/utils/highlight.test.ts +++ b/packages/cli/src/ui/utils/highlight.test.ts @@ -67,6 +67,28 @@ describe('parseInputForHighlighting', () => { ]); }); + // Must stay in sync with parseAllAtCommands: the parser consumes the whole + // presigned URL (query string included), so the input box has to paint the + // same span — a half-highlighted URL reads as "the rest will be dropped". + it('should highlight a URL ref through its full query string', () => { + const url = + 'https://bucket.oss-cn-wulanchabu.aliyuncs.com/dir/%E8%BE%93%E5%87%BA.mp4' + + '?x-oss-credential=LTAI%2F20260805%2Foss&x-oss-signature=f1454dcb'; + const text = `@${url} 这个视频是谁做的`; + expect(parseInputForHighlighting(text, 0)).toEqual([ + { text: `@${url}`, type: 'file' }, + { text: ' 这个视频是谁做的', type: 'default' }, + ]); + }); + + it('should not paint trailing sentence punctuation of a URL ref', () => { + const text = '@https://example.com/a.mp4. Explain it'; + expect(parseInputForHighlighting(text, 0)).toEqual([ + { text: '@https://example.com/a.mp4', type: 'file' }, + { text: '. Explain it', type: 'default' }, + ]); + }); + it('should highlight a command in the middle when preceded by whitespace', () => { const text = 'I need /help with this'; expect(parseInputForHighlighting(text, 0)).toEqual([ diff --git a/packages/cli/src/ui/utils/highlight.ts b/packages/cli/src/ui/utils/highlight.ts index bee9a3abf81..7d2652ca810 100644 --- a/packages/cli/src/ui/utils/highlight.ts +++ b/packages/cli/src/ui/utils/highlight.ts @@ -13,8 +13,17 @@ export type HighlightToken = { type: 'default' | 'command' | 'file'; }; +// The @-ref alternatives must stay in sync with parseAllAtCommands in +// hooks/atCommandProcessor.ts, or the input box paints a different span than +// the parser will actually consume. A URL ref (`@https://…`) runs through the +// full RFC 3986 charset — `%`, `?`, `&`, `=` are structural in a presigned +// URL — while a filesystem ref keeps the narrower path charset. const HIGHLIGHT_REGEX = - /(^\/[a-zA-Z][a-zA-Z0-9:_-]*)|((?<=\s)\/[a-zA-Z][a-zA-Z0-9:_-]*)|(@(?:\\ |[a-zA-Z0-9_.:/-])+)/g; + /(^\/[a-zA-Z][a-zA-Z0-9:_-]*)|((?<=\s)\/[a-zA-Z][a-zA-Z0-9:_-]*)|(@https?:\/\/[A-Za-z0-9\-._~:/?#[\]@!$&'()*+,;=%]+)|(@(?:\\ |[a-zA-Z0-9_.:/-])+)/g; + +// Mirrors the parser's trailing-punctuation trim: '.'/','/';'/':'/'!'/'?' at +// the very end of a URL are prose, not part of the ref. +const TRAILING_URL_PUNCTUATION = /[.,;:!?]+$/; export function parseInputForHighlighting( text: string, @@ -38,9 +47,20 @@ export function parseInputForHighlighting( HIGHLIGHT_REGEX.lastIndex = 0; while ((match = HIGHLIGHT_REGEX.exec(text)) !== null) { - const [fullMatch] = match; + let [fullMatch] = match; const matchIndex = match.index; + if (match[3] !== undefined) { + // URL ref: trailing sentence punctuation is prose (parser trims it the + // same way), so paint it default rather than file-colored. + const trimmed = fullMatch.replace(TRAILING_URL_PUNCTUATION, ''); + if (trimmed.length > 1) { + fullMatch = trimmed; + // Rewind so the trimmed tail is re-scanned as default text. + HIGHLIGHT_REGEX.lastIndex = matchIndex + trimmed.length; + } + } + // Add the text before the match as a default token if (matchIndex > lastIndex) { tokens.push({ From eca83f3d2e2398dba56ac6220e798225c8359cc4 Mon Sep 17 00:00:00 2001 From: LaZzyMan Date: Wed, 5 Aug 2026 18:07:39 +0800 Subject: [PATCH 12/14] fix(omni): close review round-2 findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Criticals: - tool-result-media: staging-dir setup moved inside convertPart's try — a mkdir failure (ENOSPC, EACCES, ~/.qwen/omni as a regular file) now degrades that part to inline instead of rejecting the whole tool result and reporting a succeeded tool as failed - sanitizeErrorMessage: exact split/join replacement of the known filePath before the pattern pass (immune to CJK segments, ~-prefixed and special-character basenames), with separator-based — not ASCII-word-based — fallback classes and a Windows-path pass - the fail-closed delivery result now sanitizes BOTH llmContent and the error field (the scheduler sends 'error', not llmContent, to the model on READ_CONTENT_FAILURE) and names the file by displayName - the 100 MB image-source cap only applies when the overview decoder will actually run: it protects the decoder, and the omni path uploads original bytes under its own 1 GiB ceiling without decoding Suggestions: - malformed redirect Location surfaces as a named OmniDownloadError instead of a raw TypeError (header value kept out of the message) - header/idle watchdog timeouts are now identifiable in the surfaced error instead of a generic 'This operation was aborted' - effectiveMaxDownloadFileBytes is clamped to the upload cap even when omni.download.maxFileBytes is configured higher - estimation rejects degenerate present values (0/negative/non-finite) as 'unavailable' instead of emitting a 0-token 'ok' estimate - corrected the 0xFE sniff comment: 0xFF 0xFE is a valid MPEG-1 Layer I header (reserved encodings are version 01 / layer 00); the exclusion is a deliberate trade against UTF-16 LE false positives - SSRF tests now cover private IPv6 (loopback, link-local, ULA, IPv4-mapped) plus a public-IPv6 acceptance case Regression tests are mutation-verified: reverting the mkdir placement or the error-field sanitization makes exactly the new tests fail. --- packages/core/src/omni/download.test.ts | 57 +++++++++++++++++++ packages/core/src/omni/download.ts | 30 +++++++++- packages/core/src/omni/estimation.test.ts | 23 ++++++++ packages/core/src/omni/estimation.ts | 22 ++++--- packages/core/src/omni/index.test.ts | 53 +++++++++++++++++ packages/core/src/omni/index.ts | 53 ++++++++++++----- packages/core/src/omni/recognition.ts | 10 +++- .../core/src/omni/tool-result-media.test.ts | 26 +++++++++ packages/core/src/omni/tool-result-media.ts | 7 ++- packages/core/src/utils/fileUtils.ts | 27 ++++++--- 10 files changed, 275 insertions(+), 33 deletions(-) diff --git a/packages/core/src/omni/download.test.ts b/packages/core/src/omni/download.test.ts index 2327e7ddf1d..9d62e6356bc 100644 --- a/packages/core/src/omni/download.test.ts +++ b/packages/core/src/omni/download.test.ts @@ -141,6 +141,42 @@ describe('downloadMediaUrl', () => { expect(await listParts()).toEqual([]); }); + it('refuses a public-looking host that resolves to a private IPv6 address', async () => { + const fetchFn = vi.fn(); + // Without these the suite would stay green under a classifier that fails + // open on IPv6: loopback, link-local (incl. the fe80 metadata analogue), + // ULA, and the IPv4-mapped form of a private IPv4. + for (const address of ['::1', 'fe80::1', 'fd00::2', '::ffff:10.0.0.5']) { + dnsLookupMock.mockResolvedValueOnce([{ address, family: 6 }]); + await expect( + downloadMediaUrl({ + url: 'https://media.example.com/clip.mp4', + downloadsDir, + maxBytes: 1_000_000, + fetchFn, + }), + ).rejects.toThrow(/refused for safety/); + } + expect(fetchFn).not.toHaveBeenCalled(); + expect(await listParts()).toEqual([]); + }); + + it('accepts a host resolving to a public IPv6 address', async () => { + // The acceptance counterpart: proves the IPv6 refusals above come from + // classification rather than from IPv6 being rejected wholesale. + const bytes = Buffer.from('v6-ok'); + dnsLookupMock.mockResolvedValueOnce([ + { address: '2606:4700:4700::1111', family: 6 }, + ]); + const result = await downloadMediaUrl({ + url: 'https://media.example.com/clip.mp4', + downloadsDir, + maxBytes: 1_000_000, + fetchFn: fetchOk(bytes), + }); + expect(result.sizeBytes).toBe(bytes.length); + }); + it('refuses when ANY resolved address is private (mixed A records)', async () => { const fetchFn = vi.fn(); // The connect may pick either address, so one bad entry is fatal. @@ -526,6 +562,27 @@ describe('downloadMediaUrl', () => { ).rejects.toThrow(/Cross-origin redirect refused/); }); + it('surfaces a malformed redirect Location as a named download error', async () => { + // `new URL('http://[', base)` throws a raw TypeError; that must become an + // OmniDownloadError (and not echo the server-controlled header value). + const fetchFn = vi.fn( + async () => + new Response(null, { + status: 302, + headers: { location: 'http://[' }, + }), + ) as unknown as typeof fetch; + await expect( + downloadMediaUrl({ + url: 'https://m.example.com/a.mp4', + downloadsDir, + maxBytes: 1000, + fetchFn, + }), + ).rejects.toThrow(/malformed Location header from m\.example\.com/); + expect(await listParts()).toEqual([]); + }); + it('propagates user aborts and cleans up', async () => { const controller = new AbortController(); const fetchFn = vi.fn(async () => { diff --git a/packages/core/src/omni/download.ts b/packages/core/src/omni/download.ts index 7a03f300301..0460f938258 100644 --- a/packages/core/src/omni/download.ts +++ b/packages/core/src/omni/download.ts @@ -236,6 +236,16 @@ export async function downloadMediaUrl(params: { } as RequestInit); } catch (err) { if (signal?.aborted) throw err; + // Name the watchdog explicitly: the abort surfaces as a generic + // AbortError ("This operation was aborted") with the real reason on + // `cause`, so reading err.message alone would make a timeout + // indistinguishable from any other failed request. + if (headerAbort.signal.aborted) { + throw new OmniDownloadError( + `Download timed out waiting for response headers from ` + + `${new URL(currentUrl).hostname} (${HEADER_TIMEOUT_MS / 1000}s)`, + ); + } throw new OmniDownloadError( `Download request failed for ${new URL(currentUrl).hostname}: ${ err instanceof Error ? err.message : String(err) @@ -253,7 +263,17 @@ export async function downloadMediaUrl(params: { `Too many or invalid redirects downloading from ${new URL(currentUrl).hostname}`, ); } - const redirectUrl = new URL(location, currentUrl).toString(); + let redirectUrl: string; + try { + redirectUrl = new URL(location, currentUrl).toString(); + } catch { + // A malformed Location must surface as a download error, not a + // raw TypeError; the header value itself stays out of the message + // (server-controlled, and OmniDownloadError reaches the UI). + throw new OmniDownloadError( + `Redirect with a malformed Location header from ${new URL(currentUrl).hostname}`, + ); + } if (!isPermittedRedirect(currentUrl, redirectUrl)) { throw new OmniDownloadError( `Cross-origin redirect refused: ${new URL(currentUrl).hostname} → ${new URL(redirectUrl).hostname}`, @@ -322,6 +342,14 @@ export async function downloadMediaUrl(params: { } catch (err) { if (signal?.aborted) throw err; if (err instanceof OmniDownloadError) throw err; + // Same identifiability rule as the header path: the idle abort + // reaches here as an AbortError whose message hides the reason. + if (idleAbort.signal.aborted) { + throw new OmniDownloadError( + `Download stalled from ${new URL(currentUrl).hostname}: no data ` + + `for ${IDLE_TIMEOUT_MS / 1000}s`, + ); + } throw new OmniDownloadError( `Download interrupted from ${new URL(currentUrl).hostname}: ${ err instanceof Error ? err.message : String(err) diff --git a/packages/core/src/omni/estimation.test.ts b/packages/core/src/omni/estimation.test.ts index 8e05e1b1862..91c4a2271ce 100644 --- a/packages/core/src/omni/estimation.test.ts +++ b/packages/core/src/omni/estimation.test.ts @@ -71,4 +71,27 @@ describe('estimateRawResourceTokens (raw-resource-v1)', () => { ).status, ).toBe('unavailable'); }); + + it('degenerate present values → unavailable, never a 0-token ok', () => { + // A truncated/still-being-written capture can probe as duration 0 — + // an 'ok' 0-token estimate would sail under the transport guard while + // violating the invariant that 'ok' means a real estimate. + for (const metadata of [ + { width: 100, height: 100, durationMs: 0, frameRate: 30 }, + { width: 0, height: 100, durationMs: 1000, frameRate: 30 }, + { width: 100, height: 100, durationMs: 1000, frameRate: NaN }, + { width: 100, height: -1, durationMs: 1000, frameRate: 30 }, + ]) { + expect(estimateRawResourceTokens(media('video', metadata)).status).toBe( + 'unavailable', + ); + } + expect( + estimateRawResourceTokens(media('audio', { durationMs: 0 })).status, + ).toBe('unavailable'); + expect( + estimateRawResourceTokens(media('image', { width: 0, height: 50 })) + .status, + ).toBe('unavailable'); + }); }); diff --git a/packages/core/src/omni/estimation.ts b/packages/core/src/omni/estimation.ts index 831b3606e1a..8cc84ff62b3 100644 --- a/packages/core/src/omni/estimation.ts +++ b/packages/core/src/omni/estimation.ts @@ -36,8 +36,16 @@ const VISUAL_PIXELS_PER_TOKEN = 32 * 32 * 2; * - video → frameCount from ceil(duration × frameRate); audio track of a * video is NOT separately estimated in v1 (the visual term dominates and * the raw formula is already a conservative upper bound) - * - any required field missing → status 'unavailable' + * - any required field missing OR degenerate (zero/negative/non-finite) → + * status 'unavailable'. `'ok'` means a real estimate: a truncated capture + * probing as duration 0 must not produce a 0-token 'ok' that sails under + * the transport guard. */ + +/** Positive, finite number — rejects 0, negatives, NaN and Infinity. */ +function usable(value: number | undefined): value is number { + return value !== undefined && Number.isFinite(value) && value > 0; +} export function estimateRawResourceTokens( media: RecognizedMedia, ): OmniTokenEstimate { @@ -50,7 +58,7 @@ export function estimateRawResourceTokens( switch (media.modality) { case 'audio': { const durationMs = media.metadata.durationMs; - if (durationMs === undefined) return unavailable; + if (!usable(durationMs)) return unavailable; return { estimatedTokenCount: Math.ceil( (durationMs / 1000) * AUDIO_TOKENS_PER_SECOND, @@ -61,7 +69,7 @@ export function estimateRawResourceTokens( } case 'image': { const { width, height } = media.metadata; - if (width === undefined || height === undefined) return unavailable; + if (!usable(width) || !usable(height)) return unavailable; return { estimatedTokenCount: Math.ceil( (width * height) / VISUAL_PIXELS_PER_TOKEN, @@ -73,10 +81,10 @@ export function estimateRawResourceTokens( case 'video': { const { width, height, durationMs, frameRate } = media.metadata; if ( - width === undefined || - height === undefined || - durationMs === undefined || - frameRate === undefined + !usable(width) || + !usable(height) || + !usable(durationMs) || + !usable(frameRate) ) { return unavailable; } diff --git a/packages/core/src/omni/index.test.ts b/packages/core/src/omni/index.test.ts index 69aa66e7198..f60e64c498b 100644 --- a/packages/core/src/omni/index.test.ts +++ b/packages/core/src/omni/index.test.ts @@ -11,6 +11,7 @@ import path from 'node:path'; import type { Config } from '../config/config.js'; import { AuthType } from '../core/contentGenerator.js'; import { isOmniVideoDeliveryActive } from './index.js'; +import { effectiveMaxDownloadFileBytes } from './index.js'; function stubConfig(overrides: { omniEnabled?: boolean; @@ -34,6 +35,25 @@ afterEach(() => { delete process.env['QWEN_CODE_ENABLE_OMNI']; }); +describe('effectiveMaxDownloadFileBytes', () => { + const capsConfig = (download?: number, upload?: number): Config => + ({ + getOmniDownloadMaxFileBytes: () => download, + getOmniUploadMaxFileBytes: () => upload, + }) as unknown as Config; + + it('never exceeds the upload cap, even when configured higher', () => { + // Downloading more than the upload channel can deliver is pure waste: + // the bytes would be fetched, then rejected by the byte guard. + expect(effectiveMaxDownloadFileBytes(capsConfig(2_000, 1_000))).toBe(1_000); + expect(effectiveMaxDownloadFileBytes(capsConfig(500, 1_000))).toBe(500); + expect(effectiveMaxDownloadFileBytes(capsConfig(undefined, 1_000))).toBe( + 1_000, + ); + expect(effectiveMaxDownloadFileBytes(capsConfig(0, 1_000))).toBe(1_000); + }); +}); + describe('isOmniVideoDeliveryActive', () => { it('is active for a DashScope endpoint with a static API key in a trusted workspace', () => { expect( @@ -304,4 +324,37 @@ describe('readMediaViaOmniDelivery result shape', () => { expect(result.error).toMatch(/Omni media delivery failed/); expect(result.llmContent).toContain('ffmpeg/ffprobe not available'); }); + + it('never leaks the absolute path through error or llmContent on failure', async () => { + // Both fields reach the model: llmContent on success paths, `error` via + // the scheduler's functionResponse on READ_CONTENT_FAILURE. The paths + // here are the regex-hostile shapes from review: CJK segments, a + // `~`-prefixed basename, parens/apostrophe, and a Windows drive path — + // exact replacement of the known filePath must scrub all of them. + vi.doMock('./ffmpeg.js', () => ({ + isFfmpegAvailable: vi.fn().mockResolvedValue(true), + isFfprobeAvailable: vi.fn().mockResolvedValue(true), + })); + const { readMediaViaOmniDelivery } = await import('./index.js'); + + for (const [filePath, parentFragment] of [ + ['/Users/张三/视频/clip.mp4', '/Users/张三'], + ['/Users/a/videos/~draft.mp4', '/Users/a/videos'], + ["/Users/a/it's (v2)+final@x/clip.mp4", "it's (v2)+final@x"], + ['C:\\Users\\björn\\clip.mp4', 'C:\\Users'], + ] as const) { + const result = await readMediaViaOmniDelivery({ + filePath, + config: deliveryConfig(), + displayName: 'clip.mp4', + relativePathForDisplay: 'clip.mp4', + expectedModality: 'video', + }); + // The stat fails (files don't exist) and the fs error embeds the path. + expect(result.error).toMatch(/Omni media delivery failed/); + for (const field of [result.error, result.llmContent]) { + expect(String(field)).not.toContain(parentFragment); + } + } + }); }); diff --git a/packages/core/src/omni/index.ts b/packages/core/src/omni/index.ts index a22a5ae2f54..65d9ad6a318 100644 --- a/packages/core/src/omni/index.ts +++ b/packages/core/src/omni/index.ts @@ -67,14 +67,31 @@ export { processToolResultOmniMedia } from './tool-result-media.js'; const debugLogger = createDebugLogger('omni'); /** - * Scrub absolute POSIX path segments out of an error message, keeping the + * Scrub absolute path segments out of an error message, keeping the * basename — fs errors embed full paths (`ENOENT: … stat '/Users/x/…'`) * and several wraps below flow into model-visible llmContent, which must * never carry real paths. + * + * `knownPaths` are replaced EXACTLY (split/join) before the pattern pass: + * a regex can never enumerate every path shape (CJK segments, `~`-prefixed + * or special-character basenames, Windows drives), but the pipeline always + * knows which file it was working on, and exact replacement of that path is + * immune to all of them. The pattern pass then catches other embedded paths + * (e.g. the object-store destination) with separator-based — not + * ASCII-word-based — segment classes, so non-ASCII segments still match. */ -function sanitizeErrorMessage(err: unknown): string { - const msg = err instanceof Error ? err.message : String(err); - return msg.replace(/(?:\/[\w~. -]+)+\/([\w. -]+)/g, '$1'); +function sanitizeErrorMessage(err: unknown, knownPaths: string[] = []): string { + let msg = err instanceof Error ? err.message : String(err); + for (const known of knownPaths) { + if (known) msg = msg.split(known).join(path.basename(known)); + } + return ( + msg + // POSIX: two or more segments then a basename → keep the basename. + .replace(/(?:\/[^/\s'"]+)+\/([^/\s'"]+)/g, '$1') + // Windows: optional drive letter, backslash segments. + .replace(/(?:[A-Za-z]:)?(?:\\[^\\\s'"]+)+\\([^\\\s'"]+)/g, '$1') + ); } /** @@ -199,7 +216,7 @@ export async function processMediaForOmniDelivery( // must not stream through SHA-256 only to be rejected. const stat = await fs.stat(filePath).catch((err) => { throw new OmniDeliveryError( - `Cannot stat media file ${displayName}: ${sanitizeErrorMessage(err)}`, + `Cannot stat media file ${displayName}: ${sanitizeErrorMessage(err, [filePath])}`, { cause: err }, ); }); @@ -214,7 +231,7 @@ export async function processMediaForOmniDelivery( } catch (err) { if (signal?.aborted) throw err; throw new OmniDeliveryError( - `Media recognition failed for ${displayName}: ${sanitizeErrorMessage(err)}`, + `Media recognition failed for ${displayName}: ${sanitizeErrorMessage(err, [filePath])}`, { cause: err }, ); } @@ -238,7 +255,7 @@ export async function processMediaForOmniDelivery( } catch (err) { if (signal?.aborted) throw err; throw new OmniDeliveryError( - `Failed to store media in the omni object store: ${sanitizeErrorMessage(err)}`, + `Failed to store media in the omni object store: ${sanitizeErrorMessage(err, [filePath])}`, { cause: err }, ); } @@ -377,11 +394,17 @@ export async function readMediaViaOmniDelivery(params: { // Fail closed: no silent fallback to inline base64 — a fallback would // resurrect the 10MB cap surprise and mislead the user into thinking // the model saw the original media. - const message = err instanceof Error ? err.message : String(err); + // + // Both llmContent AND error are model-visible: on READ_CONTENT_FAILURE + // the scheduler puts `error` (not the sanitized llmContent) into the + // functionResponse, so an unsanitized message here would leak the + // absolute path on every failed read_file. Sanitize once, use twice, + // and name the file by displayName rather than its real path. + const message = sanitizeErrorMessage(err, [filePath]); return { llmContent: `[Omni media delivery failed for ${displayName}: ${message}]`, returnDisplay: `Failed to deliver media via omni upload: ${relativePathForDisplay}`, - error: `Omni media delivery failed: ${filePath}: ${message}`, + error: `Omni media delivery failed: ${displayName}: ${message}`, errorType: ToolErrorType.READ_CONTENT_FAILURE, }; } @@ -398,10 +421,14 @@ export async function readVideoViaOmniDelivery(params: { return readMediaViaOmniDelivery({ ...params, expectedModality: 'video' }); } -/** Effective download byte ceiling — aligned with the upload channel cap - * (downloading more than can be delivered is pointless). */ +/** Effective download byte ceiling — never above the upload channel cap + * (downloading more than can be delivered is pointless), including when + * `omni.download.maxFileBytes` is explicitly configured higher. */ export function effectiveMaxDownloadFileBytes(config: Config): number { + const uploadCap = effectiveMaxUploadFileBytes(config); const configured = config.getOmniDownloadMaxFileBytes?.(); - if (configured !== undefined && configured > 0) return configured; - return effectiveMaxUploadFileBytes(config); + if (configured !== undefined && configured > 0) { + return Math.min(configured, uploadCap); + } + return uploadCap; } diff --git a/packages/core/src/omni/recognition.ts b/packages/core/src/omni/recognition.ts index c2b3ce57017..aaaa3d4cbe7 100644 --- a/packages/core/src/omni/recognition.ts +++ b/packages/core/src/omni/recognition.ts @@ -121,9 +121,13 @@ export function sniffMediaType(header: Buffer): SniffedType | null { return { mimeType: 'audio/mpeg', modality: 'audio', extension: '.mp3' }; } // 0xFF 0xFE is the UTF-16 LE BOM and also passes the sync mask - // (0xFE & 0xE0 === 0xE0); excluding it keeps the "null for non-media" - // contract honest for callers without a secondary modality gate. Real - // MPEG frames never use 0xFE: bits 4-3 (layer) would be 11, reserved. + // (0xFE & 0xE0 === 0xE0). Per the MPEG audio header 0xFF 0xFE is + // technically VALID (version bits 11 = MPEG-1, layer bits 11 = Layer I, + // CRC-protected) — the reserved encodings are version 01 and layer 00 — + // so this exclusion deliberately trades away that rare shape + // (CRC-protected Layer I) to keep UTF-16 LE text files from sniffing as + // audio, preserving the "null for non-media" contract for callers + // without a secondary modality gate. if ( header[0] === 0xff && header[1] !== 0xfe && diff --git a/packages/core/src/omni/tool-result-media.test.ts b/packages/core/src/omni/tool-result-media.test.ts index 06ab457fed5..02a92c0d893 100644 --- a/packages/core/src/omni/tool-result-media.test.ts +++ b/packages/core/src/omni/tool-result-media.test.ts @@ -134,4 +134,30 @@ describe('processToolResultOmniMedia', () => { expect(result[0]!.inlineData).toBeDefined(); expect(result[1]!.fileData?.fileUri).toBe('oss://bucket/key2'); }); + + it('keeps the part inline when staging-dir setup itself fails', async () => { + // ~/.qwen/omni existing as a regular FILE makes mkdir fail with ENOTDIR. + // That failure must degrade THIS part to inline like any other delivery + // failure — not reject the whole call, which would report a tool that + // succeeded as failed. + const os = await import('node:os'); + const nodePath = await import('node:path'); + const fs = await import('node:fs/promises'); + const qwenDir = await fs.mkdtemp(nodePath.join(os.tmpdir(), 'omni-trm-')); + // OmniObjectStore roots at /omni; make that path a plain file. + await fs.writeFile(nodePath.join(qwenDir, 'omni'), 'not a directory'); + try { + const config = { + isOmniEnabled: () => true, + getContentGeneratorConfig: () => ({ modalities: { image: true } }), + storage: { getQwenDir: () => qwenDir }, + } as unknown as Config; + const parts = [inlinePart('image/png', PNG_BYTES)]; + const result = await processToolResultOmniMedia(parts, config, signal); + expect(result).toBe(parts); + expect(deliverMock).not.toHaveBeenCalled(); + } finally { + await fs.rm(qwenDir, { recursive: true, force: true }); + } + }); }); diff --git a/packages/core/src/omni/tool-result-media.ts b/packages/core/src/omni/tool-result-media.ts index 3f15244131f..16fba03ce3f 100644 --- a/packages/core/src/omni/tool-result-media.ts +++ b/packages/core/src/omni/tool-result-media.ts @@ -79,14 +79,19 @@ export async function processToolResultOmniMedia( return part; } + // Everything from staging-dir setup onward sits inside the try: mkdir + // itself can fail (ENOSPC, EACCES on ~/.qwen, ~/.qwen/omni existing as a + // regular file → ENOTDIR), and the contract is that failure of any single + // part leaves THAT part inline — not that the whole tool result rejects, + // which would report a tool that succeeded as failed. const store = new OmniObjectStore(config.storage.getQwenDir()); const stagingDir = path.join(store.getOmniRootDir(), 'downloads'); - await fs.mkdir(stagingDir, { recursive: true, mode: 0o700 }); const tempPath = path.join( stagingDir, `${randomBytes(8).toString('hex')}.part`, ); try { + await fs.mkdir(stagingDir, { recursive: true, mode: 0o700 }); await fs.writeFile(tempPath, bytes, { mode: 0o600 }); const delivery = await processMediaForOmniDelivery(tempPath, config, { expectedModality: sniffed.modality, diff --git a/packages/core/src/utils/fileUtils.ts b/packages/core/src/utils/fileUtils.ts index aa2d9c30b88..b838cfeecd6 100644 --- a/packages/core/src/utils/fileUtils.ts +++ b/packages/core/src/utils/fileUtils.ts @@ -1252,14 +1252,6 @@ export async function processSingleFileContent( }; } } - if (shouldRenderImageOverview && stats.size > IMAGE_MAX_SOURCE_BYTES) { - return { - llmContent: 'Image file exceeds the 100 MB source limit.', - returnDisplay: 'Image file exceeds the 100 MB source limit.', - error: `Image file exceeds the 100 MB source limit: ${filePath}`, - errorType: ToolErrorType.FILE_TOO_LARGE, - }; - } // Omni experiment: when active, local media files (image/audio/video) // are delivered via the DashScope upload channel instead of inline // base64, with a 1 GiB per-file ceiling replacing the 10MB inline cap @@ -1296,6 +1288,25 @@ export async function processSingleFileContent( } } + // 100 MB source cap protects the overview DECODER, so it only applies + // when the overview will actually decode — i.e. when omni is not taking + // this file. The omni path uploads original bytes without decoding and + // enforces its own omni.upload.maxFileBytes ceiling (1 GiB default); + // gating it here too would reject a 150 MB PNG while delivering a + // 500 MB GIF, purely on whether the format has an overview renderer. + if ( + shouldRenderImageOverview && + omniModule === undefined && + stats.size > IMAGE_MAX_SOURCE_BYTES + ) { + return { + llmContent: 'Image file exceeds the 100 MB source limit.', + returnDisplay: 'Image file exceeds the 100 MB source limit.', + error: `Image file exceeds the 100 MB source limit: ${filePath}`, + errorType: ToolErrorType.FILE_TOO_LARGE, + }; + } + if ( fileSizeInMB > 9.9 && !willExtractPdfText && From f5f8bcf98f7c2fb43d9f3369e99123f05ecec632 Mon Sep 17 00:00:00 2001 From: LaZzyMan Date: Thu, 6 Aug 2026 10:57:31 +0800 Subject: [PATCH 13/14] fix(omni): close review round-3 criticals - sanitizeErrorMessage: a path segment containing a space defeats the pattern pass (segment classes break at whitespace), so 'john doe' survived scrubbing. Call sites now pass every path they know about: the store wrap adds the store root (covers putFile throwing before objectPath is assigned), and the upload wrap - previously passing err.message through verbatim - sanitizes with the object path and store root. Function exported for direct shape tests. - estimation: animated images (GIF/APNG/animated WebP) no longer estimate as a single frame. The image probe branch retains ffprobe's nb_frames as frameCount, and the estimator applies the design 6.4 visual formula with the real count - a 480x480 300-frame GIF now estimates ~33,750 tokens instead of 113, so an enabled token guard actually engages. Static images and containers that do not report a frame count keep frameCount 1. - tool-result media: a transport-guard rejection is a policy verdict, not a transfer failure - the part is now withheld with a text placeholder instead of being delivered inline as base64, which silently bypassed an enabled guard at greater request cost than the upload it was rejecting. Transfer failures keep the inline degradation; the batch never fails for one part. - mid-turn URL media: resolveAtCommandQuery runs under a fixed 10s mid-turn budget that structurally killed every mid-turn @https reference (the download alone can exceed it, and inside the resolver the timeout abort is indistinguishable from a user cancel). URL-media turns are now exempt from that timer - the download path carries its own 30s header and 60s idle watchdogs - while filesystem resolution keeps the 10s cap unchanged. - input-box highlighter: match the URL scheme case-insensitively, like the parser it mirrors; an uppercase @HTTPS ref no longer half-paints. --- packages/cli/src/ui/hooks/useGeminiStream.ts | 23 +++++++-- packages/cli/src/ui/utils/highlight.test.ts | 10 ++++ packages/cli/src/ui/utils/highlight.ts | 2 +- packages/core/src/omni/estimation.test.ts | 16 +++++++ packages/core/src/omni/estimation.ts | 13 +++-- packages/core/src/omni/ffmpeg.test.ts | 48 +++++++++++++++++++ packages/core/src/omni/ffmpeg.ts | 15 +++++- packages/core/src/omni/index.test.ts | 34 +++++++++++++ packages/core/src/omni/index.ts | 16 +++++-- .../core/src/omni/tool-result-media.test.ts | 36 ++++++++++++++ packages/core/src/omni/tool-result-media.ts | 18 ++++++- 11 files changed, 217 insertions(+), 14 deletions(-) diff --git a/packages/cli/src/ui/hooks/useGeminiStream.ts b/packages/cli/src/ui/hooks/useGeminiStream.ts index bfd9389b2ed..72edf85c38d 100644 --- a/packages/cli/src/ui/hooks/useGeminiStream.ts +++ b/packages/cli/src/ui/hooks/useGeminiStream.ts @@ -2462,11 +2462,24 @@ export const useGeminiStream = ( if (isAtCommand(message)) { const timeout = new AbortController(); const atCommandSignal = AbortSignal.any([signal, timeout.signal]); - const timeoutId = setTimeout(() => { - timeout.abort( - new Error(MID_TURN_AT_COMMAND_RESOLVE_TIMEOUT_MESSAGE), - ); - }, MID_TURN_AT_COMMAND_RESOLVE_TIMEOUT_MS); + // URL media refs are exempt from the fixed mid-turn budget: the + // omni URL path downloads and uploads media end-to-end inside + // resolveAtCommandQuery under its own watchdogs (30s header, 60s + // idle), so a 10s cap would structurally kill every mid-turn + // @https reference — and inside the resolver the timeout abort is + // indistinguishable from a user cancel, so the message would be + // dropped quietly rather than failing with a named error. + // Filesystem resolution keeps the cap unchanged. + const hasUrlMediaRef = + /@https?:\/\//i.test(message) && + (config.isOmniEnabled?.() ?? false); + const timeoutId = hasUrlMediaRef + ? undefined + : setTimeout(() => { + timeout.abort( + new Error(MID_TURN_AT_COMMAND_RESOLVE_TIMEOUT_MESSAGE), + ); + }, MID_TURN_AT_COMMAND_RESOLVE_TIMEOUT_MS); try { const atCommandResult = await resolveWithAbort( atCommandSignal, diff --git a/packages/cli/src/ui/utils/highlight.test.ts b/packages/cli/src/ui/utils/highlight.test.ts index f03f4c2d693..e50c0c50c72 100644 --- a/packages/cli/src/ui/utils/highlight.test.ts +++ b/packages/cli/src/ui/utils/highlight.test.ts @@ -89,6 +89,16 @@ describe('parseInputForHighlighting', () => { ]); }); + it('should match the URL scheme case-insensitively, like the parser', () => { + // parseAllAtCommands matches /^https?:\/\//i — an uppercase scheme must + // not fall back to the narrower filesystem charset and half-paint. + const url = 'HTTPS://example.com/a.mp4?sig=abc%2Fdef'; + expect(parseInputForHighlighting(`@${url} check`, 0)).toEqual([ + { text: `@${url}`, type: 'file' }, + { text: ' check', type: 'default' }, + ]); + }); + it('should highlight a command in the middle when preceded by whitespace', () => { const text = 'I need /help with this'; expect(parseInputForHighlighting(text, 0)).toEqual([ diff --git a/packages/cli/src/ui/utils/highlight.ts b/packages/cli/src/ui/utils/highlight.ts index 7d2652ca810..5ffe35e0abc 100644 --- a/packages/cli/src/ui/utils/highlight.ts +++ b/packages/cli/src/ui/utils/highlight.ts @@ -19,7 +19,7 @@ export type HighlightToken = { // full RFC 3986 charset — `%`, `?`, `&`, `=` are structural in a presigned // URL — while a filesystem ref keeps the narrower path charset. const HIGHLIGHT_REGEX = - /(^\/[a-zA-Z][a-zA-Z0-9:_-]*)|((?<=\s)\/[a-zA-Z][a-zA-Z0-9:_-]*)|(@https?:\/\/[A-Za-z0-9\-._~:/?#[\]@!$&'()*+,;=%]+)|(@(?:\\ |[a-zA-Z0-9_.:/-])+)/g; + /(^\/[a-zA-Z][a-zA-Z0-9:_-]*)|((?<=\s)\/[a-zA-Z][a-zA-Z0-9:_-]*)|(@[hH][tT][tT][pP][sS]?:\/\/[A-Za-z0-9\-._~:/?#[\]@!$&'()*+,;=%]+)|(@(?:\\ |[a-zA-Z0-9_.:/-])+)/g; // Mirrors the parser's trailing-punctuation trim: '.'/','/';'/':'/'!'/'?' at // the very end of a URL are prose, not part of the ref. diff --git a/packages/core/src/omni/estimation.test.ts b/packages/core/src/omni/estimation.test.ts index 91c4a2271ce..19469da40e1 100644 --- a/packages/core/src/omni/estimation.test.ts +++ b/packages/core/src/omni/estimation.test.ts @@ -45,6 +45,22 @@ describe('estimateRawResourceTokens (raw-resource-v1)', () => { expect(est.status).toBe('ok'); }); + it('animated image: real frameCount from the probe, not 1', () => { + // A 480×480 300-frame GIF at one frame would estimate ~113 tokens and + // sail under any realistic guard threshold; the real estimate is ~33,750. + const est = estimateRawResourceTokens( + media('image', { width: 480, height: 480, frameCount: 300 }), + ); + expect(est.estimatedTokenCount).toBe(Math.ceil((480 * 480 * 300) / 2048)); + expect(est.status).toBe('ok'); + // Degenerate frame counts fall back to the static single frame. + expect( + estimateRawResourceTokens( + media('image', { width: 480, height: 480, frameCount: 0 }), + ).estimatedTokenCount, + ).toBe(Math.ceil((480 * 480) / 2048)); + }); + it('video: frameCount from duration × frameRate', () => { const est = estimateRawResourceTokens( media('video', { diff --git a/packages/core/src/omni/estimation.ts b/packages/core/src/omni/estimation.ts index 8cc84ff62b3..480b49ee7aa 100644 --- a/packages/core/src/omni/estimation.ts +++ b/packages/core/src/omni/estimation.ts @@ -32,7 +32,8 @@ const VISUAL_PIXELS_PER_TOKEN = 32 * 32 * 2; * audioTokens = ceil(durationSeconds × 7) * visualTokens = ceil(width × height × frameCount / (32 × 32 × 2)) * - * - static image → frameCount 1 + * - static image → frameCount 1; animated image (GIF/APNG/animated WebP) + * → real frameCount from the probe when reported * - video → frameCount from ceil(duration × frameRate); audio track of a * video is NOT separately estimated in v1 (the visual term dominates and * the raw formula is already a conservative upper bound) @@ -68,11 +69,17 @@ export function estimateRawResourceTokens( }; } case 'image': { - const { width, height } = media.metadata; + const { width, height, frameCount } = media.metadata; if (!usable(width) || !usable(height)) return unavailable; + // Animated images (GIF/APNG/animated WebP) carry their real frame + // count from the probe — a 300-frame GIF estimated as one frame would + // sail under the transport guard at ~1/300 of its real cost. Static + // images (and containers whose frame count ffprobe does not report) + // keep frameCount 1 per the design formula. + const frames = usable(frameCount) ? frameCount : 1; return { estimatedTokenCount: Math.ceil( - (width * height) / VISUAL_PIXELS_PER_TOKEN, + (width * height * frames) / VISUAL_PIXELS_PER_TOKEN, ), method: 'raw-resource-v1', status: 'ok', diff --git a/packages/core/src/omni/ffmpeg.test.ts b/packages/core/src/omni/ffmpeg.test.ts index 847ad7a5325..b4075e85e35 100644 --- a/packages/core/src/omni/ffmpeg.test.ts +++ b/packages/core/src/omni/ffmpeg.test.ts @@ -228,6 +228,54 @@ describe('probeMediaMetadata per-modality branches', () => { }); }); + it('reports frameCount for animated images (nb_frames)', async () => { + // Animated GIF/APNG/WebP report nb_frames on the video stream; the token + // estimator needs the real count — a 300-frame GIF estimated as a single + // frame would sail under the transport guard at ~1/300 of its real cost. + mockExecResult(() => ({ + stdout: JSON.stringify({ + format: { format_name: 'gif' }, + streams: [ + { + codec_type: 'video', + codec_name: 'gif', + width: 480, + height: 480, + nb_frames: '300', + }, + ], + }), + })); + await expect(probeMediaMetadata('/anim.gif', 'image')).resolves.toEqual({ + formatName: 'gif', + width: 480, + height: 480, + codec: 'gif', + frameCount: 300, + }); + }); + + it('omits frameCount when nb_frames is absent or unusable', async () => { + for (const nb of [undefined, '0', 'N/A']) { + mockExecResult(() => ({ + stdout: JSON.stringify({ + format: { format_name: 'webp' }, + streams: [ + { + codec_type: 'video', + codec_name: 'webp', + width: 64, + height: 64, + ...(nb === undefined ? {} : { nb_frames: nb }), + }, + ], + }), + })); + const result = await probeMediaMetadata('/i.webp', 'image'); + expect(result.frameCount).toBeUndefined(); + } + }); + it("audio with no audio stream yields undefined codec, not the video's", async () => { mockExecResult(() => ({ stdout: JSON.stringify({ diff --git a/packages/core/src/omni/ffmpeg.ts b/packages/core/src/omni/ffmpeg.ts index 9ae682f7f10..2023a0a74f9 100644 --- a/packages/core/src/omni/ffmpeg.ts +++ b/packages/core/src/omni/ffmpeg.ts @@ -135,6 +135,9 @@ export interface MediaProbeResult { frameRate?: number; /** Codec name of the primary stream for the modality. */ codec?: string; + /** Frame count of the primary video stream (image: >1 means animated — + * GIF/APNG/animated WebP; absent when the container does not report it). */ + frameCount?: number; /** Sample rate in Hz of the first audio stream (audio only). */ sampleRateHz?: number; /** Channel count of the first audio stream (audio only). */ @@ -200,6 +203,7 @@ export async function probeMediaMetadata( height?: number; avg_frame_rate?: string; r_frame_rate?: string; + nb_frames?: string; sample_rate?: string; channels?: number; }>; @@ -221,13 +225,22 @@ export async function probeMediaMetadata( const base: MediaProbeResult = { formatName: parsed.format?.format_name }; switch (modality) { - case 'image': + case 'image': { + // Animated images (GIF/APNG/animated WebP) report nb_frames on their + // video stream; a single-frame image reports 1 or omits it. The token + // estimator needs the real count — an animated GIF estimated as one + // frame sails under the transport guard at ~1/300 of its real cost. + const nbFrames = Number(videoStream?.nb_frames); return { ...base, width: videoStream?.width, height: videoStream?.height, codec: videoStream?.codec_name, + ...(Number.isFinite(nbFrames) && nbFrames > 0 + ? { frameCount: nbFrames } + : {}), }; + } case 'audio': return { ...base, diff --git a/packages/core/src/omni/index.test.ts b/packages/core/src/omni/index.test.ts index f60e64c498b..660b7e56f0d 100644 --- a/packages/core/src/omni/index.test.ts +++ b/packages/core/src/omni/index.test.ts @@ -12,6 +12,7 @@ import type { Config } from '../config/config.js'; import { AuthType } from '../core/contentGenerator.js'; import { isOmniVideoDeliveryActive } from './index.js'; import { effectiveMaxDownloadFileBytes } from './index.js'; +import { sanitizeErrorMessage } from './index.js'; function stubConfig(overrides: { omniEnabled?: boolean; @@ -35,6 +36,39 @@ afterEach(() => { delete process.env['QWEN_CODE_ENABLE_OMNI']; }); +describe('sanitizeErrorMessage', () => { + it('scrubs known paths exactly, including segments with spaces', () => { + // A space inside a segment defeats the pattern pass (segment classes + // break at whitespace — '/Users/john doe/…' would surface 'john doe'); + // known-path exact replacement is the only mechanism immune to it. + const spaced = '/Users/john doe/.qwen/omni/objects/ab/abcd1234deadbeef.png'; + const err = new Error(`EACCES: permission denied, open '${spaced}'`); + const out = sanitizeErrorMessage(err, [spaced]); + expect(out).not.toContain('john doe'); + expect(out).toContain('abcd1234deadbeef.png'); + }); + + it('scrubs a known store ROOT even when the full object path is unknown', () => { + // putFile can throw before objectPath is assigned; passing the store + // root as a known path still removes the user-identifying prefix. + const root = '/Users/john doe/.qwen/omni'; + const err = new Error( + `ENOSPC: no space left on device, write '${root}/objects/cd/ef99.webm'`, + ); + const out = sanitizeErrorMessage(err, [root]); + expect(out).not.toContain('john doe'); + expect(out).toContain('ef99.webm'); + }); + + it('pattern pass still collapses unknown space-free absolute paths', () => { + const err = new Error( + "ENOENT: no such file or directory, stat '/opt/data/media/clip.mp4'", + ); + expect(sanitizeErrorMessage(err)).not.toContain('/opt/data'); + expect(sanitizeErrorMessage(err)).toContain('clip.mp4'); + }); +}); + describe('effectiveMaxDownloadFileBytes', () => { const capsConfig = (download?: number, upload?: number): Config => ({ diff --git a/packages/core/src/omni/index.ts b/packages/core/src/omni/index.ts index 65d9ad6a318..c97721916b2 100644 --- a/packages/core/src/omni/index.ts +++ b/packages/core/src/omni/index.ts @@ -80,7 +80,12 @@ const debugLogger = createDebugLogger('omni'); * (e.g. the object-store destination) with separator-based — not * ASCII-word-based — segment classes, so non-ASCII segments still match. */ -function sanitizeErrorMessage(err: unknown, knownPaths: string[] = []): string { +// Exported for direct unit testing of the path shapes (visible only via the +// module namespace; not re-exported from any barrel). +export function sanitizeErrorMessage( + err: unknown, + knownPaths: string[] = [], +): string { let msg = err instanceof Error ? err.message : String(err); for (const known of knownPaths) { if (known) msg = msg.split(known).join(path.basename(known)); @@ -255,7 +260,8 @@ export async function processMediaForOmniDelivery( } catch (err) { if (signal?.aborted) throw err; throw new OmniDeliveryError( - `Failed to store media in the omni object store: ${sanitizeErrorMessage(err, [filePath])}`, + `Failed to store media in the omni object store: ` + + `${sanitizeErrorMessage(err, [filePath, store.getOmniRootDir()])}`, { cause: err }, ); } @@ -275,8 +281,12 @@ export async function processMediaForOmniDelivery( }); } catch (err) { if (signal?.aborted) throw err; + // Upload errors can embed the object-store path (spawn/fs failures) — + // sanitize with the concrete path AND the store root, since a path with + // a space in a segment defeats the pattern pass (segment classes break + // at whitespace) and only exact replacement is immune. throw new OmniDeliveryError( - err instanceof Error ? err.message : String(err), + sanitizeErrorMessage(err, [objectPath, store.getOmniRootDir()]), { cause: err }, ); } diff --git a/packages/core/src/omni/tool-result-media.test.ts b/packages/core/src/omni/tool-result-media.test.ts index 02a92c0d893..03a1c38aa08 100644 --- a/packages/core/src/omni/tool-result-media.test.ts +++ b/packages/core/src/omni/tool-result-media.test.ts @@ -135,6 +135,42 @@ describe('processToolResultOmniMedia', () => { expect(result[1]!.fileData?.fileUri).toBe('oss://bucket/key2'); }); + it('withholds the part with a text placeholder on a transport-guard rejection', async () => { + // A guard rejection is a policy verdict — keeping the part inline would + // deliver the exact bytes the guard was configured to reject. Must become + // a text placeholder, NOT stay inlineData, and NOT fail the whole batch. + const { OmniTransportGuardError } = await import('./guard.js'); + deliverMock + .mockRejectedValueOnce( + new OmniTransportGuardError('x.png exceeds the omni upload limit'), + ) + .mockResolvedValueOnce({ + fileUri: 'oss://bucket/key3', + mimeType: 'image/png', + sha256: 'c'.repeat(64), + recognized: { modality: 'image' }, + tokenEstimate: { + estimatedTokenCount: 1, + method: 'raw-resource-v1', + status: 'ok', + }, + deduped: false, + }); + const parts = [ + inlinePart('image/png', PNG_BYTES), + inlinePart('image/png', PNG_BYTES), + ]; + const result = await processToolResultOmniMedia( + parts, + cfg({ image: true }), + signal, + ); + expect(result[0]!.inlineData).toBeUndefined(); + expect(result[0]!.text).toMatch(/withheld by the omni transport guard/); + expect(result[0]!.text).toMatch(/exceeds the omni upload limit/); + expect(result[1]!.fileData?.fileUri).toBe('oss://bucket/key3'); + }); + it('keeps the part inline when staging-dir setup itself fails', async () => { // ~/.qwen/omni existing as a regular FILE makes mkdir fail with ENOTDIR. // That failure must degrade THIS part to inline like any other delivery diff --git a/packages/core/src/omni/tool-result-media.ts b/packages/core/src/omni/tool-result-media.ts index 16fba03ce3f..0b15915a3a0 100644 --- a/packages/core/src/omni/tool-result-media.ts +++ b/packages/core/src/omni/tool-result-media.ts @@ -11,6 +11,7 @@ import type { Part } from '@google/genai'; import type { Config } from '../config/config.js'; import { createDebugLogger } from '../utils/debugLogger.js'; import { isOmniDeliveryActive, processMediaForOmniDelivery } from './index.js'; +import { OmniTransportGuardError } from './guard.js'; import { OmniObjectStore } from './storage.js'; import { sniffMediaType } from './recognition.js'; @@ -38,7 +39,10 @@ const MAX_UPLOAD_BYTES_PER_TOOL_RESULT = 128 * 1024 * 1024; * - failure of any single part leaves that part inline (tool results were * produced locally and already fit in memory — degrading to the S1-era * inline behavior is safe here, unlike user-input delivery where inline - * silently violates the size contract; the failure is logged); + * silently violates the size contract; the failure is logged) — EXCEPT + * transport-guard rejections, which are policy verdicts rather than + * transfer failures: those parts are withheld with a text placeholder, + * never delivered inline (that would bypass the enabled guard); * - user aborts propagate. */ export async function processToolResultOmniMedia( @@ -109,6 +113,18 @@ export async function processToolResultOmniMedia( }; } catch (err) { if (signal.aborted) throw err; + if (err instanceof OmniTransportGuardError) { + // A guard rejection is a policy verdict, not a transfer failure — + // keeping the part inline would deliver the exact bytes the guard + // was configured to reject (at greater request cost than the + // upload). Withhold the media and say so; the inline-degradation + // rationale ("produced locally, already in memory") covers only + // failures of the *transfer*. + changed = true; + return { + text: `[Tool media part withheld by the omni transport guard: ${err.message}]`, + }; + } debugLogger.debug( `tool-result media upload failed, keeping inline: ${ err instanceof Error ? err.message : String(err) From e4daca68c8d92090489c36f4ed0e1479d43ab2a6 Mon Sep 17 00:00:00 2001 From: LaZzyMan Date: Thu, 6 Aug 2026 13:24:03 +0800 Subject: [PATCH 14/14] fix(omni): close review round-3 findings - download: resolve-phase watchdog, malformed-URL and proxy refusal, resolver-error translation, path scrubbing, cause-chain surfacing, DownloadedMedia slimmed to partPath; mutation-proven tests for redirect cap, both watchdogs, byte-cap streaming, abort wiring - recognition: direct recognizeMediaFile tests, GIF 6-byte signature, guarded probe-handle close; sha256 moved out of RecognizedMedia - pipeline: hash after guards (stat -> byte guard -> probe -> token guard -> hash); displayName override so URL-funnel errors name the remote file, not the staging path - fileUtils: omni gating suite incl. 100 MB image-cap bypass pin - dead S1 shims deleted (video-only probe/delivery/read wrappers) - URL funnel: post-download modality gate mirroring the local-file gate --- packages/acp-bridge/tsconfig.json | 1 - packages/acp-bridge/vitest.config.ts | 4 - packages/cli/src/config/settingsSchema.ts | 20 +- .../src/ui/hooks/atCommandProcessor.test.ts | 92 +++- .../cli/src/ui/hooks/atCommandProcessor.ts | 31 +- packages/cli/src/ui/utils/highlight.test.ts | 11 + packages/cli/src/ui/utils/highlight.ts | 7 +- .../openaiContentGenerator/converter.test.ts | 49 ++ .../core/openaiContentGenerator/converter.ts | 7 + packages/core/src/omni/download.test.ts | 483 +++++++++++++++++- packages/core/src/omni/download.ts | 283 ++++++++-- packages/core/src/omni/estimation.test.ts | 1 - packages/core/src/omni/estimation.ts | 10 +- packages/core/src/omni/ffmpeg.test.ts | 17 +- packages/core/src/omni/ffmpeg.ts | 21 - packages/core/src/omni/guard.test.ts | 15 +- packages/core/src/omni/index.test.ts | 132 ++++- packages/core/src/omni/index.ts | 77 ++- packages/core/src/omni/recognition.test.ts | 123 ++++- packages/core/src/omni/recognition.ts | 89 ++-- .../core/src/omni/tool-result-media.test.ts | 144 +++++- packages/core/src/omni/upload.ts | 2 +- packages/core/src/utils/fileUtils.test.ts | 134 +++++ packages/core/src/utils/fileUtils.ts | 3 +- .../schemas/settings.schema.json | 4 +- scripts/dev.js | 20 + 26 files changed, 1509 insertions(+), 271 deletions(-) diff --git a/packages/acp-bridge/tsconfig.json b/packages/acp-bridge/tsconfig.json index 38b6a3fc0b0..ce1a0e43a84 100644 --- a/packages/acp-bridge/tsconfig.json +++ b/packages/acp-bridge/tsconfig.json @@ -12,7 +12,6 @@ "@qwen-code/qwen-code-core/transcriptRecords": [ "../core/src/utils/transcript-records.ts" ], - "@qwen-code/qwen-code-core/omni": ["../core/src/omni/index.ts"], "@qwen-code/qwen-code-core/*": ["../core/src/*"] } }, diff --git a/packages/acp-bridge/vitest.config.ts b/packages/acp-bridge/vitest.config.ts index ac66bfb7778..48e2356ea40 100644 --- a/packages/acp-bridge/vitest.config.ts +++ b/packages/acp-bridge/vitest.config.ts @@ -18,10 +18,6 @@ export default defineConfig({ __dirname, '../core/src/utils/transcript-records.ts', ), - '@qwen-code/qwen-code-core/omni': path.resolve( - __dirname, - '../core/src/omni/index.ts', - ), }, }, test: { diff --git a/packages/cli/src/config/settingsSchema.ts b/packages/cli/src/config/settingsSchema.ts index 4d3963b15ce..ff66c534ba1 100644 --- a/packages/cli/src/config/settingsSchema.ts +++ b/packages/cli/src/config/settingsSchema.ts @@ -3641,12 +3641,13 @@ const SETTINGS_SCHEMA = { requiresRestart: true, default: false, description: - 'Enable the omni media pipeline. Local video files referenced ' + - 'with @ are recognized (ffprobe), stored content-addressed under ' + - '.qwen/omni/objects/, uploaded through the DashScope temporary ' + - 'upload channel, and delivered as oss:// URLs instead of inline ' + - 'base64. Only active for DashScope-compatible endpoints. Can ' + - 'also be enabled via QWEN_CODE_ENABLE_OMNI=1.', + 'Enable the omni media pipeline. Media files (video, image, ' + + 'audio) referenced with @ — and media served from @https:// ' + + 'URLs — are recognized (ffprobe), stored content-addressed ' + + 'under .qwen/omni/objects/, uploaded through the DashScope ' + + 'temporary upload channel, and delivered as oss:// URLs instead ' + + 'of inline base64. Only active for DashScope-compatible ' + + 'endpoints. Can also be enabled via QWEN_CODE_ENABLE_OMNI=1.', showInDialog: true, }, upload: { @@ -3697,10 +3698,9 @@ const SETTINGS_SCHEMA = { description: 'Estimated-token ceiling for a single omni media input, ' + 'checked before upload using the versioned raw-resource ' + - 'estimator. 0 disables the token guard (estimates are still ' + - 'attached for observability) — the estimation formula is ' + - 'pending confirmation with the model provider; set a ' + - 'positive threshold to enforce fail-closed rejection.', + 'estimator. 0 disables the token guard — the estimation ' + + 'formula is pending confirmation with the model provider; ' + + 'set a positive threshold to enforce fail-closed rejection.', showInDialog: false, jsonSchemaOverride: { type: 'number', diff --git a/packages/cli/src/ui/hooks/atCommandProcessor.test.ts b/packages/cli/src/ui/hooks/atCommandProcessor.test.ts index c3b8e05a56d..791164ba7f5 100644 --- a/packages/cli/src/ui/hooks/atCommandProcessor.test.ts +++ b/packages/cli/src/ui/hooks/atCommandProcessor.test.ts @@ -1746,6 +1746,7 @@ describe('handleAtCommand', () => { downloadMediaUrl: vi.fn(), processMediaForOmniDelivery: vi.fn(), effectiveMaxDownloadFileBytes: vi.fn(), + sniffFileModality: vi.fn(), })); vi.mock('@qwen-code/qwen-code-core/omni', () => ({ @@ -1757,10 +1758,18 @@ describe('handleAtCommand', () => { }, })); - function omniConfig(enabled: boolean): Config { + function omniConfig( + enabled: boolean, + modalities: Record = { + image: true, + audio: true, + video: true, + }, + ): Config { return { ...mockConfig, isOmniEnabled: () => enabled, + getContentGeneratorConfig: () => ({ modalities }), storage: { getQwenDir: () => path.join(os.tmpdir(), 'omni-at-test') }, } as unknown as Config; } @@ -1769,6 +1778,7 @@ describe('handleAtCommand', () => { omniMocks.isOmniDeliveryActive.mockReturnValue(true); omniMocks.parseHttpUrlRef.mockImplementation((url: string) => ({ url })); omniMocks.effectiveMaxDownloadFileBytes.mockReturnValue(1024 * 1024); + omniMocks.sniffFileModality.mockResolvedValue('video'); const partPath = path.join(testRootDir, 'downloaded.mp4'); await createTestFile(partPath, 'fake media bytes'); omniMocks.downloadMediaUrl.mockResolvedValue({ partPath }); @@ -1792,6 +1802,13 @@ describe('handleAtCommand', () => { expect(omniMocks.downloadMediaUrl).toHaveBeenCalledWith( expect.objectContaining({ url: 'https://example.com/clip.mp4' }), ); + // Guard/error messages inside the pipeline must name the URL's file, + // not the opaque staging path the download landed under. + expect(omniMocks.processMediaForOmniDelivery).toHaveBeenCalledWith( + expect.any(String), + expect.anything(), + expect.objectContaining({ displayName: 'clip.mp4' }), + ); const parts = result.processedQuery as Array>; expect(parts).toContainEqual({ fileData: { @@ -1843,5 +1860,78 @@ describe('handleAtCommand', () => { 'No valid file paths found in @ commands to read.', ); }); + + it('skips store/upload when the sniffed modality is not in the model modalities', async () => { + // Mirrors the local-file path's per-modality gate: media the active + // model cannot consume must not be uploaded only to be replaced by an + // unsupported-modality placeholder in the converter. + omniMocks.sniffFileModality.mockResolvedValue('image'); + const result = await handleAtCommand({ + query: 'look @https://example.com/pic.jpg please', + config: omniConfig(true, { image: false, audio: true, video: true }), + onDebugMessage: mockOnDebugMessage, + messageId: 703, + signal: abortController.signal, + }); + + expect(result.shouldProceed).toBe(true); + expect(omniMocks.processMediaForOmniDelivery).not.toHaveBeenCalled(); + const parts = result.processedQuery as Array>; + expect(parts.some((p) => 'fileData' in p)).toBe(false); + expect(result.toolDisplays![0]).toMatchObject({ + name: 'Fetch Media URL', + status: ToolCallStatus.Error, + }); + expect( + (result.toolDisplays![0] as { resultDisplay: string }).resultDisplay, + ).toContain('does not accept image input'); + }); + + it('isolates a failing URL: error card shown, remaining URLs still delivered', async () => { + omniMocks.downloadMediaUrl + .mockRejectedValueOnce(new Error('URL host did not resolve')) + .mockResolvedValueOnce({ + partPath: path.join(testRootDir, 'downloaded.mp4'), + }); + const result = await handleAtCommand({ + query: 'compare @https://bad.example/a.mp4 @https://good.example/b.mp4', + config: omniConfig(true), + onDebugMessage: mockOnDebugMessage, + messageId: 704, + signal: abortController.signal, + }); + + expect(result.shouldProceed).toBe(true); + expect(result.toolDisplays).toHaveLength(2); + expect(result.toolDisplays![0]).toMatchObject({ + status: ToolCallStatus.Error, + }); + expect( + (result.toolDisplays![0] as { resultDisplay: string }).resultDisplay, + ).toContain('bad.example'); + expect(result.toolDisplays![1]).toMatchObject({ + status: ToolCallStatus.Success, + }); + const parts = result.processedQuery as Array>; + expect(parts.filter((p) => 'fileData' in p)).toHaveLength(1); + }); + + it('ends the turn quietly (shouldProceed=false) on a user abort mid-download', async () => { + omniMocks.downloadMediaUrl.mockImplementation(async () => { + abortController.abort(); + throw new Error('aborted'); + }); + const result = await handleAtCommand({ + query: 'summarize @https://example.com/clip.mp4', + config: omniConfig(true), + onDebugMessage: mockOnDebugMessage, + messageId: 705, + signal: abortController.signal, + }); + + expect(result.shouldProceed).toBe(false); + expect(result.processedQuery).toBeNull(); + expect(omniMocks.processMediaForOmniDelivery).not.toHaveBeenCalled(); + }); }); }); diff --git a/packages/cli/src/ui/hooks/atCommandProcessor.ts b/packages/cli/src/ui/hooks/atCommandProcessor.ts index 6ff9259ef3a..8aca3981fc9 100644 --- a/packages/cli/src/ui/hooks/atCommandProcessor.ts +++ b/packages/cli/src/ui/hooks/atCommandProcessor.ts @@ -590,13 +590,38 @@ export async function resolveAtCommandQuery({ // Full local-file pipeline on the downloaded bytes: sniff/probe/ // hash again (the design requires re-recognition from the local // file — never trust transfer-time observations), guard, store, - // upload. readMediaViaOmniDelivery needs a stable name for - // display; use the URL basename. + // upload. Displays and error messages need a stable name; use the + // URL basename. const urlBase = path.basename(new URL(ref.url).pathname) || hostLabel; + // Per-modality gate, mirroring the local-file path in fileUtils: + // media the active model cannot consume must not be stored and + // uploaded — the fileData part would only be replaced by an + // unsupported-modality placeholder in the converter, after paying + // for the upload. (The download itself is unavoidable: modality + // is only knowable from the bytes.) A null sniff falls through to + // the pipeline, whose recognition failure names the problem. + const modalities = + config.getContentGeneratorConfig?.()?.modalities ?? {}; + const sniffedModality = await core.sniffFileModality( + downloaded.partPath, + ); + if (sniffedModality && !modalities[sniffedModality]) { + urlMediaDisplays.push({ + callId, + name: 'Fetch Media URL', + description: `Download ${ref.url}`, + status: ToolCallStatus.Error, + resultDisplay: `Skipped ${urlBase}: the active model does not accept ${sniffedModality} input.`, + confirmationDetails: undefined, + }); + continue; + } const delivery = await core.processMediaForOmniDelivery( downloaded.partPath, config, - { signal }, + // displayName: guard/error messages must name the URL's file, not + // the opaque staging path the download landed under. + { signal, displayName: urlBase }, ); urlMediaParts.push({ fileData: { diff --git a/packages/cli/src/ui/utils/highlight.test.ts b/packages/cli/src/ui/utils/highlight.test.ts index e50c0c50c72..1b125ed1282 100644 --- a/packages/cli/src/ui/utils/highlight.test.ts +++ b/packages/cli/src/ui/utils/highlight.test.ts @@ -206,6 +206,17 @@ describe('parseInputForHighlighting', () => { ]); }); + it('should paint a URL ref with a backslash-escaped span in full', () => { + // The parser's escape branch runs for URL refs too, so the highlighter + // must not split the token at the backslash. + const text = 'play @https://host/my\\ video.mp4 now'; + expect(parseInputForHighlighting(text, 0)).toEqual([ + { text: 'play ', type: 'default' }, + { text: '@https://host/my\\ video.mp4', type: 'file' }, + { text: ' now', type: 'default' }, + ]); + }); + it('should only highlight valid slash commands when command metadata is provided', () => { const text = '/help please /review this /clear and /missing plus /usr/bin'; expect(parseInputForHighlighting(text, 0, slashCommands)).toEqual([ diff --git a/packages/cli/src/ui/utils/highlight.ts b/packages/cli/src/ui/utils/highlight.ts index 5ffe35e0abc..c9592b04c7e 100644 --- a/packages/cli/src/ui/utils/highlight.ts +++ b/packages/cli/src/ui/utils/highlight.ts @@ -17,9 +17,12 @@ export type HighlightToken = { // hooks/atCommandProcessor.ts, or the input box paints a different span than // the parser will actually consume. A URL ref (`@https://…`) runs through the // full RFC 3986 charset — `%`, `?`, `&`, `=` are structural in a presigned -// URL — while a filesystem ref keeps the narrower path charset. +// URL — while a filesystem ref keeps the narrower path charset. The `\\ ` +// alternative and the `\` in the class mirror the parser's backslash-escape +// branch, which runs for URL refs too — an escaped span must stay painted +// rather than split the token. const HIGHLIGHT_REGEX = - /(^\/[a-zA-Z][a-zA-Z0-9:_-]*)|((?<=\s)\/[a-zA-Z][a-zA-Z0-9:_-]*)|(@[hH][tT][tT][pP][sS]?:\/\/[A-Za-z0-9\-._~:/?#[\]@!$&'()*+,;=%]+)|(@(?:\\ |[a-zA-Z0-9_.:/-])+)/g; + /(^\/[a-zA-Z][a-zA-Z0-9:_-]*)|((?<=\s)\/[a-zA-Z][a-zA-Z0-9:_-]*)|(@[hH][tT][tT][pP][sS]?:\/\/(?:\\ |[A-Za-z0-9\-._~:/?#[\]@!$&'()*+,;=%\\])+)|(@(?:\\ |[a-zA-Z0-9_.:/-])+)/g; // Mirrors the parser's trailing-punctuation trim: '.'/','/';'/':'/'!'/'?' at // the very end of a URL are prose, not part of the ref. diff --git a/packages/core/src/core/openaiContentGenerator/converter.test.ts b/packages/core/src/core/openaiContentGenerator/converter.test.ts index 51b223bf100..f2168cf8c07 100644 --- a/packages/core/src/core/openaiContentGenerator/converter.test.ts +++ b/packages/core/src/core/openaiContentGenerator/converter.test.ts @@ -2468,6 +2468,55 @@ describe('OpenAIContentConverter', () => { } }); + it('should convert inline flac/ogg/m4a audio instead of textifying', () => { + // Mirrors the fileData case above: the inline branch shares + // getAudioFormat, and a regression that re-textifies inline + // flac/ogg/m4a must not ship green. + for (const [mime, format] of [ + ['audio/flac', 'flac'], + ['audio/ogg', 'ogg'], + ['audio/mp4', 'm4a'], + ] as const) { + const request: GenerateContentParameters = { + model: 'models/test', + contents: [ + { + role: 'user', + parts: [ + { + inlineData: { + mimeType: mime, + data: 'YXVkaW8=', + displayName: 'clip', + }, + }, + ], + }, + ], + }; + const messages = converter.convertGeminiRequestToOpenAI( + request, + requestContext, + ); + const userMessage = messages.find((m) => m.role === 'user'); + const contentArray = userMessage?.content as Array<{ + type: string; + input_audio?: { data: string; format: string }; + text?: string; + }>; + const audioPart = contentArray.find((p) => p.type === 'input_audio'); + expect(audioPart?.input_audio?.format).toBe(format); + expect(audioPart?.input_audio?.data).toBe( + `data:${mime};base64,YXVkaW8=`, + ); + expect( + contentArray.some((p) => + p.text?.includes('Unsupported inline media type'), + ), + ).toBe(false); + } + }); + it('should render unsupported inlineData file types as a text block', () => { const request: GenerateContentParameters = { model: 'models/test', diff --git a/packages/core/src/core/openaiContentGenerator/converter.ts b/packages/core/src/core/openaiContentGenerator/converter.ts index 5b69f6c24f7..d8dd2dcd038 100644 --- a/packages/core/src/core/openaiContentGenerator/converter.ts +++ b/packages/core/src/core/openaiContentGenerator/converter.ts @@ -1075,6 +1075,13 @@ function getMediaType(mimeType: string): 'image' | 'audio' | 'video' | 'file' { * Audio formats the DashScope input_audio channel accepts. Kept in * lockstep with the omni recognizer's audio sniff set — an upload the * recognizer accepts must never textify here after paying for transfer. + * + * flac/ogg/m4a are a DashScope acceptance extension beyond the OpenAI + * SDK's wav|mp3 union, applied unconditionally because RequestContext + * carries no provider identity to scope on. The trade is deliberate: + * a non-DashScope endpoint that rejects the format returns an explicit + * 400, whereas textifying would silently drop the audio. Revisit if a + * provider flag ever lands on RequestContext. */ function getAudioFormat( mimeType: string, diff --git a/packages/core/src/omni/download.test.ts b/packages/core/src/omni/download.test.ts index 9d62e6356bc..d880c7c3b2c 100644 --- a/packages/core/src/omni/download.test.ts +++ b/packages/core/src/omni/download.test.ts @@ -5,13 +5,17 @@ */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { createHash } from 'node:crypto'; import fs from 'node:fs/promises'; import net from 'node:net'; import type { AddressInfo, LookupFunction } from 'node:net'; import os from 'node:os'; import path from 'node:path'; -import { downloadMediaUrl, OmniDownloadError } from './download.js'; +import { + downloadMediaUrl, + parseHttpUrlRef, + OmniDownloadError, + MAX_REDIRECTS, +} from './download.js'; // Keep the suite hermetic: the SSRF gate resolves hostnames, and tests that do // not inject a fake target must not depend on (or wait for) real DNS. @@ -68,13 +72,51 @@ async function pinnedAddressOf(init: RequestInit | undefined): Promise { }); } +/** + * A pull-counting body: `bytesPulled()` reports how much the consumer has + * actually drawn from the stream, which is what distinguishes a streaming + * implementation (stops pulling once the verdict is known) from one that + * buffers the whole body first. Counted in bytes, not pulls — Node's + * Response plumbing pulls a chunk or two on its own. + */ +function countingBody(chunkSize: number, chunkCount: number) { + let pulled = 0; + let index = 0; + const stream = new ReadableStream({ + pull(controller) { + if (index >= chunkCount) { + controller.close(); + return; + } + index += 1; + pulled += chunkSize; + controller.enqueue(new Uint8Array(chunkSize)); + }, + }); + return { stream, bytesPulled: () => pulled }; +} + let downloadsDir: string; beforeEach(async () => { downloadsDir = await fs.mkdtemp(path.join(os.tmpdir(), 'omni-dl-')); + // The proxy gate reads the process environment when no detectProxy is + // injected; a developer machine with HTTP(S)_PROXY set must not flip the + // verdict for every test in this file. + for (const key of [ + 'https_proxy', + 'HTTPS_PROXY', + 'http_proxy', + 'HTTP_PROXY', + 'all_proxy', + 'ALL_PROXY', + ]) { + vi.stubEnv(key, ''); + } }); afterEach(async () => { + vi.unstubAllEnvs(); await fs.rm(downloadsDir, { recursive: true, force: true }); }); @@ -89,8 +131,33 @@ async function listParts(): Promise { return (await fs.readdir(downloadsDir)).filter((f) => f.endsWith('.part')); } +describe('parseHttpUrlRef', () => { + it('accepts http and https refs, scheme case-insensitively', () => { + expect(parseHttpUrlRef('https://example.com/a.mp4')?.hostname).toBe( + 'example.com', + ); + expect(parseHttpUrlRef('http://example.com/a.mp4')?.protocol).toBe('http:'); + expect(parseHttpUrlRef('HTTPS://EXAMPLE.COM/A.MP4')).toBeInstanceOf(URL); + expect(parseHttpUrlRef('HtTp://example.com/a.mp4')).toBeInstanceOf(URL); + }); + + it('returns null for non-URL paths and unparseable URLs', () => { + for (const ref of [ + 'src/media/clip.mp4', + './clip.mp4', + 'clip.mp4', + 'ftp://example.com/clip.mp4', + 'httpx://example.com/clip.mp4', + 'https://', + 'https://[', + ]) { + expect(parseHttpUrlRef(ref)).toBeNull(); + } + }); +}); + describe('downloadMediaUrl', () => { - it('streams to a .part file and hashes while writing', async () => { + it('streams the body to a .part file inside downloads/', async () => { const bytes = Buffer.from('media-bytes-here'); const result = await downloadMediaUrl({ url: 'https://media.example.com/a.mp4', @@ -98,14 +165,30 @@ describe('downloadMediaUrl', () => { maxBytes: 1_000_000, fetchFn: fetchOk(bytes, { 'content-type': 'video/mp4' }), }); - expect(result.sha256).toBe( - createHash('sha256').update(bytes).digest('hex'), - ); - expect(result.sizeBytes).toBe(bytes.length); - expect(result.contentType).toBe('video/mp4'); + expect(path.dirname(result.partPath)).toBe(downloadsDir); + expect(result.partPath.endsWith('.part')).toBe(true); await expect(fs.readFile(result.partPath)).resolves.toEqual(bytes); }); + it('rejects a malformed URL as a named download error, not a TypeError', async () => { + // isPrivateHost fails open on unparseable input, so the re-parse inside + // downloadMediaUrl is where a malformed ref surfaces — it must do so as + // the documented error type, without echoing the ref. + const fetchFn = vi.fn(); + for (const url of ['https://exa mple.com/a.mp4', 'https://[']) { + const err = await downloadMediaUrl({ + url, + downloadsDir, + maxBytes: 1000, + fetchFn, + }).catch((e: Error) => e); + expect(err).toBeInstanceOf(OmniDownloadError); + expect((err as Error).message).toMatch(/not a valid URL/); + } + expect(fetchFn).not.toHaveBeenCalled(); + expect(await listParts()).toEqual([]); + }); + it('refuses private/loopback hosts (SSRF)', async () => { const fetchFn = vi.fn(); for (const url of [ @@ -128,14 +211,17 @@ describe('downloadMediaUrl', () => { // text-level gate passes; only resolution reveals the target. for (const address of ['127.0.0.1', '169.254.169.254', '10.1.2.3']) { dnsLookupMock.mockResolvedValueOnce([{ address, family: 4 }]); - await expect( - downloadMediaUrl({ - url: 'https://media.example.com/clip.mp4', - downloadsDir, - maxBytes: 1_000_000, - fetchFn, - }), - ).rejects.toThrow(/refused for safety/); + const err = await downloadMediaUrl({ + url: 'https://media.example.com/clip.mp4', + downloadsDir, + maxBytes: 1_000_000, + fetchFn, + }).catch((e: Error) => e); + expect(err).toBeInstanceOf(OmniDownloadError); + expect((err as Error).message).toMatch(/refused for safety/); + // The resolver phrases its errors for its original caller ("Extension + // network host …"); that misattribution must not reach download errors. + expect((err as Error).message).not.toMatch(/Extension/); } expect(fetchFn).not.toHaveBeenCalled(); expect(await listParts()).toEqual([]); @@ -174,7 +260,7 @@ describe('downloadMediaUrl', () => { maxBytes: 1_000_000, fetchFn: fetchOk(bytes), }); - expect(result.sizeBytes).toBe(bytes.length); + await expect(fs.readFile(result.partPath)).resolves.toEqual(bytes); }); it('refuses when ANY resolved address is private (mixed A records)', async () => { @@ -215,6 +301,77 @@ describe('downloadMediaUrl', () => { expect(await listParts()).toEqual([]); }); + it('translates resolver errors into download-domain phrasing', async () => { + // resolveNetworkTarget phrases its errors for its original caller + // ("Extension network host …"). Passing that through verbatim would + // misattribute the refusal — the same problem the credentials gate + // already avoids — so the reachable resolver errors are translated, and + // nothing beyond the hostname is echoed. + const cases: Array<[string, RegExp]> = [ + [ + 'Extension network host resolved to a blocked address: media.example.com', + /URL host resolved to a non-public address/, + ], + [ + 'Extension network host did not resolve: media.example.com', + /URL host did not resolve/, + ], + ['socket hang up', /URL host could not be verified/], + ]; + for (const [resolverText, expected] of cases) { + const err = await downloadMediaUrl({ + url: 'https://media.example.com/a.mp4', + downloadsDir, + maxBytes: 1000, + fetchFn: vi.fn(), + resolveTarget: async () => { + throw new Error(resolverText); + }, + }).catch((e: Error) => e); + expect(err).toBeInstanceOf(OmniDownloadError); + expect((err as Error).message).toMatch(expected); + expect((err as Error).message).not.toMatch(/Extension/); + } + }); + + it('times out a resolve step that never answers', async () => { + // Resolution is otherwise the only phase with no watchdog: a resolver + // that never settles (and a caller that never aborts) would spin the + // turn forever. + const fetchFn = vi.fn(); + await expect( + downloadMediaUrl({ + url: 'https://media.example.com/a.mp4', + downloadsDir, + maxBytes: 1000, + fetchFn, + resolveTarget: () => new Promise(() => {}), + resolveTimeoutMs: 50, + }), + ).rejects.toThrow(/timed out resolving media\.example\.com/); + expect(fetchFn).not.toHaveBeenCalled(); + expect(await listParts()).toEqual([]); + }); + + it('propagates a user abort during resolve as an abort, not a timeout', async () => { + const controller = new AbortController(); + const err = await downloadMediaUrl({ + url: 'https://media.example.com/a.mp4', + downloadsDir, + maxBytes: 1000, + signal: controller.signal, + fetchFn: vi.fn(), + resolveTarget: () => + new Promise((_, reject) => { + controller.abort(); + reject(Object.assign(new Error('aborted'), { name: 'AbortError' })); + }), + resolveTimeoutMs: 50, + }).catch((e: Error) => e); + expect((err as Error).name).toBe('AbortError'); + expect(err).not.toBeInstanceOf(OmniDownloadError); + }); + it('refuses a target that cannot be pinned to a vetted address', async () => { // If the resolve step yields no pinned lookup, the connection cannot be // bound, so claiming rebinding protection would be false. Refuse instead. @@ -252,7 +409,72 @@ describe('downloadMediaUrl', () => { fetchFn: fetchOk(Buffer.from('ok-bytes')), resolveTarget: async (u) => pinnedTarget(u), }); - expect(result.sizeBytes).toBe(8); + await expect(fs.readFile(result.partPath)).resolves.toEqual( + Buffer.from('ok-bytes'), + ); + }); + + it('refuses to download when a proxy is configured (pin unenforceable)', async () => { + // Same doctrine as the Bun refusal: the bare pinned Agent dispatches + // directly, silently bypassing a configured proxy — while routing + // through the proxy would hand resolution to its CONNECT handling, + // where the pinned lookup never runs. Either way, refuse. + const fetchFn = vi.fn(); + await expect( + downloadMediaUrl({ + url: 'https://media.example.com/a.mp4', + downloadsDir, + maxBytes: 1000, + fetchFn, + detectProxy: () => true, + resolveTarget: async (u) => pinnedTarget(u), + }), + ).rejects.toThrow(/not supported behind a proxy/); + expect(fetchFn).not.toHaveBeenCalled(); + expect(await listParts()).toEqual([]); + }); + + it('detects a configured proxy from the environment', async () => { + // Production detection path: no detectProxy injected, HTTPS_PROXY set — + // the same signal EnvHttpProxyAgent (and config.ts) honors. + vi.stubEnv('HTTPS_PROXY', 'http://proxy.corp.example:8080'); + const fetchFn = vi.fn(); + await expect( + downloadMediaUrl({ + url: 'https://media.example.com/a.mp4', + downloadsDir, + maxBytes: 1000, + fetchFn, + resolveTarget: async (u) => pinnedTarget(u), + }), + ).rejects.toThrow(/not supported behind a proxy/); + expect(fetchFn).not.toHaveBeenCalled(); + }); + + it('wraps downloads-directory failures without leaking the absolute path', async () => { + // fs.mkdir error text embeds the absolute directory path; the wrap must + // keep the failure inside the OmniDownloadError contract and scrub the + // path (messages reach the UI and the debug log). + const blockerFile = path.join(downloadsDir, 'blocker'); + await fs.writeFile(blockerFile, 'not a directory'); + const badDir = path.join(blockerFile, 'nested'); + const fetchFn = vi.fn(); + const err = await downloadMediaUrl({ + url: 'https://media.example.com/a.mp4', + downloadsDir: badDir, + maxBytes: 1000, + fetchFn, + resolveTarget: async (u) => pinnedTarget(u), + }).catch((e: Error) => e); + expect(err).toBeInstanceOf(OmniDownloadError); + expect((err as Error).message).toMatch( + /Could not prepare the downloads directory/, + ); + expect((err as Error).message).not.toContain(downloadsDir); + expect((err as Error).message).not.toMatch( + /\/home|\/Users|\/private|\/tmp\//, + ); + expect(fetchFn).not.toHaveBeenCalled(); }); it('pins the connection to the vetted address (real socket)', async () => { @@ -291,6 +513,31 @@ describe('downloadMediaUrl', () => { } }); + it('surfaces the cause chain of connection-level fetch failures', async () => { + // Undici reports connection failures as a bare `TypeError: fetch failed` + // with the actionable reason (ECONNREFUSED, TLS codes) on `cause`; the + // wrap must surface it or the user has nothing to act on. + const fetchFn = vi.fn(async () => { + throw new TypeError('fetch failed', { + cause: Object.assign(new Error('connect ECONNREFUSED 127.0.0.1:1'), { + code: 'ECONNREFUSED', + }), + }); + }) as unknown as typeof fetch; + const err = await downloadMediaUrl({ + url: 'https://media.example.com/a.mp4', + downloadsDir, + maxBytes: 1000, + fetchFn, + resolveTarget: async (u) => pinnedTarget(u), + }).catch((e: Error) => e); + expect(err).toBeInstanceOf(OmniDownloadError); + expect((err as Error).message).toMatch(/ECONNREFUSED/); + expect((err as Error & { cause?: unknown }).cause).toBeInstanceOf( + TypeError, + ); + }); + it('re-resolves and re-pins at every redirect hop', async () => { // Hop 1 resolves public; the same-host redirect target resolves private // (rebinding between hops) and must be refused before the second fetch. @@ -423,7 +670,9 @@ describe('downloadMediaUrl', () => { maxBytes: 1_000_000, fetchFn: fetchOk(Buffer.from('ok')), }), - ).resolves.toMatchObject({ sizeBytes: 2 }); + ).resolves.toMatchObject({ + partPath: expect.stringContaining('.part'), + }); expect(dnsLookupMock).toHaveBeenCalledWith('media.example.com', { all: true, verbatim: true, @@ -490,6 +739,33 @@ describe('downloadMediaUrl', () => { expect(await listParts()).toEqual([]); }); + it('does not read the body when Content-Length already exceeds the cap', async () => { + // The pre-check must reject on the header alone, not after buffering the + // body: assert on bytes actually pulled from the stream. (Node's Response + // plumbing pulls a chunk or two by itself, so the bound is "far less + // than the body", not zero.) + const chunkSize = 8 * 1024; + const chunkCount = 100; + const { stream, bytesPulled } = countingBody(chunkSize, chunkCount); + const fetchFn = vi.fn( + async () => + new Response(stream, { + status: 200, + headers: { 'content-length': String(chunkSize * chunkCount) }, + }), + ) as unknown as typeof fetch; + await expect( + downloadMediaUrl({ + url: 'https://m.example.com/big.mp4', + downloadsDir, + maxBytes: 64 * 1024, + fetchFn, + }), + ).rejects.toThrow(/Content-Length 819200 > 65536 bytes/); + expect(bytesPulled()).toBeLessThan(chunkSize * chunkCount); + expect(await listParts()).toEqual([]); + }); + it('enforces the byte cap on actual bytes even without Content-Length', async () => { // Streamed response with no content-length header. const stream = new ReadableStream({ @@ -513,6 +789,28 @@ describe('downloadMediaUrl', () => { expect(await listParts()).toEqual([]); // .part cleaned on failure }); + it('stops pulling once the streamed byte cap trips (does not buffer the body)', async () => { + // A buffering implementation that reads the whole body and THEN rejects + // would pass the cap test above; asserting bytes pulled < body size is + // what pins the streaming behavior. + const chunkSize = 8 * 1024; + const chunkCount = 100; + const { stream, bytesPulled } = countingBody(chunkSize, chunkCount); + const fetchFn = vi.fn( + async () => new Response(stream, { status: 200 }), + ) as unknown as typeof fetch; + await expect( + downloadMediaUrl({ + url: 'https://m.example.com/nolen.mp4', + downloadsDir, + maxBytes: 64 * 1024, + fetchFn, + }), + ).rejects.toThrow(/>65536 bytes received/); + expect(bytesPulled()).toBeLessThan(chunkSize * chunkCount); + expect(await listParts()).toEqual([]); + }); + it('surfaces HTTP errors with status and host, cleaning the .part', async () => { await expect( downloadMediaUrl({ @@ -525,25 +823,37 @@ describe('downloadMediaUrl', () => { expect(await listParts()).toEqual([]); }); - it('follows same-origin redirects and refuses cross-origin ones', async () => { + it('follows same-origin redirects with manual redirect handling', async () => { + // `redirect: 'manual'` is load-bearing: without it, production undici + // auto-follows and every per-hop re-vet (re-resolve, re-pin, origin + // check) is silently skipped — so assert it inside the fetch mock. const bytes = Buffer.from('after-redirect'); + const redirects: Array = []; const sameOrigin = vi .fn() - .mockResolvedValueOnce( - new Response(null, { + .mockImplementationOnce(async (_u, init) => { + redirects.push(init?.redirect); + return new Response(null, { status: 302, headers: { location: 'https://m.example.com/real.mp4' }, - }), - ) - .mockResolvedValueOnce(new Response(bytes, { status: 200 })); + }); + }) + .mockImplementationOnce(async (_u, init) => { + redirects.push(init?.redirect); + return new Response(bytes, { status: 200 }); + }); const ok = await downloadMediaUrl({ url: 'https://m.example.com/a.mp4', downloadsDir, maxBytes: 1000, fetchFn: sameOrigin, }); - expect(ok.finalUrl).toBe('https://m.example.com/real.mp4'); - expect(ok.sizeBytes).toBe(bytes.length); + expect(redirects).toEqual(['manual', 'manual']); + expect(sameOrigin).toHaveBeenLastCalledWith( + 'https://m.example.com/real.mp4', + expect.anything(), + ); + await expect(fs.readFile(ok.partPath)).resolves.toEqual(bytes); const crossOrigin = vi.fn( async () => @@ -562,6 +872,28 @@ describe('downloadMediaUrl', () => { ).rejects.toThrow(/Cross-origin redirect refused/); }); + it('gives up after MAX_REDIRECTS same-origin hops', async () => { + // An unbounded loop here is a self-inflicted infinite redirect chase; + // pin both the refusal and the exact fetch budget. + const fetchFn = vi.fn( + async () => + new Response(null, { + status: 302, + headers: { location: 'https://m.example.com/next.mp4' }, + }), + ) as unknown as typeof fetch; + await expect( + downloadMediaUrl({ + url: 'https://m.example.com/a.mp4', + downloadsDir, + maxBytes: 1000, + fetchFn, + }), + ).rejects.toThrow(/Too many or invalid redirects/); + expect(fetchFn).toHaveBeenCalledTimes(MAX_REDIRECTS + 1); + expect(await listParts()).toEqual([]); + }); + it('surfaces a malformed redirect Location as a named download error', async () => { // `new URL('http://[', base)` throws a raw TypeError; that must become an // OmniDownloadError (and not echo the server-controlled header value). @@ -583,6 +915,62 @@ describe('downloadMediaUrl', () => { expect(await listParts()).toEqual([]); }); + it('times out when response headers never arrive', async () => { + // The mock only rejects when the signal handed to it aborts, so this + // test also pins the `signal` wiring in the fetch init: drop it and the + // promise never settles (the test itself times out) instead of naming + // the watchdog. + const fetchFn = vi.fn( + (_u: unknown, init?: RequestInit) => + new Promise((_, reject) => { + init?.signal?.addEventListener( + 'abort', + () => + reject( + Object.assign(new Error('aborted'), { name: 'AbortError' }), + ), + { once: true }, + ); + }), + ) as unknown as typeof fetch; + await expect( + downloadMediaUrl({ + url: 'https://m.example.com/slow.mp4', + downloadsDir, + maxBytes: 1000, + fetchFn, + headerTimeoutMs: 50, + }), + ).rejects.toThrow( + /timed out waiting for response headers from m\.example\.com/, + ); + expect(await listParts()).toEqual([]); + }); + + it('declares a stalled body dead after the idle window and cleans up', async () => { + // First chunk arrives, then nothing — the idle watchdog (not the header + // timer, already cleared) must abort the stream with a named message. + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array(8)); + // Never enqueues again, never closes. + }, + }); + const fetchFn = vi.fn( + async () => new Response(stream, { status: 200 }), + ) as unknown as typeof fetch; + await expect( + downloadMediaUrl({ + url: 'https://m.example.com/stall.mp4', + downloadsDir, + maxBytes: 1000, + fetchFn, + idleTimeoutMs: 50, + }), + ).rejects.toThrow(/Download stalled from m\.example\.com/); + expect(await listParts()).toEqual([]); + }); + it('propagates user aborts and cleans up', async () => { const controller = new AbortController(); const fetchFn = vi.fn(async () => { @@ -600,4 +988,45 @@ describe('downloadMediaUrl', () => { expect(err).not.toBeInstanceOf(OmniDownloadError); expect(await listParts()).toEqual([]); }); + + it('stops a mid-stream download when the caller aborts', async () => { + // Unlike the test above, the fetch mock never throws on its own: the + // abort can only take effect through the {signal} handed to + // Readable.fromWeb. Remove that wiring and the body streams to + // completion, resolving instead of rejecting. + let pulls = 0; + let index = 0; + const stream = new ReadableStream({ + async pull(controller) { + pulls += 1; + await new Promise((r) => setTimeout(r, 15)); + if (index >= 40) { + controller.close(); + return; + } + index += 1; + controller.enqueue(new Uint8Array(1024)); + }, + }); + const fetchFn = vi.fn( + async () => new Response(stream, { status: 200 }), + ) as unknown as typeof fetch; + const controller = new AbortController(); + setTimeout(() => controller.abort(), 60); + const err = await downloadMediaUrl({ + url: 'https://m.example.com/slowbody.mp4', + downloadsDir, + maxBytes: 1_000_000, + signal: controller.signal, + fetchFn, + }).catch((e: Error) => e); + expect((err as Error).name).toBe('AbortError'); + expect(err).not.toBeInstanceOf(OmniDownloadError); + // No further chunks are drawn once the abort lands (one in-flight pull + // may still complete). + const pullsAtRejection = pulls; + await new Promise((r) => setTimeout(r, 100)); + expect(pulls).toBeLessThanOrEqual(pullsAtRejection + 1); + expect(await listParts()).toEqual([]); + }); }); diff --git a/packages/core/src/omni/download.ts b/packages/core/src/omni/download.ts index 0460f938258..2fabf1fdb1e 100644 --- a/packages/core/src/omni/download.ts +++ b/packages/core/src/omni/download.ts @@ -4,7 +4,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { createHash, randomBytes } from 'node:crypto'; +import { randomBytes } from 'node:crypto'; import { createWriteStream } from 'node:fs'; import fs from 'node:fs/promises'; import path from 'node:path'; @@ -21,31 +21,36 @@ import { createDebugLogger } from '../utils/debugLogger.js'; const debugLogger = createDebugLogger('omni:download'); /** Thrown for URL localization failures. Messages must stay free of - * credentials; they may include the URL host and HTTP status. */ + * credentials and local filesystem paths; they may include the URL host and + * HTTP status. */ export class OmniDownloadError extends Error { - constructor(message: string) { - super(message); + constructor(message: string, options?: ErrorOptions) { + super(message, options); this.name = 'OmniDownloadError'; } } export interface DownloadedMedia { /** Path of the fully-written temp file (inside downloads/). Caller is - * responsible for recognition, promotion into objects/, and cleanup. */ + * responsible for recognition, promotion into objects/, and cleanup. This + * is deliberately the ONLY field: the pipeline re-derives identity (hash, + * sniff, size) from the file on disk — transfer-time observations such as + * the server's Content-Type or the final URL must never be trusted, so + * they are not carried here at all. */ partPath: string; - /** SHA-256 computed while streaming (identity of the downloaded bytes). */ - sha256: string; - /** Total bytes written. */ - sizeBytes: number; - /** Final URL after any permitted redirects. */ - finalUrl: string; - /** Content-Type reported by the server (advisory only — sniff decides). */ - contentType?: string; } +/** How long the DNS resolve-and-pin step may take before the download is + * declared stuck. Without it, resolution would be the only phase with no + * watchdog: a resolver that never answers (and a caller that never aborts) + * would spin the turn forever. The race abandons rather than cancels the + * underlying lookup — it only stops waiting for it. */ +const RESOLVE_TIMEOUT_MS = 30_000; const HEADER_TIMEOUT_MS = 30_000; const IDLE_TIMEOUT_MS = 60_000; -const MAX_REDIRECTS = 5; +/** Exported so the redirect-budget test can pin the exact fetch count to + * this constant rather than to a magic number that drifts. */ +export const MAX_REDIRECTS = 5; /** * Test whether an @-path is a downloadable http(s) URL. @@ -65,7 +70,79 @@ export function parseHttpUrlRef(pathName: string): URL | null { } /** - * Stream a media URL into `downloads/.part`, hashing while writing. + * Strip absolute filesystem paths from a message before it enters an + * OmniDownloadError, keeping only the basename. fs and stream errors embed + * full local paths (`ENOTDIR: not a directory, mkdir '/home/x/…'`), and + * OmniDownloadError messages reach the UI and the debug log. The wrap sites + * below only interpolate fs/undici error text — never URLs — so a + * separator-based pattern cannot eat anything it should keep. + */ +function scrubPathsFromMessage(message: string): string { + return message.replace( + /(?:[A-Za-z]:)?(?:[\\/][^\s'"`\\/]+){2,}[\\/]?/g, + (match) => path.basename(match), + ); +} + +/** + * Flatten an error's `cause` chain into "CODE: message" fragments. Undici + * reports connection-level failures as a bare `TypeError: fetch failed` with + * the actionable reason (ECONNREFUSED, TLS codes) on `cause`; without this + * the user would see "fetch failed" and nothing they can act on. + */ +function formatCauseChain(err: unknown): string | undefined { + const parts: string[] = []; + let cause: unknown = err instanceof Error ? err.cause : undefined; + for (let depth = 0; cause != null && depth < 4; depth++) { + const code = (cause as { code?: unknown }).code; + const message = + cause instanceof Error + ? cause.message + : typeof cause === 'string' + ? cause + : undefined; + if (typeof code === 'string' && message && !message.includes(code)) { + parts.push(`${code}: ${message}`); + } else if (message) { + parts.push(message); + } else if (typeof code === 'string') { + parts.push(code); + } + cause = cause instanceof Error ? cause.cause : undefined; + } + return parts.length > 0 ? scrubPathsFromMessage(parts.join('; ')) : undefined; +} + +/** + * Detect a configured HTTP(S) proxy. Mirrors how the rest of the codebase + * decides to proxy: config.ts installs a process-wide `EnvHttpProxyAgent` as + * the global dispatcher when `--proxy`/`settings.proxy` resolves to a value + * (see Config.initialize), and `EnvHttpProxyAgent` itself honors the + * `HTTP(S)_PROXY` environment variables — so a proxy is in play if either + * the env vars are set or that global dispatcher has been installed. + */ +function detectConfiguredProxy( + undici: Awaited>, +): boolean { + for (const key of [ + 'https_proxy', + 'HTTPS_PROXY', + 'http_proxy', + 'HTTP_PROXY', + 'all_proxy', + 'ALL_PROXY', + ]) { + if (process.env[key]) return true; + } + try { + return undici.getGlobalDispatcher() instanceof undici.EnvHttpProxyAgent; + } catch { + return false; + } +} + +/** + * Stream a media URL into `downloads/.part`. * * Policy (mirrors utils/fetch.ts fetchWithPolicy, but streaming — that * helper buffers whole bodies in memory and is unsuitable for GiB media): @@ -84,12 +161,18 @@ export function parseHttpUrlRef(pathName: string): URL | null { * redirect can still re-resolve elsewhere between hops); * - Node only: refused outright on runtimes that accept the pinning agent and * ignore it (Bun), since there the pin is unenforceable; + * - refused outright behind a configured proxy, for the same reason as the + * Bun case: through a proxy CONNECT tunnel the proxy resolves the name + * itself and the pinned `connect.lookup` never runs, so the pin would be a + * claim we cannot keep — while the bare pinned Agent, dispatching directly, + * would silently bypass the proxy users are required to route through; * - redirects followed only when same-host/protocol/port (isPermittedRedirect), * bounded by MAX_REDIRECTS; * - `maxBytes` enforced against BOTH Content-Length and actual bytes written; - * - two-timer model: header timeout (30s, cleared when headers arrive) plus - * an idle watchdog reset per chunk (60s) — a slow large body is fine, a - * stalled one is not; + * - three-timer model: resolve watchdog (30s, covering the DNS + * resolve-and-pin step), header timeout (30s, cleared when headers arrive), + * plus an idle watchdog reset per chunk (60s) — a slow large body is fine, + * a stalled one is not; * - on ANY failure the `.part` file is removed (no half-download survives). */ export async function downloadMediaUrl(params: { @@ -104,8 +187,28 @@ export async function downloadMediaUrl(params: { * they neither touch real DNS nor need a listening socket. */ resolveTarget?: (url: string) => Promise; + /** + * Test seam for the proxy gate; production detects via the env vars plus + * the global dispatcher (see detectConfiguredProxy). Tests inject a fake + * so they neither depend on nor mutate process-wide state. + */ + detectProxy?: () => boolean; + /** Test seams for the three watchdogs. Real timers with tiny injected + * values are more faithful here than fake timers, which fight undici's + * internals and the stream plumbing. Production always uses the defaults. */ + resolveTimeoutMs?: number; + headerTimeoutMs?: number; + idleTimeoutMs?: number; }): Promise { - const { url, downloadsDir, maxBytes, signal } = params; + const { + url, + downloadsDir, + maxBytes, + signal, + resolveTimeoutMs = RESOLVE_TIMEOUT_MS, + headerTimeoutMs = HEADER_TIMEOUT_MS, + idleTimeoutMs = IDLE_TIMEOUT_MS, + } = params; // `'public'` is what turns on resolution + address vetting + pinning; the // helper is a no-op passthrough for any other policy. const resolveTarget = @@ -118,13 +221,23 @@ export async function downloadMediaUrl(params: { ); } + // isPrivateHost above fails OPEN on unparseable input (returns false), so + // this re-parse is where a malformed ref first surfaces — unguarded it + // would escape as a raw TypeError instead of the documented error type. + // The ref itself stays out of the message (it may embed anything). + let parsed: URL; + try { + parsed = new URL(url); + } catch { + throw new OmniDownloadError('URL media ref is not a valid URL (refused).'); + } + // https only, checked here so the message names the real reason: the // resolve-and-pin step below refuses plaintext (its error would just say // "could not be verified"). Media fetched over http can be swapped in // flight, and these bytes are uploaded to a third party under the user's // account — a URL whose contents cannot be attributed to the named host has // no business being delivered to the model. - const parsed = new URL(url); if (parsed.protocol !== 'https:') { throw new OmniDownloadError( `URL media must be https (refused for safety): ` + @@ -162,7 +275,40 @@ export async function downloadMediaUrl(params: { ); } - await fs.mkdir(downloadsDir, { recursive: true, mode: 0o700 }); + const undici = await loadUndici(); + + // Refuse outright behind a configured proxy, for the same reason as the + // Bun case above: the bare pinned Agent below dispatches DIRECTLY, so it + // would silently bypass the proxy config.ts installs process-wide (and any + // TLS-inspection policy riding on it) — while routing THROUGH the proxy + // would hand name resolution to the proxy's CONNECT handling, where the + // pinned `connect.lookup` never runs and the SSRF pin becomes + // unenforceable. Either way the promise cannot be kept, so refuse. + const proxied = params.detectProxy + ? params.detectProxy() + : detectConfiguredProxy(undici); + if (proxied) { + throw new OmniDownloadError( + `URL media fetching is not supported behind a proxy yet (refused): ` + + `the connection pinning that enforces SSRF protection cannot be ` + + `applied through a proxy. Download the file manually and use a ` + + `local path instead.`, + ); + } + + // Local filesystem failures (ENOTDIR, EROFS, EACCES) must stay inside the + // OmniDownloadError contract, and fs error text embeds the absolute + // directory path, which must not reach the UI or the debug log. + try { + await fs.mkdir(downloadsDir, { recursive: true, mode: 0o700 }); + } catch (err) { + throw new OmniDownloadError( + `Could not prepare the downloads directory for URL media: ${ + err instanceof Error ? scrubPathsFromMessage(err.message) : String(err) + }`, + { cause: err }, + ); + } const partPath = path.join( downloadsDir, `${randomBytes(8).toString('hex')}.part`, @@ -181,18 +327,56 @@ export async function downloadMediaUrl(params: { // `fetch` would resolve the name a second time and could still be // handed a private address. let target: ResolvedNetworkTarget; + const resolveAbort = new AbortController(); + const resolveTimer = setTimeout( + () => resolveAbort.abort(new Error('resolve timeout')), + resolveTimeoutMs, + ); try { - target = await resolveTarget(currentUrl); + const pending = resolveTarget(currentUrl); + // The race abandons `pending` on timeout; without this no-op handler + // a late rejection from the abandoned lookup would surface as an + // unhandled rejection. + pending.catch(() => {}); + target = await Promise.race([ + pending, + new Promise((_, reject) => { + resolveAbort.signal.addEventListener( + 'abort', + () => reject(resolveAbort.signal.reason), + { once: true }, + ); + }), + ]); } catch (err) { if (signal?.aborted) throw err; + if (resolveAbort.signal.aborted) { + throw new OmniDownloadError( + `Download timed out resolving ${new URL(currentUrl).hostname} ` + + `(${resolveTimeoutMs / 1000}s)`, + ); + } // Fail closed: unlike the general web-fetch path, these bytes are // uploaded to a third party, so "could not verify" must mean "do not - // fetch" rather than "connect and hope". + // fetch" rather than "connect and hope". The resolver's own text is + // translated rather than echoed: resolveNetworkTarget phrases its + // errors as "Extension network host …" (that helper's original + // caller), the same misattribution the credentials check above + // avoids — and nothing server-controlled beyond the hostname may + // enter the message anyway. + const text = err instanceof Error ? err.message : String(err); + const reason = /blocked address/i.test(text) + ? 'resolved to a non-public address' + : /did not resolve/i.test(text) + ? 'did not resolve' + : 'could not be verified'; throw new OmniDownloadError( - `URL host is not publicly routable or could not be verified ` + - `(refused for safety): ${new URL(currentUrl).hostname}: ` + - `${err instanceof Error ? err.message : String(err)}`, + `URL host ${reason} (refused for safety): ` + + `${new URL(currentUrl).hostname}`, + { cause: err }, ); + } finally { + clearTimeout(resolveTimer); } if (!target.lookup) { // No pinned lookup means the connection cannot be bound to the vetted @@ -203,7 +387,6 @@ export async function downloadMediaUrl(params: { `(refused for safety): ${new URL(currentUrl).hostname}`, ); } - const undici = await loadUndici(); // Default to undici's OWN fetch, not the global one. The dispatcher below // comes from the bundled undici, and Node's built-in fetch may ship a // different major version whose handler-interface check rejects it @@ -220,7 +403,7 @@ export async function downloadMediaUrl(params: { const headerAbort = new AbortController(); const headerTimer = setTimeout( () => headerAbort.abort(new Error('header timeout')), - HEADER_TIMEOUT_MS, + headerTimeoutMs, ); const combined = AbortSignal.any( signal ? [signal, headerAbort.signal] : [headerAbort.signal], @@ -243,13 +426,21 @@ export async function downloadMediaUrl(params: { if (headerAbort.signal.aborted) { throw new OmniDownloadError( `Download timed out waiting for response headers from ` + - `${new URL(currentUrl).hostname} (${HEADER_TIMEOUT_MS / 1000}s)`, + `${new URL(currentUrl).hostname} (${headerTimeoutMs / 1000}s)`, ); } + // Undici reports connection-level failures as a bare "fetch failed" + // TypeError with the actionable reason (ECONNREFUSED, TLS codes) on + // `cause` — surface that chain, or the user has nothing to act on. + const detail = + err instanceof Error + ? scrubPathsFromMessage(err.message) + : String(err); + const causeDetail = formatCauseChain(err); throw new OmniDownloadError( - `Download request failed for ${new URL(currentUrl).hostname}: ${ - err instanceof Error ? err.message : String(err) - }`, + `Download request failed for ${new URL(currentUrl).hostname}: ` + + `${detail}${causeDetail ? ` (${causeDetail})` : ''}`, + { cause: err }, ); } finally { clearTimeout(headerTimer); @@ -301,13 +492,15 @@ export async function downloadMediaUrl(params: { throw new OmniDownloadError('Download response had no body.'); } - // Stream to disk with hash + byte cap + idle watchdog. - const hash = createHash('sha256'); + // Stream to disk with byte cap + idle watchdog. Deliberately no hashing + // here: the pipeline re-derives identity from the file on disk (see + // DownloadedMedia), so a transfer-time hash would be dead work over up + // to a GiB of media. let written = 0; const idleAbort = new AbortController(); let idleTimer = setTimeout( () => idleAbort.abort(new Error('idle timeout')), - IDLE_TIMEOUT_MS, + idleTimeoutMs, ); const streamSignal = AbortSignal.any( signal ? [signal, idleAbort.signal] : [idleAbort.signal], @@ -322,7 +515,7 @@ export async function downloadMediaUrl(params: { clearTimeout(idleTimer); idleTimer = setTimeout( () => idleAbort.abort(new Error('idle timeout')), - IDLE_TIMEOUT_MS, + idleTimeoutMs, ); written += chunk.length; if (written > maxBytes) { @@ -330,7 +523,6 @@ export async function downloadMediaUrl(params: { `Download exceeds the omni media limit: >${maxBytes} bytes received.`, ); } - hash.update(chunk); yield chunk; } }; @@ -347,13 +539,18 @@ export async function downloadMediaUrl(params: { if (idleAbort.signal.aborted) { throw new OmniDownloadError( `Download stalled from ${new URL(currentUrl).hostname}: no data ` + - `for ${IDLE_TIMEOUT_MS / 1000}s`, + `for ${idleTimeoutMs / 1000}s`, ); } + // Write-stream failures embed the absolute `.part` path in their + // message; scrub before interpolating (same contract as mkdir above). throw new OmniDownloadError( `Download interrupted from ${new URL(currentUrl).hostname}: ${ - err instanceof Error ? err.message : String(err) + err instanceof Error + ? scrubPathsFromMessage(err.message) + : String(err) }`, + { cause: err }, ); } finally { clearTimeout(idleTimer); @@ -362,13 +559,7 @@ export async function downloadMediaUrl(params: { debugLogger.debug( `downloaded ${written} bytes from ${new URL(currentUrl).hostname}`, ); - return { - partPath, - sha256: hash.digest('hex'), - sizeBytes: written, - finalUrl: currentUrl, - contentType: res.headers.get('content-type') ?? undefined, - }; + return { partPath }; } } catch (err) { await fs.rm(partPath, { force: true }); diff --git a/packages/core/src/omni/estimation.test.ts b/packages/core/src/omni/estimation.test.ts index 19469da40e1..e9202bce3b3 100644 --- a/packages/core/src/omni/estimation.test.ts +++ b/packages/core/src/omni/estimation.test.ts @@ -14,7 +14,6 @@ function media( ): RecognizedMedia { return { modality, - sha256: 'a'.repeat(64), detectedMimeType: 'x/y', sizeBytes: 1, metadata, diff --git a/packages/core/src/omni/estimation.ts b/packages/core/src/omni/estimation.ts index 480b49ee7aa..42c8b4ac760 100644 --- a/packages/core/src/omni/estimation.ts +++ b/packages/core/src/omni/estimation.ts @@ -26,6 +26,11 @@ const AUDIO_TOKENS_PER_SECOND = 7; /** Pixels per visual token: 32×32 patch, 2 pixels per patch cell. */ const VISUAL_PIXELS_PER_TOKEN = 32 * 32 * 2; +/** Positive, finite number — rejects 0, negatives, NaN and Infinity. */ +function usable(value: number | undefined): value is number { + return value !== undefined && Number.isFinite(value) && value > 0; +} + /** * Estimate tokens from recognized raw-resource metadata (design doc §6.4): * @@ -42,11 +47,6 @@ const VISUAL_PIXELS_PER_TOKEN = 32 * 32 * 2; * probing as duration 0 must not produce a 0-token 'ok' that sails under * the transport guard. */ - -/** Positive, finite number — rejects 0, negatives, NaN and Infinity. */ -function usable(value: number | undefined): value is number { - return value !== undefined && Number.isFinite(value) && value > 0; -} export function estimateRawResourceTokens( media: RecognizedMedia, ): OmniTokenEstimate { diff --git a/packages/core/src/omni/ffmpeg.test.ts b/packages/core/src/omni/ffmpeg.test.ts index b4075e85e35..72914799c2e 100644 --- a/packages/core/src/omni/ffmpeg.test.ts +++ b/packages/core/src/omni/ffmpeg.test.ts @@ -16,7 +16,6 @@ import { isFfmpegAvailable, isFfprobeAvailable, probeMediaMetadata, - probeVideoMetadata, resetFfmpegCachesForTests, } from './ffmpeg.js'; @@ -116,7 +115,7 @@ describe('assertOmniRuntimeDependencies', () => { }); }); -describe('probeVideoMetadata', () => { +describe('probeMediaMetadata (video)', () => { it('parses format and first-video-stream fields', async () => { mockExecResult((command) => { expect(command).toBe('ffprobe'); @@ -136,7 +135,7 @@ describe('probeVideoMetadata', () => { }), }; }); - await expect(probeVideoMetadata('/v.mp4')).resolves.toEqual({ + await expect(probeMediaMetadata('/v.mp4', 'video')).resolves.toEqual({ formatName: 'mov,mp4,m4a,3gp,3g2,mj2', durationMs: 5500, width: 1280, @@ -159,7 +158,7 @@ describe('probeVideoMetadata', () => { ], }), })); - const result = await probeVideoMetadata('/v.mp4'); + const result = await probeMediaMetadata('/v.mp4', 'video'); expect(result.frameRate).toBe(25); expect(result.durationMs).toBeUndefined(); }); @@ -169,14 +168,14 @@ describe('probeVideoMetadata', () => { error: Object.assign(new Error('bad'), { code: 1 }), stderr: 'moov atom not found', })); - await expect(probeVideoMetadata('/broken.mp4')).rejects.toThrow( + await expect(probeMediaMetadata('/broken.mp4', 'video')).rejects.toThrow( /ffprobe failed \(exit 1\).*moov atom/s, ); }); it('throws on unparseable ffprobe output', async () => { mockExecResult(() => ({ stdout: 'not-json' })); - await expect(probeVideoMetadata('/v.mp4')).rejects.toThrow( + await expect(probeMediaMetadata('/v.mp4', 'video')).rejects.toThrow( /unparseable output/, ); }); @@ -185,8 +184,7 @@ describe('probeVideoMetadata', () => { describe('probeMediaMetadata per-modality branches', () => { it("reads the AUDIO stream (not video) for modality 'audio'", async () => { // A file carrying both streams proves the audio branch selects the - // audio stream: reading videoStream?.codec_name here would yield 'h264' - // and drop sampleRate/channels entirely. + // audio stream: reading videoStream?.codec_name here would yield 'h264'. mockExecResult(() => ({ stdout: JSON.stringify({ format: { format_name: 'mov,mp4', duration: '12.5' }, @@ -205,8 +203,6 @@ describe('probeMediaMetadata per-modality branches', () => { formatName: 'mov,mp4', durationMs: 12_500, codec: 'aac', - sampleRateHz: 44_100, - channels: 2, }); }); @@ -285,6 +281,5 @@ describe('probeMediaMetadata per-modality branches', () => { })); const result = await probeMediaMetadata('/silent.mp4', 'audio'); expect(result.codec).toBeUndefined(); - expect(result.sampleRateHz).toBeUndefined(); }); }); diff --git a/packages/core/src/omni/ffmpeg.ts b/packages/core/src/omni/ffmpeg.ts index 2023a0a74f9..736fbfc5c7f 100644 --- a/packages/core/src/omni/ffmpeg.ts +++ b/packages/core/src/omni/ffmpeg.ts @@ -138,15 +138,8 @@ export interface MediaProbeResult { /** Frame count of the primary video stream (image: >1 means animated — * GIF/APNG/animated WebP; absent when the container does not report it). */ frameCount?: number; - /** Sample rate in Hz of the first audio stream (audio only). */ - sampleRateHz?: number; - /** Channel count of the first audio stream (audio only). */ - channels?: number; } -/** @deprecated S1 name kept for existing imports. */ -export type VideoProbeResult = MediaProbeResult; - /** Parse an ffprobe rational like "30000/1001" (or plain "25") into fps. */ function parseFrameRate(raw: string | undefined): number | undefined { if (!raw) return undefined; @@ -204,8 +197,6 @@ export async function probeMediaMetadata( avg_frame_rate?: string; r_frame_rate?: string; nb_frames?: string; - sample_rate?: string; - channels?: number; }>; }; try { @@ -246,10 +237,6 @@ export async function probeMediaMetadata( ...base, durationMs, codec: audioStream?.codec_name, - sampleRateHz: audioStream?.sample_rate - ? Number(audioStream.sample_rate) - : undefined, - channels: audioStream?.channels, }; case 'video': default: @@ -265,11 +252,3 @@ export async function probeMediaMetadata( }; } } - -/** @deprecated S1 wrapper: video probe. */ -export async function probeVideoMetadata( - filePath: string, - signal?: AbortSignal, -): Promise { - return probeMediaMetadata(filePath, 'video', signal); -} diff --git a/packages/core/src/omni/guard.test.ts b/packages/core/src/omni/guard.test.ts index 201db6a7fff..d1b1d825374 100644 --- a/packages/core/src/omni/guard.test.ts +++ b/packages/core/src/omni/guard.test.ts @@ -10,7 +10,6 @@ import { assertWithinByteLimit, assertWithinTokenLimit, effectiveMaxUploadFileBytes, - DEFAULT_OMNI_MAX_UPLOAD_FILE_BYTES, OmniTransportGuardError, } from './guard.js'; import type { RecognizedMedia } from './recognition.js'; @@ -24,7 +23,6 @@ function cfg(overrides: { maxBytes?: number; maxTokens?: number }): Config { const AUDIO_5S: RecognizedMedia = { modality: 'audio', - sha256: 'a'.repeat(64), detectedMimeType: 'audio/mpeg', sizeBytes: 40_000, metadata: { durationMs: 5000 }, @@ -32,7 +30,6 @@ const AUDIO_5S: RecognizedMedia = { const VIDEO_8MIN: RecognizedMedia = { modality: 'video', - sha256: 'b'.repeat(64), detectedMimeType: 'video/mp4', sizeBytes: 36_000_000, metadata: { width: 852, height: 480, durationMs: 506_000, frameRate: 30 }, @@ -40,14 +37,16 @@ const VIDEO_8MIN: RecognizedMedia = { describe('byte guard', () => { it('uses the default 1 GiB when unset or non-positive', () => { - expect(effectiveMaxUploadFileBytes(cfg({}))).toBe( - DEFAULT_OMNI_MAX_UPLOAD_FILE_BYTES, - ); + // Assert the literal, not the exported constant: settingsSchema.ts + // hardcodes 1073741824 as the documented default (the DashScope + // temporary-upload per-file cap), so a drifted constant must fail here + // rather than ship green by comparing the implementation to itself. + expect(effectiveMaxUploadFileBytes(cfg({}))).toBe(1024 * 1024 * 1024); expect(effectiveMaxUploadFileBytes(cfg({ maxBytes: 0 }))).toBe( - DEFAULT_OMNI_MAX_UPLOAD_FILE_BYTES, + 1024 * 1024 * 1024, ); expect(effectiveMaxUploadFileBytes(cfg({ maxBytes: -5 }))).toBe( - DEFAULT_OMNI_MAX_UPLOAD_FILE_BYTES, + 1024 * 1024 * 1024, ); }); diff --git a/packages/core/src/omni/index.test.ts b/packages/core/src/omni/index.test.ts index 660b7e56f0d..cb8025f4cf7 100644 --- a/packages/core/src/omni/index.test.ts +++ b/packages/core/src/omni/index.test.ts @@ -10,7 +10,7 @@ import os from 'node:os'; import path from 'node:path'; import type { Config } from '../config/config.js'; import { AuthType } from '../core/contentGenerator.js'; -import { isOmniVideoDeliveryActive } from './index.js'; +import { isOmniDeliveryActive } from './index.js'; import { effectiveMaxDownloadFileBytes } from './index.js'; import { sanitizeErrorMessage } from './index.js'; @@ -88,18 +88,16 @@ describe('effectiveMaxDownloadFileBytes', () => { }); }); -describe('isOmniVideoDeliveryActive', () => { +describe('isOmniDeliveryActive', () => { it('is active for a DashScope endpoint with a static API key in a trusted workspace', () => { expect( - isOmniVideoDeliveryActive( - stubConfig({ trusted: true, cgc: DASHSCOPE_CGC }), - ), + isOmniDeliveryActive(stubConfig({ trusted: true, cgc: DASHSCOPE_CGC })), ).toBe(true); }); it('is inactive when omni is disabled', () => { expect( - isOmniVideoDeliveryActive( + isOmniDeliveryActive( stubConfig({ omniEnabled: false, trusted: true, cgc: DASHSCOPE_CGC }), ), ).toBe(false); @@ -107,15 +105,13 @@ describe('isOmniVideoDeliveryActive', () => { it('is inactive in an untrusted workspace', () => { expect( - isOmniVideoDeliveryActive( - stubConfig({ trusted: false, cgc: DASHSCOPE_CGC }), - ), + isOmniDeliveryActive(stubConfig({ trusted: false, cgc: DASHSCOPE_CGC })), ).toBe(false); }); it('treats unknown trust (undefined) as trusted', () => { expect( - isOmniVideoDeliveryActive( + isOmniDeliveryActive( stubConfig({ trusted: undefined, cgc: DASHSCOPE_CGC }), ), ).toBe(true); @@ -123,7 +119,7 @@ describe('isOmniVideoDeliveryActive', () => { it('is inactive under Qwen OAuth even though a placeholder apiKey exists', () => { expect( - isOmniVideoDeliveryActive( + isOmniDeliveryActive( stubConfig({ trusted: true, cgc: { @@ -138,7 +134,7 @@ describe('isOmniVideoDeliveryActive', () => { it('is inactive when the apiKey is the OAuth placeholder regardless of authType', () => { expect( - isOmniVideoDeliveryActive( + isOmniDeliveryActive( stubConfig({ trusted: true, cgc: { ...DASHSCOPE_CGC, apiKey: 'QWEN_OAUTH_DYNAMIC_TOKEN' }, @@ -149,7 +145,7 @@ describe('isOmniVideoDeliveryActive', () => { it('is inactive without a baseUrl (never sends the key to a default origin)', () => { expect( - isOmniVideoDeliveryActive( + isOmniDeliveryActive( stubConfig({ trusted: true, cgc: { authType: AuthType.USE_OPENAI, apiKey: 'sk-openai-key' }, @@ -160,7 +156,7 @@ describe('isOmniVideoDeliveryActive', () => { it('is inactive for non-DashScope endpoints', () => { expect( - isOmniVideoDeliveryActive( + isOmniDeliveryActive( stubConfig({ trusted: true, cgc: { @@ -175,7 +171,7 @@ describe('isOmniVideoDeliveryActive', () => { it('is inactive without a content generator config', () => { expect( - isOmniVideoDeliveryActive(stubConfig({ trusted: true, cgc: undefined })), + isOmniDeliveryActive(stubConfig({ trusted: true, cgc: undefined })), ).toBe(false); }); }); @@ -208,7 +204,6 @@ describe('readMediaViaOmniDelivery result shape', () => { : modality === 'audio' ? 'audio/mpeg' : 'video/mp4', - sha256: 'a'.repeat(64), sizeBytes: 1234, metadata, }; @@ -252,6 +247,7 @@ describe('readMediaViaOmniDelivery result shape', () => { .mockResolvedValue( mockRecognized('image', { width: 1920, height: 1080 }), ), + hashFileSha256: vi.fn().mockResolvedValue('a'.repeat(64)), extensionForMime: vi.fn().mockReturnValue('.png'), })); vi.doMock('./storage.js', () => ({ @@ -302,6 +298,7 @@ describe('readMediaViaOmniDelivery result shape', () => { recognizeMediaFile: vi .fn() .mockResolvedValue(mockRecognized('audio', { durationMs: 60_000 })), + hashFileSha256: vi.fn().mockResolvedValue('a'.repeat(64)), extensionForMime: vi.fn().mockReturnValue('.mp3'), })); vi.doMock('./storage.js', () => ({ @@ -340,6 +337,109 @@ describe('readMediaViaOmniDelivery result shape', () => { expect(result.returnDisplay).toContain('audio'); }); + it('rejects with OmniTransportGuardError before storing or uploading when the token guard trips', async () => { + // Every other pipeline test disables the guard (threshold 0); this one + // pins the guard call itself: a positive threshold with an over-budget + // estimate must reject BEFORE the hash/copy/upload stages run. + const putFileMock = vi.fn(); + const uploadFileMock = vi.fn(); + vi.doMock('./ffmpeg.js', () => ({ + isFfmpegAvailable: vi.fn().mockResolvedValue(true), + isFfprobeAvailable: vi.fn().mockResolvedValue(true), + })); + vi.doMock('./recognition.js', () => ({ + recognizeMediaFile: vi.fn().mockResolvedValue( + // 852×480×(506s × 30fps) / 2048 ≈ 3M tokens — far above 100. + mockRecognized('video', { + width: 852, + height: 480, + durationMs: 506_000, + frameRate: 30, + }), + ), + hashFileSha256: vi.fn().mockResolvedValue('a'.repeat(64)), + extensionForMime: vi.fn().mockReturnValue('.mp4'), + })); + vi.doMock('./storage.js', () => ({ + OmniObjectStore: class { + putFile = putFileMock; + getOmniRootDir() { + return '/tmp/omni-test-qwen/omni'; + } + }, + })); + vi.doMock('./upload.js', () => ({ + DashScopeUploader: class { + uploadFile = uploadFileMock; + }, + OSS_URL_PREFIX: 'oss://', + })); + const { processMediaForOmniDelivery, OmniTransportGuardError } = + await import('./index.js'); + + const config = { + ...deliveryConfig(), + getOmniMaxEstimatedTokens: vi.fn().mockReturnValue(100), + } as unknown as Config; + await expect( + processMediaForOmniDelivery(await realFile('long.mp4'), config, { + expectedModality: 'video', + }), + ).rejects.toThrow(OmniTransportGuardError); + expect(putFileMock).not.toHaveBeenCalled(); + expect(uploadFileMock).not.toHaveBeenCalled(); + }); + + it('propagates an abort from uploadFile instead of returning a fail-closed result', async () => { + // A user abort must surface to the caller's abort handling — wrapping it + // in the error-result shape would report a cancellation as a failed read. + const controller = new AbortController(); + vi.doMock('./ffmpeg.js', () => ({ + isFfmpegAvailable: vi.fn().mockResolvedValue(true), + isFfprobeAvailable: vi.fn().mockResolvedValue(true), + })); + vi.doMock('./recognition.js', () => ({ + recognizeMediaFile: vi + .fn() + .mockResolvedValue(mockRecognized('audio', { durationMs: 60_000 })), + hashFileSha256: vi.fn().mockResolvedValue('a'.repeat(64)), + extensionForMime: vi.fn().mockReturnValue('.mp3'), + })); + vi.doMock('./storage.js', () => ({ + OmniObjectStore: class { + async putFile() { + return { objectPath: '/tmp/obj.mp3', deduped: false }; + } + getOmniRootDir() { + return '/tmp/omni-test-qwen/omni'; + } + }, + })); + vi.doMock('./upload.js', () => ({ + DashScopeUploader: class { + async uploadFile() { + controller.abort(); + const err = new Error('The operation was aborted'); + err.name = 'AbortError'; + throw err; + } + }, + OSS_URL_PREFIX: 'oss://', + })); + const { readMediaViaOmniDelivery } = await import('./index.js'); + + await expect( + readMediaViaOmniDelivery({ + filePath: await realFile('song.mp3'), + config: deliveryConfig(), + displayName: 'song.mp3', + relativePathForDisplay: 'song.mp3', + expectedModality: 'audio', + signal: controller.signal, + }), + ).rejects.toMatchObject({ name: 'AbortError' }); + }); + it('fails closed with an error result (never inline base64) on failure', async () => { vi.doMock('./ffmpeg.js', () => ({ isFfmpegAvailable: vi.fn().mockResolvedValue(false), diff --git a/packages/core/src/omni/index.ts b/packages/core/src/omni/index.ts index c97721916b2..fd1004f00c0 100644 --- a/packages/core/src/omni/index.ts +++ b/packages/core/src/omni/index.ts @@ -21,6 +21,7 @@ import { } from './guard.js'; import { extensionForMime, + hashFileSha256, recognizeMediaFile, type OmniModality, type RecognizedMedia, @@ -38,7 +39,6 @@ export { OmniObjectStore } from './storage.js'; export { DashScopeUploader, OSS_URL_PREFIX } from './upload.js'; export { recognizeMediaFile, - recognizeVideoFile, sniffMediaType, sniffFileModality, sniffVideoMimeType, @@ -123,9 +123,6 @@ export interface OmniMediaDelivery { deduped: boolean; } -/** @deprecated S1 name kept for existing imports. */ -export type OmniVideoDelivery = OmniMediaDelivery; - /** Thrown for omni pipeline failures. The pipeline fails closed: callers * surface the message instead of silently falling back to inline base64. * Messages must not contain absolute paths or raw upstream response @@ -183,11 +180,8 @@ export function isOmniDeliveryActive(config: Config): boolean { return DashScopeOpenAICompatibleProvider.isDashScopeProvider(cgc); } -/** @deprecated S1 name kept for existing imports. */ -export const isOmniVideoDeliveryActive = isOmniDeliveryActive; - /** - * Omni pipeline: recognize → transport guard → promote into the + * Omni pipeline: recognize → transport guard → hash → promote into the * content-addressed store → upload via the DashScope temporary channel → * return the oss:// URL plus the token estimate. * @@ -200,10 +194,20 @@ export const isOmniVideoDeliveryActive = isOmniDeliveryActive; export async function processMediaForOmniDelivery( filePath: string, config: Config, - options?: { expectedModality?: OmniModality; signal?: AbortSignal }, + options?: { + expectedModality?: OmniModality; + signal?: AbortSignal; + /** + * Name used for the file in guard/error messages. Defaults to the + * file's basename — callers whose input is not a user-visible path + * (the URL funnel stages downloads under opaque temp names) pass the + * user-recognizable name instead. + */ + displayName?: string; + }, ): Promise { const { expectedModality, signal } = options ?? {}; - const displayName = path.basename(filePath); + const displayName = options?.displayName ?? path.basename(filePath); // Defense in depth: startup validation already asserted this, but the // pipeline can also be reached in embedders that skip Config.initialize. @@ -241,20 +245,29 @@ export async function processMediaForOmniDelivery( ); } - // Token guard AFTER probe (needs metadata), BEFORE copy/upload. + // Token guard AFTER probe (needs metadata), BEFORE hash/copy/upload — a + // token-oversized input must not pay a full-file SHA-256 to be rejected. const tokenEstimate = assertWithinTokenLimit(config, recognized, displayName); + // Content hash: identity of the stored object. Computed only once all + // guards have passed, immediately before promotion into the store. + let sha256: string; + try { + sha256 = await hashFileSha256(filePath, signal); + } catch (err) { + if (signal?.aborted) throw err; + throw new OmniDeliveryError( + `Failed to hash media file ${displayName}: ${sanitizeErrorMessage(err, [filePath])}`, + { cause: err }, + ); + } + const store = new OmniObjectStore(config.storage.getQwenDir()); const extension = extensionForMime(recognized.detectedMimeType); let objectPath: string; let deduped: boolean; try { - const put = await store.putFile( - filePath, - recognized.sha256, - extension, - signal, - ); + const put = await store.putFile(filePath, sha256, extension, signal); objectPath = put.objectPath; deduped = put.deduped; } catch (err) { @@ -292,32 +305,20 @@ export async function processMediaForOmniDelivery( } debugLogger.debug( - `omni ${recognized.modality} delivered: sha256=${recognized.sha256.slice(0, 12)}… ` + + `omni ${recognized.modality} delivered: sha256=${sha256.slice(0, 12)}… ` + `size=${recognized.sizeBytes} est=${tokenEstimate.estimatedTokenCount}(${tokenEstimate.status}) ` + `deduped=${deduped} uri=${fileUri}`, ); return { fileUri, mimeType: recognized.detectedMimeType, - sha256: recognized.sha256, + sha256, recognized, tokenEstimate, deduped, }; } -/** @deprecated S1 wrapper: video-only pipeline. */ -export async function processVideoForOmniDelivery( - filePath: string, - config: Config, - signal?: AbortSignal, -): Promise { - return processMediaForOmniDelivery(filePath, config, { - expectedModality: 'video', - signal, - }); -} - /** Read-result shape consumed by fileUtils.processSingleFileContent for * media Parts (structurally mirrors ProcessedFileReadResult's media case * without importing fileUtils, which would create a cycle). */ @@ -341,9 +342,6 @@ export interface OmniMediaReadResult { tokenEstimate?: OmniTokenEstimate; } -/** @deprecated S1 name kept for existing imports. */ -export type OmniVideoReadResult = OmniMediaReadResult; - /** * fileUtils-facing wrapper: run the delivery pipeline and shape the * outcome as a file-read result. Fails closed on delivery errors (no @@ -420,17 +418,6 @@ export async function readMediaViaOmniDelivery(params: { } } -/** @deprecated S1 wrapper. */ -export async function readVideoViaOmniDelivery(params: { - filePath: string; - config: Config; - displayName: string; - relativePathForDisplay: string; - signal?: AbortSignal; -}): Promise { - return readMediaViaOmniDelivery({ ...params, expectedModality: 'video' }); -} - /** Effective download byte ceiling — never above the upload channel cap * (downloading more than can be delivered is pointless), including when * `omni.download.maxFileBytes` is explicitly configured higher. */ diff --git a/packages/core/src/omni/recognition.test.ts b/packages/core/src/omni/recognition.test.ts index 04e25d67ef0..cb936b76634 100644 --- a/packages/core/src/omni/recognition.test.ts +++ b/packages/core/src/omni/recognition.test.ts @@ -4,7 +4,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { describe, expect, it } from 'vitest'; +import { afterEach, describe, expect, it, vi } from 'vitest'; import { createHash, randomBytes } from 'node:crypto'; import fs from 'node:fs/promises'; import os from 'node:os'; @@ -12,9 +12,15 @@ import path from 'node:path'; import { extensionForVideoMime, hashFileSha256, + recognizeMediaFile, sniffFileModality, sniffVideoMimeType, } from './recognition.js'; +import { probeMediaMetadata } from './ffmpeg.js'; + +vi.mock('./ffmpeg.js', () => ({ + probeMediaMetadata: vi.fn(), +})); function mp4Header(brand = 'isom'): Buffer { // [size:4]["ftyp"][major brand:4][minor version:4] @@ -163,10 +169,27 @@ describe('sniffMediaType (S2 modalities)', async () => { expect(sniffMediaType(Buffer.alloc(0))).toBeNull(); }); + it('requires the full 6-byte GIF signature, not just the "GIF" prefix', () => { + // Plain text happening to start with "GIF " must not sniff as an image. + expect(sniffMediaType(Buffer.from('GIF export notes, take 2'))).toBeNull(); + expect(sniffMediaType(Buffer.from('GIF90a....'))).toBeNull(); + // Both real signatures detect. + expect(sniffMediaType(Buffer.from('GIF87a....'))).toMatchObject({ + mimeType: 'image/gif', + modality: 'image', + }); + expect(sniffMediaType(Buffer.from('GIF89a....'))).toMatchObject({ + mimeType: 'image/gif', + modality: 'image', + }); + }); + it('does not mistake the UTF-16 LE BOM for an MPEG frame sync', () => { - // 0xFF 0xFE passes the naive sync mask (0xFE & 0xE0 === 0xE0), but a - // real MPEG frame never uses 0xFE (reserved layer bits). Callers - // without a secondary modality gate must not see audio/mpeg here. + // 0xFF 0xFE passes the naive sync mask (0xFE & 0xE0 === 0xE0) and is even + // a technically valid MPEG-1 Layer I header, but it is also the UTF-16 LE + // BOM; the sniffer deliberately excludes it so UTF-16 LE text does not + // sniff as audio. Callers without a secondary modality gate must not see + // audio/mpeg here. const utf16le = Buffer.concat([ Buffer.from([0xff, 0xfe]), Buffer.from('h\0e\0l\0l\0o\0', 'latin1'), @@ -224,4 +247,96 @@ describe('sniffFileModality', () => { sniffFileModality(path.join(os.tmpdir(), 'omni-absent-xyz.mp4')), ).resolves.toBeNull(); }); + + it('returns the sniffed modality even when close() rejects (never throws)', async () => { + // Network mounts can fail the final close() (EIO/ESTALE); the pre-gate's + // never-throws contract must survive a rejecting close, not turn a + // successful sniff into a crash. + const header = mp4Header('isom'); + const openSpy = vi.spyOn(fs, 'open').mockResolvedValue({ + stat: async () => ({ size: header.length }), + read: async (buf: Buffer) => { + header.copy(buf); + return { bytesRead: header.length, buffer: buf }; + }, + close: async () => { + throw Object.assign(new Error('EIO: i/o error, close'), { + code: 'EIO', + }); + }, + } as unknown as fs.FileHandle); + try { + await expect(sniffFileModality('/mnt/nfs/clip.mp4')).resolves.toBe( + 'video', + ); + } finally { + openSpy.mockRestore(); + } + }); +}); + +describe('recognizeMediaFile', () => { + const probeMock = vi.mocked(probeMediaMetadata); + const tmpDirs: string[] = []; + + async function tempFile(name: string, content: Buffer): Promise { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'omni-recognize-')); + tmpDirs.push(dir); + const filePath = path.join(dir, name); + await fs.writeFile(filePath, content); + return filePath; + } + + afterEach(async () => { + probeMock.mockReset(); + await Promise.all( + tmpDirs + .splice(0) + .map((dir) => fs.rm(dir, { recursive: true, force: true })), + ); + }); + + it('fails closed on content that does not sniff as supported media', async () => { + const filePath = await tempFile( + 'notes.txt', + Buffer.from('just some plain text, definitely not media'), + ); + await expect(recognizeMediaFile(filePath)).rejects.toThrow( + /does not match a supported media container.*notes\.txt/s, + ); + expect(probeMock).not.toHaveBeenCalled(); + }); + + it('rejects a sniffed modality that contradicts expectedModality', async () => { + // A real MP3 frame sync referenced as video: the sniff succeeds (audio) + // but the modality gate must reject the mismatch. + const filePath = await tempFile( + 'clip.mp4', + Buffer.from([0xff, 0xfb, 0x90, 0x00, 0x00, 0x00]), + ); + await expect( + recognizeMediaFile(filePath, { expectedModality: 'video' }), + ).rejects.toThrow( + /sniffs as audio \(audio\/mpeg\) but was referenced as video.*clip\.mp4/s, + ); + expect(probeMock).not.toHaveBeenCalled(); + }); + + it('resolves with the sniffed modality, MIME type, size, and probe metadata', async () => { + probeMock.mockResolvedValue({ durationMs: 5000, codec: 'mp3' }); + const content = Buffer.concat([ + Buffer.from([0xff, 0xfb, 0x90, 0x00]), + Buffer.alloc(60), + ]); + const filePath = await tempFile('song.mp3', content); + await expect( + recognizeMediaFile(filePath, { expectedModality: 'audio' }), + ).resolves.toEqual({ + modality: 'audio', + detectedMimeType: 'audio/mpeg', + sizeBytes: content.length, + metadata: { durationMs: 5000, codec: 'mp3' }, + }); + expect(probeMock).toHaveBeenCalledWith(filePath, 'audio', undefined); + }); }); diff --git a/packages/core/src/omni/recognition.ts b/packages/core/src/omni/recognition.ts index aaaa3d4cbe7..18fb9ab469a 100644 --- a/packages/core/src/omni/recognition.ts +++ b/packages/core/src/omni/recognition.ts @@ -18,8 +18,6 @@ export type OmniModality = 'image' | 'audio' | 'video'; export interface RecognizedMedia { /** Modality derived from the sniffed container type. */ modality: OmniModality; - /** Hex-encoded SHA-256 of the full file content. */ - sha256: string; /** Authoritative MIME type detected from content (sniff), not extension. */ detectedMimeType: string; /** File size in bytes. */ @@ -28,28 +26,20 @@ export interface RecognizedMedia { metadata: MediaProbeResult; } -/** @deprecated S1 name — video-only alias kept for existing imports. */ -export type RecognizedVideo = RecognizedMedia; - interface SniffedType { mimeType: string; modality: OmniModality; - extension: string; } function bmffBrandType(brand: string): SniffedType { if (brand.startsWith('qt')) { - return { - mimeType: 'video/quicktime', - modality: 'video', - extension: '.mov', - }; + return { mimeType: 'video/quicktime', modality: 'video' }; } // M4A/M4B audio brands inside ISO BMFF. if (brand.startsWith('M4A') || brand.startsWith('M4B')) { - return { mimeType: 'audio/mp4', modality: 'audio', extension: '.m4a' }; + return { mimeType: 'audio/mp4', modality: 'audio' }; } - return { mimeType: 'video/mp4', modality: 'video', extension: '.mp4' }; + return { mimeType: 'video/mp4', modality: 'video' }; } /** @@ -68,21 +58,13 @@ export function sniffMediaType(header: Buffer): SniffedType | null { if (riff) { const kind = header.subarray(8, 12).toString('latin1'); if (kind === 'AVI ') { - return { - mimeType: 'video/x-msvideo', - modality: 'video', - extension: '.avi', - }; + return { mimeType: 'video/x-msvideo', modality: 'video' }; } if (kind === 'WAVE') { - return { mimeType: 'audio/wav', modality: 'audio', extension: '.wav' }; + return { mimeType: 'audio/wav', modality: 'audio' }; } if (kind === 'WEBP') { - return { - mimeType: 'image/webp', - modality: 'image', - extension: '.webp', - }; + return { mimeType: 'image/webp', modality: 'image' }; } return null; } @@ -90,35 +72,40 @@ export function sniffMediaType(header: Buffer): SniffedType | null { if (header.length >= 8) { // PNG if (header.readUInt32BE(0) === 0x89504e47) { - return { mimeType: 'image/png', modality: 'image', extension: '.png' }; + return { mimeType: 'image/png', modality: 'image' }; + } + } + if (header.length >= 6) { + // GIF: the full 6-byte GIF87a/GIF89a signature. Matching only the + // 3-byte "GIF" prefix would classify plain text starting with "GIF" + // (e.g. "GIF export notes.txt") as an image. + const gifSig = header.subarray(0, 6).toString('latin1'); + if (gifSig === 'GIF87a' || gifSig === 'GIF89a') { + return { mimeType: 'image/gif', modality: 'image' }; } } if (header.length >= 4) { // EBML (WebM/Matroska) if (header.readUInt32BE(0) === 0x1a45dfa3) { - return { mimeType: 'video/webm', modality: 'video', extension: '.webm' }; + return { mimeType: 'video/webm', modality: 'video' }; } // FLAC if (header.subarray(0, 4).toString('latin1') === 'fLaC') { - return { mimeType: 'audio/flac', modality: 'audio', extension: '.flac' }; + return { mimeType: 'audio/flac', modality: 'audio' }; } // OGG if (header.subarray(0, 4).toString('latin1') === 'OggS') { - return { mimeType: 'audio/ogg', modality: 'audio', extension: '.ogg' }; - } - // GIF87a / GIF89a - if (header.subarray(0, 3).toString('latin1') === 'GIF') { - return { mimeType: 'image/gif', modality: 'image', extension: '.gif' }; + return { mimeType: 'audio/ogg', modality: 'audio' }; } } if (header.length >= 3) { // JPEG if (header[0] === 0xff && header[1] === 0xd8 && header[2] === 0xff) { - return { mimeType: 'image/jpeg', modality: 'image', extension: '.jpg' }; + return { mimeType: 'image/jpeg', modality: 'image' }; } // MP3: ID3 tag or bare MPEG frame sync (0xFFEx/0xFFFx). if (header.subarray(0, 3).toString('latin1') === 'ID3') { - return { mimeType: 'audio/mpeg', modality: 'audio', extension: '.mp3' }; + return { mimeType: 'audio/mpeg', modality: 'audio' }; } // 0xFF 0xFE is the UTF-16 LE BOM and also passes the sync mask // (0xFE & 0xE0 === 0xE0). Per the MPEG audio header 0xFF 0xFE is @@ -133,7 +120,7 @@ export function sniffMediaType(header: Buffer): SniffedType | null { header[1] !== 0xfe && (header[1]! & 0xe0) === 0xe0 ) { - return { mimeType: 'audio/mpeg', modality: 'audio', extension: '.mp3' }; + return { mimeType: 'audio/mpeg', modality: 'audio' }; } } return null; @@ -205,16 +192,22 @@ export async function sniffFileModality( } catch { return null; } finally { - await handle.close(); + // A rejecting close() (EIO/ESTALE on network mounts) must not break the + // never-throws contract of this pre-gate. + await handle.close().catch(() => {}); } } /** - * Recognize a local media file: content sniff (magic bytes), streaming - * SHA-256, and ffprobe metadata. Throws when the content does not sniff as - * a supported media container or when probing fails — the omni pipeline - * fails closed on unrecognizable input. Error messages carry the file's - * basename only (they can reach model-visible content). + * Recognize a local media file: content sniff (magic bytes) and ffprobe + * metadata. Throws when the content does not sniff as a supported media + * container or when probing fails — the omni pipeline fails closed on + * unrecognizable input. Error messages carry the file's basename only + * (they can reach model-visible content). + * + * Hashing is deliberately NOT part of recognition: the token guard runs on + * this result, and an input it rejects must not have paid a full-file + * SHA-256 first. The pipeline hashes after the guard passes. * * When `expectedModality` is given, a successful sniff of a DIFFERENT * modality is rejected (e.g. an .mp3 that is actually an mp4 container). @@ -249,24 +242,12 @@ export async function recognizeMediaFile( ); } - const [sha256, metadata] = await Promise.all([ - hashFileSha256(filePath, signal), - probeMediaMetadata(filePath, sniffed.modality, signal), - ]); + const metadata = await probeMediaMetadata(filePath, sniffed.modality, signal); return { modality: sniffed.modality, - sha256, detectedMimeType: sniffed.mimeType, sizeBytes, metadata, }; } - -/** @deprecated S1 wrapper: video-only recognition. */ -export async function recognizeVideoFile( - filePath: string, - signal?: AbortSignal, -): Promise { - return recognizeMediaFile(filePath, { expectedModality: 'video', signal }); -} diff --git a/packages/core/src/omni/tool-result-media.test.ts b/packages/core/src/omni/tool-result-media.test.ts index 03a1c38aa08..98c3559b7ed 100644 --- a/packages/core/src/omni/tool-result-media.test.ts +++ b/packages/core/src/omni/tool-result-media.test.ts @@ -4,7 +4,18 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { + afterAll, + beforeAll, + beforeEach, + describe, + expect, + it, + vi, +} from 'vitest'; +import os from 'node:os'; +import nodePath from 'node:path'; +import nodeFs from 'node:fs/promises'; import type { Part } from '@google/genai'; import type { Config } from '../config/config.js'; @@ -33,14 +44,28 @@ function inlinePart(mimeType: string, bytes: Buffer): Part { return { inlineData: { mimeType, data: bytes.toString('base64') } }; } +// Per-run isolated qwen dir: a shared hardcoded path would leak staging +// files across runs and collide between concurrent test invocations. +let testQwenDir: string; + function cfg(modalities: Record): Config { return { isOmniEnabled: () => true, getContentGeneratorConfig: () => ({ modalities }), - storage: { getQwenDir: () => '/tmp/omni-trm-test-qwen' }, + storage: { getQwenDir: () => testQwenDir }, } as unknown as Config; } +beforeAll(async () => { + testQwenDir = await nodeFs.mkdtemp( + nodePath.join(os.tmpdir(), 'omni-trm-qwen-'), + ); +}); + +afterAll(async () => { + await nodeFs.rm(testQwenDir, { recursive: true, force: true }); +}); + beforeEach(() => { gateMock.mockReturnValue(true); deliverMock.mockReset(); @@ -176,12 +201,11 @@ describe('processToolResultOmniMedia', () => { // That failure must degrade THIS part to inline like any other delivery // failure — not reject the whole call, which would report a tool that // succeeded as failed. - const os = await import('node:os'); - const nodePath = await import('node:path'); - const fs = await import('node:fs/promises'); - const qwenDir = await fs.mkdtemp(nodePath.join(os.tmpdir(), 'omni-trm-')); + const qwenDir = await nodeFs.mkdtemp( + nodePath.join(os.tmpdir(), 'omni-trm-'), + ); // OmniObjectStore roots at /omni; make that path a plain file. - await fs.writeFile(nodePath.join(qwenDir, 'omni'), 'not a directory'); + await nodeFs.writeFile(nodePath.join(qwenDir, 'omni'), 'not a directory'); try { const config = { isOmniEnabled: () => true, @@ -193,7 +217,111 @@ describe('processToolResultOmniMedia', () => { expect(result).toBe(parts); expect(deliverMock).not.toHaveBeenCalled(); } finally { - await fs.rm(qwenDir, { recursive: true, force: true }); + await nodeFs.rm(qwenDir, { recursive: true, force: true }); } }); + + it('returns the original array untouched when the omni gate is off', async () => { + gateMock.mockReturnValue(false); + const parts = [inlinePart('image/png', PNG_BYTES)]; + const result = await processToolResultOmniMedia( + parts, + cfg({ image: true }), + signal, + ); + expect(result).toBe(parts); + expect(deliverMock).not.toHaveBeenCalled(); + }); + + it('converts media nested inside functionResponse.parts (the production funnel shape)', async () => { + // Both physical funnels deliver tool-result media wrapped by + // convertToFunctionResponse as {functionResponse: {…, parts: + // [{inlineData}]}} — not as top-level inlineData parts. This is the + // branch production actually exercises. + const parts: Part[] = [ + { + functionResponse: { + id: 'call_1', + name: 'Read', + response: { output: 'ok' }, + parts: [ + { text: 'caption' }, + inlinePart('image/png', PNG_BYTES), + ] as Part[], + }, + } as Part, + ]; + const result = await processToolResultOmniMedia( + parts, + cfg({ image: true }), + signal, + ); + expect(result).not.toBe(parts); + const nested = result[0]!.functionResponse?.parts as Part[]; + expect(nested[0]!.text).toBe('caption'); + expect(nested[1]!.fileData?.fileUri).toBe('oss://bucket/key'); + expect(nested[1]!.inlineData).toBeUndefined(); + // The wrapper's identity fields survive the rebuild. + expect(result[0]!.functionResponse?.id).toBe('call_1'); + expect(result[0]!.functionResponse?.name).toBe('Read'); + }); + + it('returns the original identity when functionResponse.parts contains no qualifying media', async () => { + const parts: Part[] = [ + { + functionResponse: { + id: 'call_1', + name: 'Read', + response: { output: 'ok' }, + parts: [{ text: 'no media here' }] as Part[], + }, + } as Part, + ]; + const result = await processToolResultOmniMedia( + parts, + cfg({ image: true }), + signal, + ); + expect(result).toBe(parts); + }); + + it('enforces the aggregate upload-byte budget (over-budget parts stay inline)', async () => { + // Two parts: the first consumes nearly the whole 128 MiB budget, the + // second no longer fits and must stay inline even though the upload + // COUNT budget still has room. + const bigBytes = Buffer.concat([ + Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), + Buffer.alloc(127 * 1024 * 1024), + ]); + const parts = [ + inlinePart('image/png', bigBytes), + inlinePart('image/png', bigBytes), + ]; + const result = await processToolResultOmniMedia( + parts, + cfg({ image: true }), + signal, + ); + expect(deliverMock).toHaveBeenCalledTimes(1); + expect(result[0]!.fileData).toBeDefined(); + expect(result[1]!.inlineData).toBeDefined(); + }); + + it('propagates an abort instead of degrading the part to inline', async () => { + // A user abort is not a delivery failure — swallowing it into the + // keep-inline path would let an aborted turn keep converting parts. + const controller = new AbortController(); + deliverMock.mockImplementation(async () => { + controller.abort(); + throw new Error('aborted mid-upload'); + }); + const parts = [inlinePart('image/png', PNG_BYTES)]; + await expect( + processToolResultOmniMedia( + parts, + cfg({ image: true }), + controller.signal, + ), + ).rejects.toThrow('aborted mid-upload'); + }); }); diff --git a/packages/core/src/omni/upload.ts b/packages/core/src/omni/upload.ts index 75ed1206367..87cd45b3c78 100644 --- a/packages/core/src/omni/upload.ts +++ b/packages/core/src/omni/upload.ts @@ -34,7 +34,7 @@ export interface DashScopeUploaderOptions { * Chat-completions base URL (e.g. https://dashscope.aliyuncs.com/compatible-mode/v1). * The uploads endpoint lives at `/api/v1/uploads`; only the origin * is used. Defaults to the official public endpoint when omitted — but the - * production gate (isOmniVideoDeliveryActive) requires a concrete DashScope + * production gate (isOmniDeliveryActive) requires a concrete DashScope * baseUrl, so the credential is never sent to an origin the user did not * configure for this provider. */ diff --git a/packages/core/src/utils/fileUtils.test.ts b/packages/core/src/utils/fileUtils.test.ts index 286e68ee484..d8f4efb9e0d 100644 --- a/packages/core/src/utils/fileUtils.test.ts +++ b/packages/core/src/utils/fileUtils.test.ts @@ -97,6 +97,18 @@ vi.mock('./pdf.js', async (importOriginal) => { return { ...actual, renderPDFPagesToImages: vi.fn() }; }); +// The omni module is imported dynamically inside processSingleFileContent; +// vitest intercepts dynamic imports too, so a registry mock is what stubs +// the delivery pipeline. Safe for every non-omni test: the cheap +// `config.isOmniEnabled?.()` gate runs BEFORE the import, and the default +// mockConfig has no isOmniEnabled. +const omniGateMocks = vi.hoisted(() => ({ + isOmniDeliveryActive: vi.fn(), + sniffFileModality: vi.fn(), + readMediaViaOmniDelivery: vi.fn(), +})); +vi.mock('../omni/index.js', () => omniGateMocks); + const mockMimeGetType = mime.getType as Mock; const mockExecFile = vi.mocked(execFile); const mockRender = vi.mocked(renderPDFPagesToImages); @@ -1428,6 +1440,128 @@ describe('fileUtils', () => { expect(result.returnDisplay).toContain('Skipped image file'); }); + describe('omni delivery gating', () => { + // Exercises the omni-vs-legacy decision in processSingleFileContent: + // enablement gate → delivery-active gate → content pre-sniff → per- + // modality config. The pipeline itself is mocked (index.test.ts owns + // it); what is pinned here is WHICH path a file takes. + function omniConfig(overrides: Record = {}): Config { + return { + ...mockConfig, + isOmniEnabled: () => true, + getContentGeneratorConfig: () => ({ + modalities: { image: true, audio: true, video: true }, + }), + ...overrides, + } as unknown as Config; + } + + beforeEach(() => { + omniGateMocks.isOmniDeliveryActive.mockReturnValue(true); + omniGateMocks.readMediaViaOmniDelivery.mockResolvedValue({ + llmContent: { + fileData: { fileUri: 'oss://bucket/key', mimeType: 'video/mp4' }, + }, + returnDisplay: 'Delivered via omni upload.', + }); + }); + + it('routes a sniff-confirmed video through readMediaViaOmniDelivery', async () => { + const videoPath = path.join(tempRootDir, 'clip.mp4'); + actualNodeFs.writeFileSync(videoPath, Buffer.from('fake mp4')); + mockMimeGetType.mockReturnValue('video/mp4'); + omniGateMocks.sniffFileModality.mockResolvedValue('video'); + + const result = await processSingleFileContent(videoPath, omniConfig()); + + expect(omniGateMocks.readMediaViaOmniDelivery).toHaveBeenCalledWith( + expect.objectContaining({ + filePath: videoPath, + expectedModality: 'video', + }), + ); + expect(result.returnDisplay).toBe('Delivered via omni upload.'); + }); + + it('falls back to the legacy inline path when the sniff disagrees with the extension', async () => { + // A file whose bytes sniff as a DIFFERENT modality than its + // extension suggests must take the legacy path (inline under its + // extension-derived type), not the fail-closed pipeline. + const videoPath = path.join(tempRootDir, 'clip.mp4'); + actualNodeFs.writeFileSync(videoPath, Buffer.from('small bytes')); + mockMimeGetType.mockReturnValue('video/mp4'); + omniGateMocks.sniffFileModality.mockResolvedValue(null); + + const result = await processSingleFileContent(videoPath, omniConfig()); + + expect(omniGateMocks.readMediaViaOmniDelivery).not.toHaveBeenCalled(); + const content = result.llmContent as Part; + expect(content.inlineData?.mimeType).toBe('video/mp4'); + }); + + it('never sniffs or delivers when the modality is disabled in config', async () => { + const videoPath = path.join(tempRootDir, 'clip.mp4'); + actualNodeFs.writeFileSync(videoPath, Buffer.from('fake mp4')); + mockMimeGetType.mockReturnValue('video/mp4'); + + const result = await processSingleFileContent( + videoPath, + omniConfig({ + getContentGeneratorConfig: () => ({ + modalities: { video: false }, + }), + }), + ); + + expect(omniGateMocks.sniffFileModality).not.toHaveBeenCalled(); + expect(omniGateMocks.readMediaViaOmniDelivery).not.toHaveBeenCalled(); + expect(result.returnDisplay).toContain('Skipped video file'); + }); + + it('takes the legacy path when omni delivery is not active for the endpoint', async () => { + const videoPath = path.join(tempRootDir, 'clip.mp4'); + actualNodeFs.writeFileSync(videoPath, Buffer.from('small bytes')); + mockMimeGetType.mockReturnValue('video/mp4'); + omniGateMocks.isOmniDeliveryActive.mockReturnValue(false); + + const result = await processSingleFileContent(videoPath, omniConfig()); + + expect(omniGateMocks.readMediaViaOmniDelivery).not.toHaveBeenCalled(); + const content = result.llmContent as Part; + expect(content.inlineData?.mimeType).toBe('video/mp4'); + }); + + it('bypasses the 100 MB image source cap when omni takes the file', async () => { + // The cap protects the overview DECODER; the omni path uploads + // original bytes without decoding and enforces its own ceiling, so + // an over-cap image must delegate to omni delivery instead of being + // rejected. Sparse file: only the stat size matters — the mocked + // pipeline never reads the bytes. + const imagePath = path.join(tempRootDir, 'huge.png'); + actualNodeFs.writeFileSync(imagePath, Buffer.alloc(0)); + actualNodeFs.truncateSync(imagePath, 101 * 1024 * 1024); + mockMimeGetType.mockReturnValue('image/png'); + omniGateMocks.sniffFileModality.mockResolvedValue('image'); + omniGateMocks.readMediaViaOmniDelivery.mockResolvedValue({ + llmContent: { + fileData: { fileUri: 'oss://bucket/key', mimeType: 'image/png' }, + }, + returnDisplay: 'Delivered via omni upload.', + }); + + const result = await processSingleFileContent(imagePath, omniConfig()); + + expect(result.error).toBeUndefined(); + expect(result.returnDisplay).toBe('Delivered via omni upload.'); + expect(omniGateMocks.readMediaViaOmniDelivery).toHaveBeenCalledWith( + expect.objectContaining({ + filePath: imagePath, + expectedModality: 'image', + }), + ); + }); + }); + it('keeps image inline when preserveUnsupportedImage is true', async () => { await sharp({ create: { diff --git a/packages/core/src/utils/fileUtils.ts b/packages/core/src/utils/fileUtils.ts index b838cfeecd6..e0687583c5c 100644 --- a/packages/core/src/utils/fileUtils.ts +++ b/packages/core/src/utils/fileUtils.ts @@ -929,7 +929,8 @@ export interface ProcessedFileReadResult { * Raw-resource token estimate for omni-delivered media (the single * storage location for the estimate — design doc §6.4). Absent for * non-omni reads; `status: 'unavailable'` when required metadata was - * missing. + * missing. No consumer reads it yet — surfaces (context accounting, + * telemetry) land with the S4 limits slice. */ tokenEstimate?: import('../omni/estimation.js').OmniTokenEstimate; } diff --git a/packages/vscode-ide-companion/schemas/settings.schema.json b/packages/vscode-ide-companion/schemas/settings.schema.json index 170c12b6ceb..01001bc0d9c 100644 --- a/packages/vscode-ide-companion/schemas/settings.schema.json +++ b/packages/vscode-ide-companion/schemas/settings.schema.json @@ -3264,7 +3264,7 @@ "type": "object", "properties": { "enabled": { - "description": "Enable the omni media pipeline. Local video files referenced with @ are recognized (ffprobe), stored content-addressed under .qwen/omni/objects/, uploaded through the DashScope temporary upload channel, and delivered as oss:// URLs instead of inline base64. Only active for DashScope-compatible endpoints. Can also be enabled via QWEN_CODE_ENABLE_OMNI=1.", + "description": "Enable the omni media pipeline. Media files (video, image, audio) referenced with @ — and media served from @https:// URLs — are recognized (ffprobe), stored content-addressed under .qwen/omni/objects/, uploaded through the DashScope temporary upload channel, and delivered as oss:// URLs instead of inline base64. Only active for DashScope-compatible endpoints. Can also be enabled via QWEN_CODE_ENABLE_OMNI=1.", "type": "boolean", "default": false }, @@ -3288,7 +3288,7 @@ "type": "number", "minimum": 0, "default": 0, - "description": "Estimated-token ceiling for a single omni media input, checked before upload using the versioned raw-resource estimator. 0 disables the token guard (estimates are still attached for observability) — the estimation formula is pending confirmation with the model provider; set a positive threshold to enforce fail-closed rejection." + "description": "Estimated-token ceiling for a single omni media input, checked before upload using the versioned raw-resource estimator. 0 disables the token guard — the estimation formula is pending confirmation with the model provider; set a positive threshold to enforce fail-closed rejection." } } }, diff --git a/scripts/dev.js b/scripts/dev.js index 34e5e5d50c8..8be9bee212a 100755 --- a/scripts/dev.js +++ b/scripts/dev.js @@ -67,11 +67,24 @@ const loaderPath = join(tmpDir, 'loader.mjs'); const coreSourcePath = join(root, 'packages', 'core', 'index.ts'); const coreSourceUrl = pathToFileURL(coreSourcePath).href; +// The `/omni` subpath export maps to dist/ in package.json; without an alias +// here, dev mode would silently load a stale build of the omni pipeline while +// the rest of core runs from source. +const coreOmniSourcePath = join( + root, + 'packages', + 'core', + 'src', + 'omni', + 'index.ts', +); +const coreOmniSourceUrl = pathToFileURL(coreOmniSourcePath).href; const loaderCode = ` import { pathToFileURL } from 'node:url'; const coreSourceUrl = '${coreSourceUrl}'; +const coreOmniSourceUrl = '${coreOmniSourceUrl}'; export function resolve(specifier, context, nextResolve) { if (specifier === '@qwen-code/qwen-code-core') { @@ -81,6 +94,13 @@ export function resolve(specifier, context, nextResolve) { format: 'module', }; } + if (specifier === '@qwen-code/qwen-code-core/omni') { + return { + shortCircuit: true, + url: coreOmniSourceUrl, + format: 'module', + }; + } return nextResolve(specifier, context); } `;