From 4a948acf8f0ae08fc3d26b9410fb2a2891e00425 Mon Sep 17 00:00:00 2001 From: LaZzyMan Date: Thu, 6 Aug 2026 23:24:42 +0800 Subject: [PATCH 01/62] refactor(omni): migrate settings to design namespace (processing/delivery/ingestion/storage) - omni.upload.maxFileBytes -> omni.processing.transportGuard.maxUploadFileBytes (<=1GiB) - omni.transport.maxEstimatedTokens -> omni.processing.transportGuard.maxEstimatedTokens - omni.upload.cacheTtlHours -> omni.delivery.upload.urlTtlHours - omni.download.maxFileBytes -> omni.ingestion.localization.url.maxFileBytes - new: omni.processing.{limits,fixedPolicies,transportGuard.policies,policyTools}, omni.storage.quarantine.{retentionDays,maxBytes} - core getters renamed to match; error messages cite new key paths Experimental branch: one-shot rename, no migration shim. --- packages/cli/src/config/config.ts | 11 +- packages/cli/src/config/settingsSchema.ts | 375 +++++++++++++++--- packages/core/src/config/config.ts | 32 +- packages/core/src/omni/guard.test.ts | 8 +- packages/core/src/omni/guard.ts | 8 +- packages/core/src/omni/index.test.ts | 10 +- packages/core/src/omni/index.ts | 7 +- packages/core/src/utils/fileUtils.ts | 2 +- .../schemas/settings.schema.json | 164 ++++++-- 9 files changed, 497 insertions(+), 120 deletions(-) diff --git a/packages/cli/src/config/config.ts b/packages/cli/src/config/config.ts index 26fa416e878..d128dfefa41 100755 --- a/packages/cli/src/config/config.ts +++ b/packages/cli/src/config/config.ts @@ -2192,10 +2192,13 @@ export async function loadCliConfig( } : undefined, omniEnabled: settings.omni?.enabled ?? false, - omniUploadMaxFileBytes: settings.omni?.upload?.maxFileBytes, - omniMaxEstimatedTokens: settings.omni?.transport?.maxEstimatedTokens, - omniDownloadMaxFileBytes: settings.omni?.download?.maxFileBytes, - omniUploadCacheTtlHours: settings.omni?.upload?.cacheTtlHours, + omniMaxUploadFileBytes: + settings.omni?.processing?.transportGuard?.maxUploadFileBytes, + omniMaxEstimatedTokens: + settings.omni?.processing?.transportGuard?.maxEstimatedTokens, + omniUrlDownloadMaxFileBytes: + settings.omni?.ingestion?.localization?.url?.maxFileBytes, + omniUploadUrlTtlHours: settings.omni?.delivery?.upload?.urlTtlHours, // CDP tunnel (Plan C, #5626): with the tunnel on, browser automation goes // through the CDP tunnel (far lighter than the OS-level computer-use // driver), so disable computer-use to keep the agent off that heavy path. diff --git a/packages/cli/src/config/settingsSchema.ts b/packages/cli/src/config/settingsSchema.ts index 498a6111e6b..eed1d7a4313 100644 --- a/packages/cli/src/config/settingsSchema.ts +++ b/packages/cli/src/config/settingsSchema.ts @@ -3650,107 +3650,366 @@ const SETTINGS_SCHEMA = { 'endpoints. Can also be enabled via QWEN_CODE_ENABLE_OMNI=1.', showInDialog: true, }, - upload: { + processing: { type: 'object', - label: 'Omni Upload', + label: 'Omni Processing', category: 'Experimental', requiresRestart: true, default: {}, - description: 'Upload-channel limits for omni media delivery.', + description: + 'Media policy processing: fixed-policy orchestration, transport ' + + 'guard, per-root derivation limits, and policy tool overrides.', showInDialog: false, properties: { - maxFileBytes: { - type: 'number', - label: 'Max Upload File Bytes', + limits: { + type: 'object', + label: 'Omni Processing Limits', category: 'Experimental', requiresRestart: true, - default: 1073741824, + default: {}, description: - 'Per-file byte ceiling for omni media uploads. Defaults to ' + - '1 GiB, the DashScope temporary-upload per-file cap. Inputs ' + - 'above the limit fail closed with an explanatory error.', + 'Per-invocation derivation budgets. Exceeding a budget stops ' + + 'further derivation for that root resource (already committed ' + + 'artifacts stand).', showInDialog: false, - jsonSchemaOverride: { - type: 'number', - minimum: 1, - default: 1073741824, + properties: { + maxConcurrentResources: { + type: 'number', + label: 'Max Concurrent Resources', + category: 'Experimental', + requiresRestart: true, + default: 1, + description: + 'Number of media resources processed by policies in ' + + 'parallel within one request.', + showInDialog: false, + jsonSchemaOverride: { type: 'number', minimum: 1, default: 1 }, + }, + reservedOutputTokens: { + type: 'number', + label: 'Reserved Output Tokens', + category: 'Experimental', + requiresRestart: true, + default: 8192, + description: + 'Tokens reserved for model output when computing ' + + 'session.availableContextTokens for when-conditions.', + showInDialog: false, + jsonSchemaOverride: { + type: 'number', + minimum: 0, + default: 8192, + }, + }, + maxLineageDepth: { + type: 'number', + label: 'Max Lineage Depth', + category: 'Experimental', + requiresRestart: true, + default: 8, + description: + 'Maximum derivation chain length from a root resource.', + showInDialog: false, + jsonSchemaOverride: { type: 'number', minimum: 1, default: 8 }, + }, + maxPolicyRunsPerRoot: { + type: 'number', + label: 'Max Policy Runs Per Root', + category: 'Experimental', + requiresRestart: true, + default: 64, + description: + 'Maximum policy invocations attributable to one root ' + + 'resource within a single orchestrator run.', + showInDialog: false, + jsonSchemaOverride: { type: 'number', minimum: 1, default: 64 }, + }, + maxArtifactsPerRoot: { + type: 'number', + label: 'Max Artifacts Per Root', + category: 'Experimental', + requiresRestart: true, + default: 256, + description: + 'Maximum derived artifacts attributable to one root ' + + 'resource within a single orchestrator run.', + showInDialog: false, + jsonSchemaOverride: { + type: 'number', + minimum: 1, + default: 256, + }, + }, + maxDerivedBytesPerRoot: { + type: 'number', + label: 'Max Derived Bytes Per Root', + category: 'Experimental', + requiresRestart: true, + default: 1073741824, + description: + 'Byte budget for derived artifacts per root resource ' + + 'within a single orchestrator run. Defaults to 1 GiB.', + showInDialog: false, + jsonSchemaOverride: { + type: 'number', + minimum: 1, + default: 1073741824, + }, + }, + maxTransportPasses: { + type: 'number', + label: 'Max Transport Passes', + category: 'Experimental', + requiresRestart: true, + default: 3, + description: + 'Maximum transport-guard policy passes per resource before ' + + 'the media is removed with an explicit omission note.', + showInDialog: false, + jsonSchemaOverride: { type: 'number', minimum: 1, default: 3 }, + }, }, }, - cacheTtlHours: { - type: 'number', - label: 'Upload Cache TTL (hours)', + fixedPolicies: { + type: 'object', + label: 'Omni Fixed Policies', category: 'Experimental', requiresRestart: true, - default: 47, + default: {} as Record | null>, description: - 'Validity horizon for cached oss:// upload URLs. DashScope ' + - 'temporary uploads live 48h; the default keeps a 1h margin. ' + - '0 disables the upload cache (every delivery re-uploads).', + 'User fixed policies keyed by policy id. Merged with system ' + + 'defaults by id (whole-entry replacement); null tombstones a ' + + 'default policy. Validated and normalized at startup.', showInDialog: false, - jsonSchemaOverride: { - type: 'number', - minimum: 0, - default: 47, + mergeStrategy: MergeStrategy.SHALLOW_MERGE, + }, + transportGuard: { + type: 'object', + label: 'Omni Transport Guard', + category: 'Experimental', + requiresRestart: true, + default: {}, + description: + 'Delivery-boundary enforcement: hard limits plus mandatory ' + + 'guard policies applied when the final delivery set still ' + + 'exceeds limits. Cannot be disabled.', + showInDialog: false, + properties: { + maxUploadFileBytes: { + type: 'number', + label: 'Max Upload File Bytes', + category: 'Experimental', + requiresRestart: true, + default: 1073741824, + description: + 'Per-file byte ceiling for omni media uploads. Defaults ' + + 'to 1 GiB, the DashScope temporary-upload per-file cap ' + + '(values above it are a startup configuration error). ' + + 'Media still above the limit after guard policies fail ' + + 'closed with an explanatory error.', + showInDialog: false, + jsonSchemaOverride: { + type: 'number', + minimum: 1, + maximum: 1073741824, + default: 1073741824, + }, + }, + maxEstimatedTokens: { + type: 'number', + label: 'Max Estimated Tokens', + category: 'Experimental', + requiresRestart: true, + default: 0, + description: + 'Estimated-token ceiling for a single omni media input, ' + + 'checked at the delivery boundary using the versioned ' + + 'raw-resource estimator. 0 disables the token guard — the ' + + 'estimation formula is pending confirmation with the ' + + 'model provider; set a positive threshold to enforce ' + + 'fail-closed rejection.', + showInDialog: false, + jsonSchemaOverride: { + type: 'number', + minimum: 0, + default: 0, + }, + }, + policies: { + type: 'object', + label: 'Omni Transport Guard Policies', + category: 'Experimental', + requiresRestart: true, + default: {} as Record | null>, + description: + 'Guard policies keyed by policy id, run only when the ' + + 'final delivery set exceeds transport limits. Merged with ' + + 'system defaults by id. The merged set must cover image, ' + + 'video, and audio and must not be empty; every policy ' + + 'output must use source: omit.', + showInDialog: false, + mergeStrategy: MergeStrategy.SHALLOW_MERGE, + }, }, }, + policyTools: { + type: 'object', + label: 'Omni Policy Tools', + category: 'Experimental', + requiresRestart: true, + default: {} as Record | null>, + description: + 'Per-tool overrides keyed by policy tool name: settings ' + + '(default arguments), runtime (timeoutMs, maxConcurrency), ' + + 'and modelAccess (enabled, defaultArguments, lockedArguments, ' + + 'parameterSchema, output).', + showInDialog: false, + mergeStrategy: MergeStrategy.SHALLOW_MERGE, + }, }, }, - transport: { + delivery: { type: 'object', - label: 'Omni Transport Guard', + label: 'Omni Delivery', category: 'Experimental', requiresRestart: true, default: {}, - description: - 'Transport guard dimensions beyond the byte ceiling for omni ' + - 'media delivery.', + description: 'Model-delivery settings for omni media.', showInDialog: false, properties: { - maxEstimatedTokens: { - type: 'number', - label: 'Max Estimated Tokens', + upload: { + type: 'object', + label: 'Omni Delivery Upload', category: 'Experimental', requiresRestart: true, - default: 0, - description: - 'Estimated-token ceiling for a single omni media input, ' + - 'checked before upload using the versioned raw-resource ' + - 'estimator. 0 disables the token guard — the estimation ' + - 'formula is pending confirmation with the model provider; ' + - 'set a positive threshold to enforce fail-closed rejection.', + default: {}, + description: 'Upload-channel delivery settings.', showInDialog: false, - jsonSchemaOverride: { - type: 'number', - minimum: 0, - default: 0, + properties: { + urlTtlHours: { + type: 'number', + label: 'Upload URL TTL (hours)', + category: 'Experimental', + requiresRestart: true, + default: 47, + description: + 'Validity horizon for cached oss:// upload URLs. ' + + 'DashScope temporary uploads live 48h; the default keeps ' + + 'a 1h margin. 0 disables the upload cache (every ' + + 'delivery re-uploads).', + showInDialog: false, + jsonSchemaOverride: { + type: 'number', + minimum: 0, + default: 47, + }, + }, }, }, }, }, - download: { + ingestion: { type: 'object', - label: 'Omni Download', + label: 'Omni Ingestion', category: 'Experimental', requiresRestart: true, default: {}, - description: 'URL media localization limits for omni delivery.', + description: 'Media input ingestion settings for omni delivery.', showInDialog: false, properties: { - maxFileBytes: { - type: 'number', - label: 'Max Download File Bytes', + localization: { + type: 'object', + label: 'Omni Ingestion Localization', category: 'Experimental', requiresRestart: true, - default: 0, + default: {}, + description: 'Remote-media localization settings.', + showInDialog: false, + properties: { + url: { + type: 'object', + label: 'Omni URL Localization', + category: 'Experimental', + requiresRestart: true, + default: {}, + description: 'URL media download settings.', + showInDialog: false, + properties: { + maxFileBytes: { + type: 'number', + label: 'Max Download File Bytes', + category: 'Experimental', + requiresRestart: true, + default: 0, + description: + 'Byte ceiling for downloading URL media inputs. 0 or ' + + 'unset follows ' + + 'omni.processing.transportGuard.maxUploadFileBytes ' + + '(downloading more than the upload channel can ' + + 'deliver is pointless).', + showInDialog: false, + jsonSchemaOverride: { + type: 'number', + minimum: 0, + default: 0, + }, + }, + }, + }, + }, + }, + }, + }, + storage: { + type: 'object', + label: 'Omni Storage', + category: 'Experimental', + requiresRestart: true, + default: {}, + description: 'Managed storage settings under .qwen/omni/.', + showInDialog: false, + properties: { + quarantine: { + type: 'object', + label: 'Omni Quarantine', + category: 'Experimental', + requiresRestart: true, + default: {}, description: - 'Byte ceiling for downloading URL media inputs. 0 or unset ' + - 'follows omni.upload.maxFileBytes (downloading more than the ' + - 'upload channel can deliver is pointless).', + 'Retention for failed policy invocations moved to ' + + '.qwen/omni/quarantine/ for diagnosis. Quarantined content ' + + 'is never recalled into recognition or delivery.', showInDialog: false, - jsonSchemaOverride: { - type: 'number', - minimum: 0, - default: 0, + properties: { + retentionDays: { + type: 'number', + label: 'Quarantine Retention (days)', + category: 'Experimental', + requiresRestart: true, + default: 7, + description: + 'Days a quarantined invocation directory is kept before ' + + 'startup recovery removes it.', + showInDialog: false, + jsonSchemaOverride: { type: 'number', minimum: 0, default: 7 }, + }, + maxBytes: { + type: 'number', + label: 'Quarantine Max Bytes', + category: 'Experimental', + requiresRestart: true, + default: 5368709120, + description: + 'Total byte budget for the quarantine directory. Startup ' + + 'recovery removes oldest entries first until within ' + + 'budget. Defaults to 5 GiB.', + showInDialog: false, + jsonSchemaOverride: { + type: 'number', + minimum: 0, + default: 5368709120, + }, + }, }, }, }, diff --git a/packages/core/src/config/config.ts b/packages/core/src/config/config.ts index 0d36e74a7f3..76feb152df6 100644 --- a/packages/core/src/config/config.ts +++ b/packages/core/src/config/config.ts @@ -1103,13 +1103,13 @@ export interface ConfigParameters { * pipeline (omni-experiment branch). */ omniEnabled?: boolean; /** Per-file byte ceiling for omni media uploads (default 1 GiB). */ - omniUploadMaxFileBytes?: number; + omniMaxUploadFileBytes?: number; /** Estimated-token ceiling for omni media (0/unset = guard disabled). */ omniMaxEstimatedTokens?: number; /** Byte ceiling for omni URL downloads (unset = follow upload cap). */ - omniDownloadMaxFileBytes?: number; - /** Upload cache TTL in hours (0 disables the cache; default 47). */ - omniUploadCacheTtlHours?: number; + omniUrlDownloadMaxFileBytes?: number; + /** Upload URL TTL in hours (0 disables the cache; default 47). */ + omniUploadUrlTtlHours?: number; /** Image generation model selected through `/model --image`. */ imageModel?: string; /** @@ -1936,10 +1936,10 @@ export class Config { private readonly artifactHost?: ArtifactHostConfig; private readonly artifactOss?: ArtifactOssConfig; private readonly omniEnabled: boolean = false; - private readonly omniUploadMaxFileBytes?: number; + private readonly omniMaxUploadFileBytes?: number; private readonly omniMaxEstimatedTokens?: number; - private readonly omniDownloadMaxFileBytes?: number; - private readonly omniUploadCacheTtlHours?: number; + private readonly omniUrlDownloadMaxFileBytes?: number; + private readonly omniUploadUrlTtlHours?: number; private workflowsEnabled = false; private readonly skipWorkflowUsageWarning: boolean = false; private readonly computerUseEnabled: boolean = true; @@ -2216,10 +2216,10 @@ export class Config { this.artifactHost = params.artifactHost; this.artifactOss = params.artifactOss; this.omniEnabled = params.omniEnabled ?? false; - this.omniUploadMaxFileBytes = params.omniUploadMaxFileBytes; + this.omniMaxUploadFileBytes = params.omniMaxUploadFileBytes; this.omniMaxEstimatedTokens = params.omniMaxEstimatedTokens; - this.omniDownloadMaxFileBytes = params.omniDownloadMaxFileBytes; - this.omniUploadCacheTtlHours = params.omniUploadCacheTtlHours; + this.omniUrlDownloadMaxFileBytes = params.omniUrlDownloadMaxFileBytes; + this.omniUploadUrlTtlHours = params.omniUploadUrlTtlHours; this.workflowsEnabled = params.workflowsEnabled ?? false; this.skipWorkflowUsageWarning = params.skipWorkflowUsageWarning ?? false; this.computerUseEnabled = params.computerUseEnabled ?? true; @@ -6380,20 +6380,20 @@ export class Config { return this.omniEnabled; } - getOmniUploadMaxFileBytes(): number | undefined { - return this.omniUploadMaxFileBytes; + getOmniMaxUploadFileBytes(): number | undefined { + return this.omniMaxUploadFileBytes; } getOmniMaxEstimatedTokens(): number | undefined { return this.omniMaxEstimatedTokens; } - getOmniDownloadMaxFileBytes(): number | undefined { - return this.omniDownloadMaxFileBytes; + getOmniUrlDownloadMaxFileBytes(): number | undefined { + return this.omniUrlDownloadMaxFileBytes; } - getOmniUploadCacheTtlHours(): number | undefined { - return this.omniUploadCacheTtlHours; + getOmniUploadUrlTtlHours(): number | undefined { + return this.omniUploadUrlTtlHours; } resolveImageGenerationModel( diff --git a/packages/core/src/omni/guard.test.ts b/packages/core/src/omni/guard.test.ts index d1b1d825374..dc18999788b 100644 --- a/packages/core/src/omni/guard.test.ts +++ b/packages/core/src/omni/guard.test.ts @@ -16,7 +16,7 @@ import type { RecognizedMedia } from './recognition.js'; function cfg(overrides: { maxBytes?: number; maxTokens?: number }): Config { return { - getOmniUploadMaxFileBytes: vi.fn().mockReturnValue(overrides.maxBytes), + getOmniMaxUploadFileBytes: vi.fn().mockReturnValue(overrides.maxBytes), getOmniMaxEstimatedTokens: vi.fn().mockReturnValue(overrides.maxTokens), } as unknown as Config; } @@ -53,7 +53,9 @@ describe('byte guard', () => { it('rejects above the configured limit with an explanatory message', () => { expect(() => assertWithinByteLimit(cfg({ maxBytes: 1000 }), 2000, 'clip.mp4'), - ).toThrow(/clip\.mp4.*2000 bytes > 1000 bytes.*omni\.upload\.maxFileBytes/); + ).toThrow( + /clip\.mp4.*2000 bytes > 1000 bytes.*omni\.processing\.transportGuard\.maxUploadFileBytes/, + ); }); it('passes at or below the limit', () => { @@ -81,7 +83,7 @@ describe('token guard', () => { expect(() => assertWithinTokenLimit(cfg({ maxTokens: 196_608 }), VIDEO_8MIN, 'p3.mp4'), ).toThrow( - /p3\.mp4.*raw-resource-v1.*196608.*omni\.transport\.maxEstimatedTokens/, + /p3\.mp4.*raw-resource-v1.*196608.*omni\.processing\.transportGuard\.maxEstimatedTokens/, ); }); diff --git a/packages/core/src/omni/guard.ts b/packages/core/src/omni/guard.ts index d1fb9ee2069..820ce706d48 100644 --- a/packages/core/src/omni/guard.ts +++ b/packages/core/src/omni/guard.ts @@ -26,7 +26,7 @@ export class OmniTransportGuardError extends Error { /** Resolve the effective byte ceiling (undefined/<=0 config → default). */ export function effectiveMaxUploadFileBytes(config: Config): number { - const configured = config.getOmniUploadMaxFileBytes?.(); + const configured = config.getOmniMaxUploadFileBytes?.(); return configured !== undefined && configured > 0 ? configured : DEFAULT_OMNI_MAX_UPLOAD_FILE_BYTES; @@ -44,7 +44,7 @@ export function assertWithinByteLimit( if (sizeBytes > maxBytes) { throw new OmniTransportGuardError( `${displayName} exceeds the omni upload limit: ${sizeBytes} bytes > ` + - `${maxBytes} bytes (omni.upload.maxFileBytes). ` + + `${maxBytes} bytes (omni.processing.transportGuard.maxUploadFileBytes). ` + `Reduce the file size before retrying.`, ); } @@ -55,7 +55,7 @@ export function assertWithinByteLimit( * BEFORE store/upload, so an oversized input costs one probe — not a copy * and a multi-minute upload. * - * Threshold semantics (`omni.transport.maxEstimatedTokens`): + * Threshold semantics (`omni.processing.transportGuard.maxEstimatedTokens`): * - unset / 0 / negative → guard disabled (the estimation formula is still * pending confirmation with the model provider; estimates are attached * for observability but must not reject until a threshold is set); @@ -77,7 +77,7 @@ export function assertWithinTokenLimit( throw new OmniTransportGuardError( `${displayName} exceeds the omni estimated-token limit: ` + `~${estimate.estimatedTokenCount} tokens (${estimate.method}) > ` + - `${maxTokens} (omni.transport.maxEstimatedTokens). ` + + `${maxTokens} (omni.processing.transportGuard.maxEstimatedTokens). ` + `Reduce duration/resolution or raise the limit.`, ); } diff --git a/packages/core/src/omni/index.test.ts b/packages/core/src/omni/index.test.ts index 8bfa8577b5e..8688a712250 100644 --- a/packages/core/src/omni/index.test.ts +++ b/packages/core/src/omni/index.test.ts @@ -72,8 +72,8 @@ describe('sanitizeErrorMessage', () => { describe('effectiveMaxDownloadFileBytes', () => { const capsConfig = (download?: number, upload?: number): Config => ({ - getOmniDownloadMaxFileBytes: () => download, - getOmniUploadMaxFileBytes: () => upload, + getOmniUrlDownloadMaxFileBytes: () => download, + getOmniMaxUploadFileBytes: () => upload, }) as unknown as Config; it('never exceeds the upload cap, even when configured higher', () => { @@ -186,7 +186,7 @@ describe('readMediaViaOmniDelivery result shape', () => { isTrustedFolder: vi.fn().mockReturnValue(true), getContentGeneratorConfig: vi.fn().mockReturnValue(DASHSCOPE_CGC), getModel: vi.fn().mockReturnValue('qwen3.5-omni-plus'), - getOmniUploadMaxFileBytes: vi.fn().mockReturnValue(0), + getOmniMaxUploadFileBytes: vi.fn().mockReturnValue(0), getOmniMaxEstimatedTokens: vi.fn().mockReturnValue(0), storage: { getQwenDir: () => '/tmp/omni-test-qwen' }, } as unknown as Config; @@ -540,9 +540,9 @@ describe('processMediaForOmniDelivery upload cache integration', () => { .fn() .mockReturnValue(overrides?.cgc ?? DASHSCOPE_CGC), getModel: vi.fn().mockReturnValue('qwen3.5-omni-plus'), - getOmniUploadMaxFileBytes: vi.fn().mockReturnValue(0), + getOmniMaxUploadFileBytes: vi.fn().mockReturnValue(0), getOmniMaxEstimatedTokens: vi.fn().mockReturnValue(0), - getOmniUploadCacheTtlHours: vi.fn().mockReturnValue(overrides?.ttlHours), + getOmniUploadUrlTtlHours: vi.fn().mockReturnValue(overrides?.ttlHours), storage: { getQwenDir: () => tmpDir }, } as unknown as Config; } diff --git a/packages/core/src/omni/index.ts b/packages/core/src/omni/index.ts index b7fbca799df..89174de96bf 100644 --- a/packages/core/src/omni/index.ts +++ b/packages/core/src/omni/index.ts @@ -294,7 +294,7 @@ export async function processMediaForOmniDelivery( .update(`${cgc.baseUrl ?? ''}|${cgc.apiKey ?? ''}`) .digest('hex') .slice(0, 16); - const configuredTtl = config.getOmniUploadCacheTtlHours?.(); + const configuredTtl = config.getOmniUploadUrlTtlHours?.(); const uploadCache = new OmniUploadCache( store.getOmniRootDir(), configuredTtl === undefined @@ -487,10 +487,11 @@ export async function readMediaViaOmniDelivery(params: { /** Effective download byte ceiling — never above the upload channel cap * (downloading more than can be delivered is pointless), including when - * `omni.download.maxFileBytes` is explicitly configured higher. */ + * `omni.ingestion.localization.url.maxFileBytes` is explicitly configured + * higher. */ export function effectiveMaxDownloadFileBytes(config: Config): number { const uploadCap = effectiveMaxUploadFileBytes(config); - const configured = config.getOmniDownloadMaxFileBytes?.(); + const configured = config.getOmniUrlDownloadMaxFileBytes?.(); if (configured !== undefined && configured > 0) { return Math.min(configured, uploadCap); } diff --git a/packages/core/src/utils/fileUtils.ts b/packages/core/src/utils/fileUtils.ts index e0687583c5c..b7bfd39b363 100644 --- a/packages/core/src/utils/fileUtils.ts +++ b/packages/core/src/utils/fileUtils.ts @@ -1292,7 +1292,7 @@ export async function processSingleFileContent( // 100 MB source cap protects the overview DECODER, so it only applies // when the overview will actually decode — i.e. when omni is not taking // this file. The omni path uploads original bytes without decoding and - // enforces its own omni.upload.maxFileBytes ceiling (1 GiB default); + // enforces its own maxUploadFileBytes ceiling (1 GiB default); // gating it here too would reject a 150 MB PNG while delivering a // 500 MB GIF, purely on whether the format has an overview renderer. if ( diff --git a/packages/vscode-ide-companion/schemas/settings.schema.json b/packages/vscode-ide-companion/schemas/settings.schema.json index 8dcdf71cec2..4ec6084db13 100644 --- a/packages/vscode-ide-companion/schemas/settings.schema.json +++ b/packages/vscode-ide-companion/schemas/settings.schema.json @@ -3268,45 +3268,157 @@ "type": "boolean", "default": false }, - "upload": { - "description": "Upload-channel limits for omni media delivery.", + "processing": { + "description": "Media policy processing: fixed-policy orchestration, transport guard, per-root derivation limits, and policy tool overrides.", "type": "object", "properties": { - "maxFileBytes": { - "type": "number", - "minimum": 1, - "default": 1073741824, - "description": "Per-file byte ceiling for omni media uploads. Defaults to 1 GiB, the DashScope temporary-upload per-file cap. Inputs above the limit fail closed with an explanatory error." + "limits": { + "description": "Per-invocation derivation budgets. Exceeding a budget stops further derivation for that root resource (already committed artifacts stand).", + "type": "object", + "properties": { + "maxConcurrentResources": { + "type": "number", + "minimum": 1, + "default": 1, + "description": "Number of media resources processed by policies in parallel within one request." + }, + "reservedOutputTokens": { + "type": "number", + "minimum": 0, + "default": 8192, + "description": "Tokens reserved for model output when computing session.availableContextTokens for when-conditions." + }, + "maxLineageDepth": { + "type": "number", + "minimum": 1, + "default": 8, + "description": "Maximum derivation chain length from a root resource." + }, + "maxPolicyRunsPerRoot": { + "type": "number", + "minimum": 1, + "default": 64, + "description": "Maximum policy invocations attributable to one root resource within a single orchestrator run." + }, + "maxArtifactsPerRoot": { + "type": "number", + "minimum": 1, + "default": 256, + "description": "Maximum derived artifacts attributable to one root resource within a single orchestrator run." + }, + "maxDerivedBytesPerRoot": { + "type": "number", + "minimum": 1, + "default": 1073741824, + "description": "Byte budget for derived artifacts per root resource within a single orchestrator run. Defaults to 1 GiB." + }, + "maxTransportPasses": { + "type": "number", + "minimum": 1, + "default": 3, + "description": "Maximum transport-guard policy passes per resource before the media is removed with an explicit omission note." + } + } }, - "cacheTtlHours": { - "type": "number", - "minimum": 0, - "default": 47, - "description": "Validity horizon for cached oss:// upload URLs. DashScope temporary uploads live 48h; the default keeps a 1h margin. 0 disables the upload cache (every delivery re-uploads)." + "fixedPolicies": { + "description": "User fixed policies keyed by policy id. Merged with system defaults by id (whole-entry replacement); null tombstones a default policy. Validated and normalized at startup.", + "type": "object", + "additionalProperties": true + }, + "transportGuard": { + "description": "Delivery-boundary enforcement: hard limits plus mandatory guard policies applied when the final delivery set still exceeds limits. Cannot be disabled.", + "type": "object", + "properties": { + "maxUploadFileBytes": { + "type": "number", + "minimum": 1, + "maximum": 1073741824, + "default": 1073741824, + "description": "Per-file byte ceiling for omni media uploads. Defaults to 1 GiB, the DashScope temporary-upload per-file cap (values above it are a startup configuration error). Media still above the limit after guard policies fail closed with an explanatory error." + }, + "maxEstimatedTokens": { + "type": "number", + "minimum": 0, + "default": 0, + "description": "Estimated-token ceiling for a single omni media input, checked at the delivery boundary using the versioned raw-resource estimator. 0 disables the token guard — the estimation formula is pending confirmation with the model provider; set a positive threshold to enforce fail-closed rejection." + }, + "policies": { + "description": "Guard policies keyed by policy id, run only when the final delivery set exceeds transport limits. Merged with system defaults by id. The merged set must cover image, video, and audio and must not be empty; every policy output must use source: omit.", + "type": "object", + "additionalProperties": true + } + } + }, + "policyTools": { + "description": "Per-tool overrides keyed by policy tool name: settings (default arguments), runtime (timeoutMs, maxConcurrency), and modelAccess (enabled, defaultArguments, lockedArguments, parameterSchema, output).", + "type": "object", + "additionalProperties": true } } }, - "transport": { - "description": "Transport guard dimensions beyond the byte ceiling for omni media delivery.", + "delivery": { + "description": "Model-delivery settings for omni media.", "type": "object", "properties": { - "maxEstimatedTokens": { - "type": "number", - "minimum": 0, - "default": 0, - "description": "Estimated-token ceiling for a single omni media input, checked before upload using the versioned raw-resource estimator. 0 disables the token guard — the estimation formula is pending confirmation with the model provider; set a positive threshold to enforce fail-closed rejection." + "upload": { + "description": "Upload-channel delivery settings.", + "type": "object", + "properties": { + "urlTtlHours": { + "type": "number", + "minimum": 0, + "default": 47, + "description": "Validity horizon for cached oss:// upload URLs. DashScope temporary uploads live 48h; the default keeps a 1h margin. 0 disables the upload cache (every delivery re-uploads)." + } + } } } }, - "download": { - "description": "URL media localization limits for omni delivery.", + "ingestion": { + "description": "Media input ingestion settings for omni delivery.", "type": "object", "properties": { - "maxFileBytes": { - "type": "number", - "minimum": 0, - "default": 0, - "description": "Byte ceiling for downloading URL media inputs. 0 or unset follows omni.upload.maxFileBytes (downloading more than the upload channel can deliver is pointless)." + "localization": { + "description": "Remote-media localization settings.", + "type": "object", + "properties": { + "url": { + "description": "URL media download settings.", + "type": "object", + "properties": { + "maxFileBytes": { + "type": "number", + "minimum": 0, + "default": 0, + "description": "Byte ceiling for downloading URL media inputs. 0 or unset follows omni.processing.transportGuard.maxUploadFileBytes (downloading more than the upload channel can deliver is pointless)." + } + } + } + } + } + } + }, + "storage": { + "description": "Managed storage settings under .qwen/omni/.", + "type": "object", + "properties": { + "quarantine": { + "description": "Retention for failed policy invocations moved to .qwen/omni/quarantine/ for diagnosis. Quarantined content is never recalled into recognition or delivery.", + "type": "object", + "properties": { + "retentionDays": { + "type": "number", + "minimum": 0, + "default": 7, + "description": "Days a quarantined invocation directory is kept before startup recovery removes it." + }, + "maxBytes": { + "type": "number", + "minimum": 0, + "default": 5368709120, + "description": "Total byte budget for the quarantine directory. Startup recovery removes oldest entries first until within budget. Defaults to 5 GiB." + } + } } } } From 11feda7a3603ae10fa96660fbd6ef5ae4ef7610f Mon Sep 17 00:00:00 2001 From: LaZzyMan Date: Thu, 6 Aug 2026 23:27:07 +0800 Subject: [PATCH 02/62] feat(omni): surface bitRate/sampleRateHz/channels from ffprobe ffprobe already reports these; the parse branches discarded them. The policy condition DSL (resource.bitRate / resource.sampleRateHz / resource.channels) and degradation disclosures both need the real numbers. Format-level bit_rate is preferred over the stream's. --- packages/core/src/omni/ffmpeg.test.ts | 60 +++++++++++++++++++++++++++ packages/core/src/omni/ffmpeg.ts | 27 +++++++++++- 2 files changed, 85 insertions(+), 2 deletions(-) diff --git a/packages/core/src/omni/ffmpeg.test.ts b/packages/core/src/omni/ffmpeg.test.ts index 72914799c2e..dd6930efae3 100644 --- a/packages/core/src/omni/ffmpeg.test.ts +++ b/packages/core/src/omni/ffmpeg.test.ts @@ -203,9 +203,69 @@ describe('probeMediaMetadata per-modality branches', () => { formatName: 'mov,mp4', durationMs: 12_500, codec: 'aac', + sampleRateHz: 44_100, + channels: 2, }); }); + it('prefers the format-level bit rate and falls back to the stream', async () => { + mockExecResult(() => ({ + stdout: JSON.stringify({ + format: { format_name: 'mp3', duration: '10', bit_rate: '320000' }, + streams: [ + { codec_type: 'audio', codec_name: 'mp3', bit_rate: '128000' }, + ], + }), + })); + await expect(probeMediaMetadata('/a.mp3', 'audio')).resolves.toMatchObject({ + bitRate: 320_000, + }); + + mockExecResult(() => ({ + stdout: JSON.stringify({ + format: { format_name: 'mp3', duration: '10' }, + streams: [ + { codec_type: 'audio', codec_name: 'mp3', bit_rate: '128000' }, + ], + }), + })); + await expect(probeMediaMetadata('/a.mp3', 'audio')).resolves.toMatchObject({ + bitRate: 128_000, + }); + }); + + it('reports the video bit rate for modality video', async () => { + mockExecResult(() => ({ + stdout: JSON.stringify({ + format: { format_name: 'mp4', duration: '5', bit_rate: '2500000' }, + streams: [{ codec_type: 'video', codec_name: 'h264' }], + }), + })); + await expect(probeMediaMetadata('/v.mp4', 'video')).resolves.toMatchObject({ + bitRate: 2_500_000, + }); + }); + + it('omits bitRate/sampleRateHz/channels when unusable', async () => { + mockExecResult(() => ({ + stdout: JSON.stringify({ + format: { format_name: 'wav', duration: '3', bit_rate: 'N/A' }, + streams: [ + { + codec_type: 'audio', + codec_name: 'pcm_s16le', + sample_rate: 'N/A', + channels: 0, + }, + ], + }), + })); + const result = await probeMediaMetadata('/a.wav', 'audio'); + expect(result.bitRate).toBeUndefined(); + expect(result.sampleRateHz).toBeUndefined(); + expect(result.channels).toBeUndefined(); + }); + it("reads only dimensions for modality 'image' (no duration)", async () => { mockExecResult(() => ({ stdout: JSON.stringify({ diff --git a/packages/core/src/omni/ffmpeg.ts b/packages/core/src/omni/ffmpeg.ts index 736fbfc5c7f..289b09e6f34 100644 --- a/packages/core/src/omni/ffmpeg.ts +++ b/packages/core/src/omni/ffmpeg.ts @@ -138,6 +138,12 @@ export interface MediaProbeResult { /** Frame count of the primary video stream (image: >1 means animated — * GIF/APNG/animated WebP; absent when the container does not report it). */ frameCount?: number; + /** Overall bit rate in bits/second (format-level; audio/video). */ + bitRate?: number; + /** Sample rate in Hz of the first audio stream (audio only). */ + sampleRateHz?: number; + /** Channel count of the first audio stream (audio only). */ + channels?: number; } /** Parse an ffprobe rational like "30000/1001" (or plain "25") into fps. */ @@ -188,7 +194,7 @@ export async function probeMediaMetadata( ); } let parsed: { - format?: { format_name?: string; duration?: string }; + format?: { format_name?: string; duration?: string; bit_rate?: string }; streams?: Array<{ codec_type?: string; codec_name?: string; @@ -197,6 +203,9 @@ export async function probeMediaMetadata( avg_frame_rate?: string; r_frame_rate?: string; nb_frames?: string; + sample_rate?: string; + channels?: number; + bit_rate?: string; }>; }; try { @@ -213,6 +222,14 @@ export async function probeMediaMetadata( Number.isFinite(durationSeconds) && durationSeconds >= 0 ? Math.round(durationSeconds * 1000) : undefined; + const parsePositiveInt = (raw: string | undefined): number | undefined => { + const n = Number(raw); + return Number.isFinite(n) && n > 0 ? Math.round(n) : undefined; + }; + // Prefer the format-level bit rate; fall back to the primary stream's. + const bitRateFor = (stream?: { bit_rate?: string }): number | undefined => + parsePositiveInt(parsed.format?.bit_rate) ?? + parsePositiveInt(stream?.bit_rate); const base: MediaProbeResult = { formatName: parsed.format?.format_name }; switch (modality) { @@ -232,12 +249,17 @@ export async function probeMediaMetadata( : {}), }; } - case 'audio': + case 'audio': { + const channels = audioStream?.channels; return { ...base, durationMs, codec: audioStream?.codec_name, + bitRate: bitRateFor(audioStream), + sampleRateHz: parsePositiveInt(audioStream?.sample_rate), + ...(typeof channels === 'number' && channels > 0 ? { channels } : {}), }; + } case 'video': default: return { @@ -249,6 +271,7 @@ export async function probeMediaMetadata( parseFrameRate(videoStream?.avg_frame_rate) ?? parseFrameRate(videoStream?.r_frame_rate), codec: videoStream?.codec_name, + bitRate: bitRateFor(videoStream), }; } } From 9c328a45c02c3272e39fc782bee86687e6787729 Mon Sep 17 00:00:00 2001 From: LaZzyMan Date: Fri, 7 Aug 2026 00:10:58 +0800 Subject: [PATCH 03/62] feat(omni): add media-policy tool protocol (execution origin, descriptor, modelAccess gate) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Protocol core for the S4 policy pipeline: - ToolExecutionOrigin (model | client | fixed_policy) on ToolCallRequestInfo; only settable by in-process callers, never deserialized from protocol payloads; missing origin fails closed as model. - MediaPolicyToolDescriptor as a DeclarativeTool code-registration getter (default undefined) — config can never turn an ordinary tool into a policy tool. - PolicyArtifactBatch channel on ToolCallResponseInfo, capturing raw successful media-policy artifacts before PostToolUse hook merging; error/timeout results promote nothing. - Shared modelAccess resolver + call gate (omni/policy/model-access.ts): media-policy tools are fixed-policy-only unless omni.processing.policyTools..modelAccess.enabled; model calls get defaultArguments/lockedArguments merged and are rejected when they name a locked key; a forged fixed_policy origin on a non-media-policy tool is rejected. - Gate enforced on every model surface: registry declaration lists (incl. includeDeferred for subagents), ToolSearch keyword + select:, CoreToolScheduler pre-build, and ACP Session.runTool. - fixed_policy scheduler calls skip the interactive permission flow but keep PermissionManager tool-enablement and hook execution. - omni.processing.policyTools threaded through ConfigParameters (core + cli). --- .../src/acp-integration/session/Session.ts | 23 + packages/cli/src/config/config.ts | 4 + packages/core/src/config/config.ts | 10 + .../coreToolScheduler.mediaPolicy.test.ts | 456 ++++++++++++++++++ packages/core/src/core/coreToolScheduler.ts | 71 ++- packages/core/src/core/turn.ts | 55 +++ packages/core/src/index.ts | 13 + .../core/src/omni/policy/model-access.test.ts | 305 ++++++++++++ packages/core/src/omni/policy/model-access.ts | 176 +++++++ packages/core/src/omni/policy/types.ts | 66 +++ packages/core/src/tools/tool-registry.test.ts | 73 +++ packages/core/src/tools/tool-registry.ts | 32 +- packages/core/src/tools/tool-search.test.ts | 91 +++- packages/core/src/tools/tool-search.ts | 23 +- packages/core/src/tools/tools.ts | 44 ++ 15 files changed, 1425 insertions(+), 17 deletions(-) create mode 100644 packages/core/src/core/coreToolScheduler.mediaPolicy.test.ts create mode 100644 packages/core/src/omni/policy/model-access.test.ts create mode 100644 packages/core/src/omni/policy/model-access.ts create mode 100644 packages/core/src/omni/policy/types.ts diff --git a/packages/cli/src/acp-integration/session/Session.ts b/packages/cli/src/acp-integration/session/Session.ts index a34de78dceb..614ef7b0a26 100644 --- a/packages/cli/src/acp-integration/session/Session.ts +++ b/packages/cli/src/acp-integration/session/Session.ts @@ -162,6 +162,7 @@ import { runWithRuntimeContentGenerator, getInvocationContext, runWithInvocationContext, + evaluateMediaPolicyToolCall, } from '@qwen-code/qwen-code-core'; import { NOT_CURRENTLY_GENERATING_CANCEL_MESSAGE } from '@qwen-code/acp-bridge/bridgeErrors'; // Single source of truth shared with the daemon-side answerer (BridgeClient), @@ -7222,6 +7223,28 @@ export class Session implements SessionContext { ); } + // ---- Media-policy modelAccess gate (mirrors CoreToolScheduler) ---- + // Every ACP-originated call is a model call: there is no in-process + // fixed_policy caller on this path, so the origin is pinned rather + // than read from the (untrusted) protocol payload. + const mediaPolicyGate = evaluateMediaPolicyToolCall({ + config: this.config, + tool, + args, + executionOrigin: { kind: 'model' }, + }); + if (mediaPolicyGate.outcome === 'reject') { + return earlyErrorResponse( + new Error(mediaPolicyGate.message), + toolName, + { + recordInvalidToolParams: + mediaPolicyGate.reason === 'invalid_params', + }, + ); + } + args = mediaPolicyGate.args; + // Detect TodoWriteTool early - route to plan updates instead of tool_call events const isTodoWriteTool = tool.name === ToolNames.TODO_WRITE; // Core exposes TodoWriteTool as a type only. The bundle's keepNames diff --git a/packages/cli/src/config/config.ts b/packages/cli/src/config/config.ts index d128dfefa41..7863c3c9203 100755 --- a/packages/cli/src/config/config.ts +++ b/packages/cli/src/config/config.ts @@ -38,6 +38,7 @@ import { SchemaValidator, type ConfigParameters, type MCPServerConfig, + type OmniPolicyToolsSettings, type WebSearchSettings, MAX_SUBAGENT_DEPTH_LIMIT, } from '@qwen-code/qwen-code-core'; @@ -2199,6 +2200,9 @@ export async function loadCliConfig( omniUrlDownloadMaxFileBytes: settings.omni?.ingestion?.localization?.url?.maxFileBytes, omniUploadUrlTtlHours: settings.omni?.delivery?.upload?.urlTtlHours, + omniPolicyTools: settings.omni?.processing?.policyTools as + | OmniPolicyToolsSettings + | undefined, // CDP tunnel (Plan C, #5626): with the tunnel on, browser automation goes // through the CDP tunnel (far lighter than the OS-level computer-use // driver), so disable computer-use to keep the agent off that heavy path. diff --git a/packages/core/src/config/config.ts b/packages/core/src/config/config.ts index 76feb152df6..7698ec5a460 100644 --- a/packages/core/src/config/config.ts +++ b/packages/core/src/config/config.ts @@ -28,6 +28,7 @@ import { selectVisionBridgeModel, } from '../services/visionBridge/vision-bridge-service.js'; import type { AnyToolInvocation } from '../tools/tools.js'; +import type { OmniPolicyToolsSettings } from '../omni/policy/types.js'; import type { ArenaManager } from '../agents/arena/ArenaManager.js'; import { ArenaAgentClient } from '../agents/arena/ArenaAgentClient.js'; import type { TeamManager } from '../agents/team/TeamManager.js'; @@ -1110,6 +1111,9 @@ export interface ConfigParameters { omniUrlDownloadMaxFileBytes?: number; /** Upload URL TTL in hours (0 disables the cache; default 47). */ omniUploadUrlTtlHours?: number; + /** Raw `omni.processing.policyTools` map (per-tool settings/runtime/ + * modelAccess). Normalized lazily by the omni policy modules. */ + omniPolicyTools?: OmniPolicyToolsSettings; /** Image generation model selected through `/model --image`. */ imageModel?: string; /** @@ -1940,6 +1944,7 @@ export class Config { private readonly omniMaxEstimatedTokens?: number; private readonly omniUrlDownloadMaxFileBytes?: number; private readonly omniUploadUrlTtlHours?: number; + private readonly omniPolicyTools?: OmniPolicyToolsSettings; private workflowsEnabled = false; private readonly skipWorkflowUsageWarning: boolean = false; private readonly computerUseEnabled: boolean = true; @@ -2220,6 +2225,7 @@ export class Config { this.omniMaxEstimatedTokens = params.omniMaxEstimatedTokens; this.omniUrlDownloadMaxFileBytes = params.omniUrlDownloadMaxFileBytes; this.omniUploadUrlTtlHours = params.omniUploadUrlTtlHours; + this.omniPolicyTools = params.omniPolicyTools; this.workflowsEnabled = params.workflowsEnabled ?? false; this.skipWorkflowUsageWarning = params.skipWorkflowUsageWarning ?? false; this.computerUseEnabled = params.computerUseEnabled ?? true; @@ -6396,6 +6402,10 @@ export class Config { return this.omniUploadUrlTtlHours; } + getOmniPolicyToolsSettings(): OmniPolicyToolsSettings | undefined { + return this.omniPolicyTools; + } + resolveImageGenerationModel( setting: string | undefined, ): ImageGenerationConfig | undefined { diff --git a/packages/core/src/core/coreToolScheduler.mediaPolicy.test.ts b/packages/core/src/core/coreToolScheduler.mediaPolicy.test.ts new file mode 100644 index 00000000000..e39edfc4b7b --- /dev/null +++ b/packages/core/src/core/coreToolScheduler.mediaPolicy.test.ts @@ -0,0 +1,456 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Scheduler wiring tests for the omni media-policy protocol: + * modelAccess gate at schedule time, fixed-policy permission bypass, + * and raw policy-artifact capture on the success response. + */ + +import { describe, it, expect, vi } from 'vitest'; +import type { Mock } from 'vitest'; +import { executeToolCall } from './nonInteractiveToolExecutor.js'; +import type { + Config, + MediaPolicyToolDescriptor, + OmniPolicyToolsSettings, + ToolCallRequestInfo, + ToolRegistry, + ToolResult, +} from '../index.js'; +import { + ApprovalMode, + CoreToolScheduler, + DEFAULT_TRUNCATE_TOOL_OUTPUT_LINES, + DEFAULT_TRUNCATE_TOOL_OUTPUT_THRESHOLD, + ToolErrorType, +} from '../index.js'; +import type { ToolCall } from './coreToolScheduler.js'; +import { MockTool } from '../test-utils/mock-tool.js'; + +const DESCRIPTOR: MediaPolicyToolDescriptor = { + kind: 'media_policy', + inputMediaTypes: ['image'], + outputs: [{ kind: 'media', required: true }], +}; + +/** MockTool that reports itself as a media-policy tool (code-registration + * fact — the descriptor getter, never configuration). */ +class MockMediaPolicyTool extends MockTool { + override get mediaPolicyDescriptor(): MediaPolicyToolDescriptor { + return DESCRIPTOR; + } +} + +const FIXED_ORIGIN = { + kind: 'fixed_policy', + policyId: 'image-compress-v1', + stage: 'preprocessing', +} as const; + +function makeConfig(options: { + tool: MockTool; + omniPolicyTools?: OmniPolicyToolsSettings; + approvalMode?: ApprovalMode; + interactive?: boolean; + isToolEnabled?: (name: string) => Promise; +}): Config { + const mockToolRegistry = { + getTool: (name: string) => + name === options.tool.name ? options.tool : undefined, + ensureTool: async (name: string) => + name === options.tool.name ? options.tool : undefined, + getToolByName: (name: string) => + name === options.tool.name ? options.tool : undefined, + getAllToolNames: () => [options.tool.name], + getFunctionDeclarations: () => [], + getAllTools: () => [options.tool], + } as unknown as ToolRegistry; + + return { + getToolRegistry: () => mockToolRegistry, + getApprovalMode: () => options.approvalMode ?? ApprovalMode.DEFAULT, + getAllowedTools: () => [], + getPermissionsAllow: () => [], + getPermissionsDeny: () => undefined, + getSessionId: () => 'test-session-id', + getUsageStatisticsEnabled: () => true, + getDebugMode: () => false, + getContentGeneratorConfig: () => ({ + model: 'test-model', + authType: 'gemini', + }), + getEffectiveInputModalities: () => ({ image: true }), + getShellExecutionConfig: () => ({ + terminalWidth: 90, + terminalHeight: 30, + }), + storage: { + getProjectTempDir: () => '/tmp', + }, + getTruncateToolOutputThreshold: () => + DEFAULT_TRUNCATE_TOOL_OUTPUT_THRESHOLD, + getTruncateToolOutputLines: () => DEFAULT_TRUNCATE_TOOL_OUTPUT_LINES, + getUseModelRouter: () => false, + getGeminiClient: () => null, + getChatRecordingService: () => undefined, + getMessageBus: vi.fn().mockReturnValue(undefined), + getDisableAllHooks: vi.fn().mockReturnValue(true), + getHookSystem: vi.fn().mockReturnValue(undefined), + isInteractive: vi.fn().mockReturnValue(options.interactive ?? false), + getExperimentalZedIntegration: () => false, + getAutoModeDenialState: () => ({ + consecutiveBlock: 0, + consecutiveUnavailable: 0, + totalBlock: 0, + totalUnavailable: 0, + }), + setAutoModeDenialState: vi.fn(), + getAutoModeSettings: () => ({}), + getOmniPolicyToolsSettings: () => options.omniPolicyTools, + ...(options.isToolEnabled + ? { + getPermissionManager: () => ({ + isToolEnabled: options.isToolEnabled, + findMatchingDenyRule: () => undefined, + }), + } + : {}), + } as unknown as Config; +} + +const request = ( + overrides: Partial & { name: string }, +): ToolCallRequestInfo => ({ + callId: 'call-1', + args: {}, + isClientInitiated: false, + prompt_id: 'prompt-1', + ...overrides, +}); + +describe('CoreToolScheduler media-policy modelAccess gate', () => { + it('rejects a call with a missing executionOrigin (fails closed as model) when modelAccess is absent', async () => { + const executeFn = vi.fn(); + const tool = new MockMediaPolicyTool({ + name: 'omni_compress_image', + execute: executeFn, + }); + const config = makeConfig({ tool }); + + const response = await executeToolCall( + config, + request({ name: tool.name }), + new AbortController().signal, + ); + + expect(response.errorType).toBe(ToolErrorType.EXECUTION_DENIED); + expect(response.error?.message).toContain( + '"omni.processing.policyTools.omni_compress_image.modelAccess.enabled": true', + ); + expect(executeFn).not.toHaveBeenCalled(); + }); + + it('executes an enabled tool with defaults + model args + lockedArguments merged', async () => { + const executeFn: Mock = vi.fn().mockResolvedValue({ + llmContent: 'ok', + returnDisplay: 'ok', + } satisfies ToolResult); + const tool = new MockMediaPolicyTool({ + name: 'omni_compress_image', + execute: executeFn, + }); + const config = makeConfig({ + tool, + omniPolicyTools: { + omni_compress_image: { + modelAccess: { + enabled: true, + defaultArguments: { quality: 80, format: 'jpeg' }, + lockedArguments: { output_dir: '/objects' }, + }, + }, + }, + }); + + const response = await executeToolCall( + config, + request({ name: tool.name, args: { quality: 55, source: 'a.png' } }), + new AbortController().signal, + ); + + expect(response.error).toBeUndefined(); + expect(executeFn).toHaveBeenCalledWith({ + quality: 55, + format: 'jpeg', + source: 'a.png', + output_dir: '/objects', + }); + }); + + it('rejects explicit lockedArguments keys as INVALID_TOOL_PARAMS', async () => { + const executeFn = vi.fn(); + const tool = new MockMediaPolicyTool({ + name: 'omni_compress_image', + execute: executeFn, + }); + const config = makeConfig({ + tool, + omniPolicyTools: { + omni_compress_image: { + modelAccess: { + enabled: true, + lockedArguments: { output_dir: '/objects' }, + }, + }, + }, + }); + + const response = await executeToolCall( + config, + request({ name: tool.name, args: { output_dir: '/evil' } }), + new AbortController().signal, + ); + + expect(response.errorType).toBe(ToolErrorType.INVALID_TOOL_PARAMS); + expect(response.error?.message).toContain('"output_dir"'); + expect(executeFn).not.toHaveBeenCalled(); + }); + + it('rejects a forged fixed_policy origin on a non-media-policy tool', async () => { + const executeFn = vi.fn(); + const tool = new MockTool({ + name: 'run_shell_command', + execute: executeFn, + }); + const config = makeConfig({ tool }); + + const response = await executeToolCall( + config, + request({ + name: tool.name, + args: { command: 'rm -rf /' }, + executionOrigin: FIXED_ORIGIN, + }), + new AbortController().signal, + ); + + expect(response.errorType).toBe(ToolErrorType.EXECUTION_DENIED); + expect(response.error?.message).toContain('not a media policy tool'); + expect(executeFn).not.toHaveBeenCalled(); + }); +}); + +describe('CoreToolScheduler fixed_policy execution', () => { + it('executes a fixed_policy call without confirmation even when modelAccess is disabled', async () => { + const executeFn: Mock = vi.fn().mockResolvedValue({ + llmContent: 'compressed', + returnDisplay: 'compressed', + } satisfies ToolResult); + const getDefaultPermission = vi.fn(async () => 'ask' as const); + const tool = new MockMediaPolicyTool({ + name: 'omni_compress_image', + execute: executeFn, + getDefaultPermission, + }); + // No omniPolicyTools at all: modelAccess disabled by default, but the + // fixed-policy orchestrator path must still work. + const config = makeConfig({ tool }); + + const response = await executeToolCall( + config, + request({ + name: tool.name, + args: { source: 'a.png' }, + executionOrigin: FIXED_ORIGIN, + }), + new AbortController().signal, + ); + + expect(response.error).toBeUndefined(); + expect(executeFn).toHaveBeenCalledWith({ source: 'a.png' }); + // The interactive permission flow is skipped entirely. + expect(getDefaultPermission).not.toHaveBeenCalled(); + }); + + it('keeps confirmation for model-origin calls of the same enabled tool (bypass is origin-keyed)', async () => { + const executeFn = vi.fn(); + const tool = new MockMediaPolicyTool({ + name: 'omni_compress_image', + execute: executeFn, + getDefaultPermission: async () => 'ask' as const, + getConfirmationDetails: async () => ({ + type: 'info' as const, + title: 'Confirm compression', + prompt: 'Compress?', + onConfirm: async () => {}, + }), + }); + const config = makeConfig({ + tool, + interactive: true, + omniPolicyTools: { + omni_compress_image: { modelAccess: { enabled: true } }, + }, + }); + + const onToolCallsUpdate = vi.fn(); + const scheduler = new CoreToolScheduler({ + config, + onAllToolCallsComplete: vi.fn(), + onToolCallsUpdate, + getPreferredEditor: () => 'vscode', + onEditorClose: vi.fn(), + }); + + await scheduler.schedule( + [request({ name: tool.name, args: { source: 'a.png' } })], + new AbortController().signal, + ); + + await vi.waitFor(() => { + const statuses = onToolCallsUpdate.mock.calls + .flatMap((call) => call[0] as ToolCall[]) + .map((toolCall) => toolCall.status); + expect(statuses).toContain('awaiting_approval'); + }); + expect(executeFn).not.toHaveBeenCalled(); + }); + + it('still enforces PermissionManager.isToolEnabled for fixed_policy calls', async () => { + const executeFn = vi.fn(); + const tool = new MockMediaPolicyTool({ + name: 'omni_compress_image', + execute: executeFn, + }); + const config = makeConfig({ + tool, + isToolEnabled: async () => false, + }); + + const response = await executeToolCall( + config, + request({ + name: tool.name, + executionOrigin: FIXED_ORIGIN, + }), + new AbortController().signal, + ); + + expect(response.error).toBeDefined(); + expect(executeFn).not.toHaveBeenCalled(); + }); +}); + +describe('CoreToolScheduler policyArtifacts capture', () => { + const ARTIFACTS = [ + { + title: 'compressed.webp', + workspacePath: 'objects/compressed.webp', + mimeType: 'image/webp', + }, + ]; + + it('captures raw artifacts of a successful media-policy call into policyArtifacts', async () => { + const tool = new MockMediaPolicyTool({ + name: 'omni_compress_image', + execute: vi.fn().mockResolvedValue({ + llmContent: 'ok', + returnDisplay: 'ok', + artifacts: ARTIFACTS, + } satisfies ToolResult), + }); + const config = makeConfig({ tool }); + + const response = await executeToolCall( + config, + request({ + name: tool.name, + callId: 'staging-invocation-7', + executionOrigin: FIXED_ORIGIN, + }), + new AbortController().signal, + ); + + expect(response.error).toBeUndefined(); + expect(response.policyArtifacts).toEqual({ + toolName: 'omni_compress_image', + invocationId: 'staging-invocation-7', + executionOrigin: FIXED_ORIGIN, + artifacts: ARTIFACTS, + }); + }); + + it('reports a model origin in policyArtifacts for enabled model-origin calls', async () => { + const tool = new MockMediaPolicyTool({ + name: 'omni_compress_image', + execute: vi.fn().mockResolvedValue({ + llmContent: 'ok', + returnDisplay: 'ok', + artifacts: ARTIFACTS, + } satisfies ToolResult), + }); + const config = makeConfig({ + tool, + omniPolicyTools: { + omni_compress_image: { modelAccess: { enabled: true } }, + }, + }); + + const response = await executeToolCall( + config, + request({ name: tool.name }), + new AbortController().signal, + ); + + expect(response.policyArtifacts?.executionOrigin).toEqual({ + kind: 'model', + }); + }); + + it('does not emit policyArtifacts for ordinary tools with artifacts', async () => { + const tool = new MockTool({ + name: 'ordinary_tool', + execute: vi.fn().mockResolvedValue({ + llmContent: 'ok', + returnDisplay: 'ok', + artifacts: ARTIFACTS, + } satisfies ToolResult), + }); + const config = makeConfig({ tool }); + + const response = await executeToolCall( + config, + request({ name: tool.name }), + new AbortController().signal, + ); + + expect(response.error).toBeUndefined(); + expect(response.policyArtifacts).toBeUndefined(); + // The regular artifacts channel is unaffected. + expect(response.artifacts).toEqual(ARTIFACTS); + }); + + it('does not emit policyArtifacts when a media-policy call produced no artifacts', async () => { + const tool = new MockMediaPolicyTool({ + name: 'omni_compress_image', + execute: vi.fn().mockResolvedValue({ + llmContent: 'nothing to do', + returnDisplay: 'nothing to do', + } satisfies ToolResult), + }); + const config = makeConfig({ tool }); + + const response = await executeToolCall( + config, + request({ name: tool.name, executionOrigin: FIXED_ORIGIN }), + new AbortController().signal, + ); + + expect(response.error).toBeUndefined(); + expect(response.policyArtifacts).toBeUndefined(); + }); +}); diff --git a/packages/core/src/core/coreToolScheduler.ts b/packages/core/src/core/coreToolScheduler.ts index c0028afd221..9f40f3dfd1d 100644 --- a/packages/core/src/core/coreToolScheduler.ts +++ b/packages/core/src/core/coreToolScheduler.ts @@ -18,8 +18,10 @@ import type { AnyToolInvocation, ChatRecordingService, ToolArtifact, + PolicyArtifactBatch, } from '../index.js'; import { createDebugLogger } from '../utils/debugLogger.js'; +import { evaluateMediaPolicyToolCall } from '../omni/policy/model-access.js'; import { sanitizeToolNameForProvider } from '../utils/tool-name-utils.js'; import { compactToolResultDisplayForHistory } from '../utils/toolResultDisplayCompaction.js'; import { @@ -2309,10 +2311,44 @@ export class CoreToolScheduler { continue; } + // Omni media-policy protocol gate (before buildInvocation, so the + // merged arguments still go through the tool's native schema and + // business validation): + // - model/client-origin calls of a media-policy tool require + // modelAccess.enabled, are rejected when they explicitly name a + // lockedArguments key, and get defaultArguments/lockedArguments + // merged in; + // - a fixed_policy origin on a NON-media-policy tool is rejected + // (defense in depth: a forged origin must not become a + // permission bypass for Shell/Edit/MCP tools); + // - a missing origin fails closed as model. + const policyGate = evaluateMediaPolicyToolCall({ + config: this.config, + tool: toolInstance, + args: reqInfo.args, + executionOrigin: reqInfo.executionOrigin, + }); + if (policyGate.outcome === 'reject') { + newToolCalls.push({ + status: 'error', + request: reqInfo, + tool: toolInstance, + response: createErrorResponse( + reqInfo, + new Error(policyGate.message), + policyGate.reason === 'invalid_params' + ? ToolErrorType.INVALID_TOOL_PARAMS + : ToolErrorType.EXECUTION_DENIED, + ), + durationMs: 0, + }); + continue; + } + const invocationOrError = runInRequestGoalContext(reqInfo, () => this.buildInvocation( toolInstance, - reqInfo.args, + policyGate.args, reqInfo.callId, reqInfo.prompt_id, ), @@ -2436,6 +2472,22 @@ export class CoreToolScheduler { // L3→L4→L5 Permission Flow // ================================================================= + // Fixed-policy orchestrator calls skip the interactive permission + // flow entirely: no PermissionManager ask/deny evaluation, no + // confirmation dialog, no plan/auto classification. The remaining + // guards still hold — PM tool-enablement ran at schedule time, + // origin/descriptor pairing was enforced before buildInvocation, + // and PreToolUse hooks fire (a hook deny fails the call closed) + // at execution time. + if (reqInfo.executionOrigin?.kind === 'fixed_policy') { + this.setToolCallOutcome( + reqInfo.callId, + ToolConfirmationOutcome.ProceedAlways, + ); + this.setStatusInternal(reqInfo.callId, 'scheduled'); + continue; + } + // ---- L3→L4: Shared permission flow ---- let toolParams = invocation.params as Record; const flowResult = await runInRequestGoalContext(reqInfo, () => @@ -4763,6 +4815,22 @@ export class CoreToolScheduler { ...(toolResult.artifacts ?? []), ...(postToolUseArtifacts ?? []), ]; + // Raw media-policy artifacts, captured from the tool's OWN result — + // deliberately excluding the PostToolUse hook artifacts merged into + // `artifacts` above, which must never impersonate policy outputs. + const policyArtifacts: PolicyArtifactBatch | undefined = + scheduledCall.tool.mediaPolicyDescriptor && + toolResult.artifacts && + toolResult.artifacts.length > 0 + ? { + toolName: canonicalName, + invocationId: callId, + executionOrigin: scheduledCall.request.executionOrigin ?? { + kind: 'model', + }, + artifacts: toolResult.artifacts, + } + : undefined; const successResponse: ToolCallResponseInfo = { callId, responseParts: response, @@ -4786,6 +4854,7 @@ export class CoreToolScheduler { ? { visionBridgeNotice: processedImages.visionBridgeNotice } : {}), ...(artifacts.length > 0 ? { artifacts } : {}), + ...(policyArtifacts ? { policyArtifacts } : {}), }; // After an APPROVED exit_plan_mode, swap the large `plan` argument // still sitting in the model turn's functionCall for a pointer to the diff --git a/packages/core/src/core/turn.ts b/packages/core/src/core/turn.ts index 488f9662065..d7bee2809a5 100644 --- a/packages/core/src/core/turn.ts +++ b/packages/core/src/core/turn.ts @@ -122,6 +122,49 @@ export interface GeminiFinishedEventValue { usageMetadata: GenerateContentResponseUsageMetadata | undefined; } +/** + * Provenance of a tool-call request. Set exclusively by in-process callers + * when they construct the {@link ToolCallRequestInfo} — it is NEVER parsed + * from tool parameters, wire protocols, or model output, and it is NEVER + * inferred from `isClientInitiated`. A missing origin fails closed as + * `{ kind: 'model' }` (the least-privileged origin). + * + * `fixed_policy` marks calls issued by the omni fixed-policy orchestrator: + * they bypass the interactive permission flow (no confirmation dialog, no + * plan/auto classification) but still honor PreToolUse hooks and the + * PermissionManager tool-enablement check. + */ +export type ToolExecutionOrigin = + | { kind: 'model' } + | { kind: 'client' } + | { + kind: 'fixed_policy'; + /** ID of the fixed policy that issued this call. */ + policyId: string; + /** Pipeline stage the policy ran in. */ + stage: 'preprocessing' | 'transport_guard'; + }; + +/** + * Raw, successful media-policy tool artifacts captured by the scheduler + * BEFORE PostToolUse hook artifacts are merged in — hook-produced artifacts + * must never impersonate policy outputs. Carried on + * {@link ToolCallResponseInfo.policyArtifacts} for the fixed-policy + * orchestrator (and the model-call artifact bridge) to consume. + */ +export interface PolicyArtifactBatch { + /** Canonical tool name that produced the artifacts. */ + toolName: string; + /** The call id of the invocation (the orchestrator uses its staging + * invocation id as the call id, so this keys the staging directory). */ + invocationId: string; + /** Origin the call executed under (missing origins fail closed to model + * before this batch is built, so this is always concrete). */ + executionOrigin: ToolExecutionOrigin; + /** The tool's own `ToolResult.artifacts`, unmerged and in order. */ + artifacts: ToolArtifact[]; +} + export interface ToolCallRequestInfo { callId: string; /** @@ -137,6 +180,12 @@ export interface ToolCallRequestInfo { /** Set to true when the LLM response was truncated due to max_tokens. */ wasOutputTruncated?: boolean; goalContext?: GoalTurnPermit; + /** + * Provenance of this request. Only set by in-process callers; absent on + * every request materialized from model output or a wire protocol. + * Consumers treat a missing value as `{ kind: 'model' }` (fail closed). + */ + executionOrigin?: ToolExecutionOrigin; } export interface ToolCallResponseInfo { @@ -150,6 +199,12 @@ export interface ToolCallResponseInfo { modelOverride?: string; visionBridgeNotice?: string; artifacts?: ToolArtifact[]; + /** + * Raw successful artifacts of a media-policy tool, captured before + * PostToolUse hook artifact merging. Absent for non-media-policy tools, + * failed calls, and calls that produced no artifacts. + */ + policyArtifacts?: PolicyArtifactBatch; } function normalizeRequestParts(req: PartListUnion): Part[] { diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 962bf58b41b..1261e24361d 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -685,3 +685,16 @@ export { type DownloadedMedia, } from './omni/index.js'; export { processToolResultOmniMedia } from './omni/tool-result-media.js'; +export { + resolveMediaPolicyModelAccess, + isMediaPolicyToolHiddenFromModel, + evaluateMediaPolicyToolCall, + type MediaPolicyConfigView, + type MediaPolicyCallGateResult, + type ResolvedMediaPolicyModelAccess, +} from './omni/policy/model-access.js'; +export type { + OmniPolicyToolSettings, + OmniPolicyToolModelAccessSettings, + OmniPolicyToolsSettings, +} from './omni/policy/types.js'; diff --git a/packages/core/src/omni/policy/model-access.test.ts b/packages/core/src/omni/policy/model-access.test.ts new file mode 100644 index 00000000000..df38629e06d --- /dev/null +++ b/packages/core/src/omni/policy/model-access.test.ts @@ -0,0 +1,305 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { describe, expect, it } from 'vitest'; +import type { MediaPolicyToolDescriptor } from '../../tools/tools.js'; +import type { OmniPolicyToolsSettings } from './types.js'; +import { + evaluateMediaPolicyToolCall, + isMediaPolicyToolHiddenFromModel, + resolveMediaPolicyModelAccess, + type MediaPolicyConfigView, +} from './model-access.js'; + +const DESCRIPTOR: MediaPolicyToolDescriptor = { + kind: 'media_policy', + inputMediaTypes: ['image'], + outputs: [{ kind: 'media', required: true }], +}; + +const configWith = ( + settings: OmniPolicyToolsSettings | undefined, +): MediaPolicyConfigView => ({ + getOmniPolicyToolsSettings: () => settings, +}); + +const policyTool = (name = 'omni_compress_image') => ({ + name, + mediaPolicyDescriptor: DESCRIPTOR, +}); + +const ordinaryTool = (name = 'run_shell_command') => ({ name }); + +describe('resolveMediaPolicyModelAccess', () => { + it('defaults to disabled with empty projections when settings are absent', () => { + expect(resolveMediaPolicyModelAccess({}, 'omni_compress_image')).toEqual({ + enabled: false, + defaultArguments: {}, + lockedArguments: {}, + }); + }); + + it('defaults to disabled when the tool has no settings entry', () => { + const config = configWith({ + other_tool: { modelAccess: { enabled: true } }, + }); + expect( + resolveMediaPolicyModelAccess(config, 'omni_compress_image').enabled, + ).toBe(false); + }); + + it('reads enabled + argument projections when well-formed', () => { + const config = configWith({ + omni_compress_image: { + modelAccess: { + enabled: true, + defaultArguments: { quality: 80 }, + lockedArguments: { output_dir: '/tmp/objects' }, + }, + }, + }); + expect( + resolveMediaPolicyModelAccess(config, 'omni_compress_image'), + ).toEqual({ + enabled: true, + defaultArguments: { quality: 80 }, + lockedArguments: { output_dir: '/tmp/objects' }, + }); + }); + + it.each([ + ['null tombstone entry', { omni_compress_image: null }], + ['non-object modelAccess', { omni_compress_image: { modelAccess: 'yes' } }], + [ + 'array modelAccess', + { omni_compress_image: { modelAccess: [{ enabled: true }] } }, + ], + [ + 'truthy non-boolean enabled', + { omni_compress_image: { modelAccess: { enabled: 'true' } } }, + ], + ])('fails closed on malformed settings: %s', (_label, raw) => { + const config = configWith(raw as unknown as OmniPolicyToolsSettings); + expect( + resolveMediaPolicyModelAccess(config, 'omni_compress_image').enabled, + ).toBe(false); + }); + + it('ignores malformed argument projections but keeps enabled', () => { + const config = configWith({ + omni_compress_image: { + modelAccess: { + enabled: true, + defaultArguments: 'quality=80', + lockedArguments: ['output_dir'], + }, + }, + } as unknown as OmniPolicyToolsSettings); + expect( + resolveMediaPolicyModelAccess(config, 'omni_compress_image'), + ).toEqual({ enabled: true, defaultArguments: {}, lockedArguments: {} }); + }); +}); + +describe('isMediaPolicyToolHiddenFromModel', () => { + it('never hides ordinary tools', () => { + expect(isMediaPolicyToolHiddenFromModel({}, ordinaryTool())).toBe(false); + }); + + it('hides media-policy tools by default', () => { + expect(isMediaPolicyToolHiddenFromModel({}, policyTool())).toBe(true); + }); + + it('reveals media-policy tools when modelAccess.enabled is true', () => { + const config = configWith({ + omni_compress_image: { modelAccess: { enabled: true } }, + }); + expect(isMediaPolicyToolHiddenFromModel(config, policyTool())).toBe(false); + }); +}); + +describe('evaluateMediaPolicyToolCall', () => { + it('passes ordinary tools untouched regardless of settings', () => { + const args = { command: 'ls' }; + const result = evaluateMediaPolicyToolCall({ + config: configWith({ + run_shell_command: { modelAccess: { enabled: false } }, + }), + tool: ordinaryTool(), + args, + executionOrigin: { kind: 'model' }, + }); + expect(result).toEqual({ outcome: 'pass', args }); + }); + + it('treats a missing origin as a model call (fail closed)', () => { + const result = evaluateMediaPolicyToolCall({ + config: {}, + tool: policyTool(), + args: {}, + executionOrigin: undefined, + }); + expect(result).toMatchObject({ + outcome: 'reject', + reason: 'execution_denied', + }); + }); + + it('rejects model calls of media-policy tools by default, citing the setting', () => { + const result = evaluateMediaPolicyToolCall({ + config: {}, + tool: policyTool(), + args: {}, + executionOrigin: { kind: 'model' }, + }); + expect(result).toMatchObject({ + outcome: 'reject', + reason: 'execution_denied', + }); + expect((result as { message: string }).message).toContain( + '"omni.processing.policyTools.omni_compress_image.modelAccess.enabled": true', + ); + }); + + it('rejects client-origin calls the same as model calls when disabled', () => { + const result = evaluateMediaPolicyToolCall({ + config: {}, + tool: policyTool(), + args: {}, + executionOrigin: { kind: 'client' }, + }); + expect(result).toMatchObject({ + outcome: 'reject', + reason: 'execution_denied', + }); + }); + + it('rejects a forged fixed_policy origin on a non-media-policy tool', () => { + const result = evaluateMediaPolicyToolCall({ + config: {}, + tool: ordinaryTool(), + args: { command: 'rm -rf /' }, + executionOrigin: { + kind: 'fixed_policy', + policyId: 'forged', + stage: 'preprocessing', + }, + }); + expect(result).toMatchObject({ + outcome: 'reject', + reason: 'execution_denied', + }); + expect((result as { message: string }).message).toContain( + 'not a media policy tool', + ); + }); + + it('passes fixed_policy calls of media-policy tools untouched, ignoring modelAccess', () => { + const args = { quality: 55, output_dir: '/staging' }; + const result = evaluateMediaPolicyToolCall({ + // Disabled + locked keys present in args: neither applies to + // fixed-policy calls. + config: configWith({ + omni_compress_image: { + modelAccess: { + enabled: false, + lockedArguments: { output_dir: '/elsewhere' }, + }, + }, + }), + tool: policyTool(), + args, + executionOrigin: { + kind: 'fixed_policy', + policyId: 'image-compress-v1', + stage: 'preprocessing', + }, + }); + expect(result).toEqual({ outcome: 'pass', args }); + expect((result as { args: Record }).args).toBe(args); + }); + + it('rejects explicit lockedArguments keys as invalid_params, naming the keys', () => { + const result = evaluateMediaPolicyToolCall({ + config: configWith({ + omni_compress_image: { + modelAccess: { + enabled: true, + lockedArguments: { output_dir: '/tmp', format: 'webp' }, + }, + }, + }), + tool: policyTool(), + args: { output_dir: '/evil', format: 'exe', quality: 50 }, + executionOrigin: { kind: 'model' }, + }); + expect(result).toMatchObject({ + outcome: 'reject', + reason: 'invalid_params', + }); + const message = (result as { message: string }).message; + expect(message).toContain('"output_dir"'); + expect(message).toContain('"format"'); + }); + + it('rejects a locked key even when passed as undefined', () => { + const result = evaluateMediaPolicyToolCall({ + config: configWith({ + omni_compress_image: { + modelAccess: { + enabled: true, + lockedArguments: { output_dir: '/tmp' }, + }, + }, + }), + tool: policyTool(), + args: { output_dir: undefined }, + executionOrigin: { kind: 'model' }, + }); + expect(result).toMatchObject({ + outcome: 'reject', + reason: 'invalid_params', + }); + }); + + it('merges defaults < model args < lockedArguments on pass', () => { + const result = evaluateMediaPolicyToolCall({ + config: configWith({ + omni_compress_image: { + modelAccess: { + enabled: true, + defaultArguments: { quality: 80, format: 'jpeg' }, + lockedArguments: { output_dir: '/objects' }, + }, + }, + }), + tool: policyTool(), + args: { quality: 55, source: 'a.png' }, + executionOrigin: { kind: 'model' }, + }); + expect(result).toEqual({ + outcome: 'pass', + args: { + quality: 55, // model overrides default + format: 'jpeg', // default fills omitted + source: 'a.png', // model-only key preserved + output_dir: '/objects', // locked always injected + }, + }); + }); + + it('passes enabled tools with no projections through unchanged', () => { + const result = evaluateMediaPolicyToolCall({ + config: configWith({ + omni_compress_image: { modelAccess: { enabled: true } }, + }), + tool: policyTool(), + args: { source: 'a.png' }, + executionOrigin: { kind: 'model' }, + }); + expect(result).toEqual({ outcome: 'pass', args: { source: 'a.png' } }); + }); +}); diff --git a/packages/core/src/omni/policy/model-access.ts b/packages/core/src/omni/policy/model-access.ts new file mode 100644 index 00000000000..bd169ab10db --- /dev/null +++ b/packages/core/src/omni/policy/model-access.ts @@ -0,0 +1,176 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import type { ToolExecutionOrigin } from '../../core/turn.js'; +import type { MediaPolicyToolDescriptor } from '../../tools/tools.js'; +import type { + OmniPolicyToolModelAccessSettings, + OmniPolicyToolsSettings, +} from './types.js'; + +/** + * Shared modelAccess resolver + call gate for omni media-policy tools. + * + * Media-policy tools are always registered (the fixed-policy orchestrator + * must be able to find them), but they are fixed-policy-only by default: + * only `omni.processing.policyTools..modelAccess.enabled: true` + * makes them callable by the model or by direct client calls. The gate + * must hold on every surface at once — declaration lists, ToolSearch + * keyword + select, the CoreToolScheduler, and ACP's Session.runTool() — + * so all of them call into this module rather than re-deriving the rule. + */ + +/** Minimal structural view of Config used by this module. All calls are + * optional so partial/stub configs (tests, embedders) fail closed. */ +export interface MediaPolicyConfigView { + getOmniPolicyToolsSettings?: () => OmniPolicyToolsSettings | undefined; +} + +/** Resolved modelAccess for one tool: always concrete (defaults applied). */ +export interface ResolvedMediaPolicyModelAccess { + /** Whether model/client-origin calls are allowed. Default false. */ + enabled: boolean; + defaultArguments: Record; + lockedArguments: Record; +} + +const isPlainRecord = (value: unknown): value is Record => + typeof value === 'object' && value !== null && !Array.isArray(value); + +/** + * Read `omni.processing.policyTools..modelAccess` leniently: + * anything absent or malformed resolves to the fail-closed default + * (`enabled: false`, no argument projection). + */ +export function resolveMediaPolicyModelAccess( + config: MediaPolicyConfigView, + toolName: string, +): ResolvedMediaPolicyModelAccess { + const entry = config.getOmniPolicyToolsSettings?.()?.[toolName]; + const modelAccess: OmniPolicyToolModelAccessSettings | undefined = + isPlainRecord(entry) && isPlainRecord(entry['modelAccess']) + ? (entry['modelAccess'] as OmniPolicyToolModelAccessSettings) + : undefined; + return { + enabled: modelAccess?.enabled === true, + defaultArguments: isPlainRecord(modelAccess?.defaultArguments) + ? modelAccess.defaultArguments + : {}, + lockedArguments: isPlainRecord(modelAccess?.lockedArguments) + ? modelAccess.lockedArguments + : {}, + }; +} + +/** + * Whether a tool must be hidden from model-facing declaration surfaces + * (initial declarations, subagent filtered declarations, ToolSearch + * keyword candidates and exact-select). True iff the tool is a + * media-policy tool whose modelAccess is not enabled. + */ +export function isMediaPolicyToolHiddenFromModel( + config: MediaPolicyConfigView, + tool: { name: string; mediaPolicyDescriptor?: MediaPolicyToolDescriptor }, +): boolean { + if (!tool.mediaPolicyDescriptor) return false; + return !resolveMediaPolicyModelAccess(config, tool.name).enabled; +} + +/** Outcome of {@link evaluateMediaPolicyToolCall}. */ +export type MediaPolicyCallGateResult = + | { + outcome: 'pass'; + /** Arguments to build the invocation with. For gated model/client + * calls this is defaults + caller args + lockedArguments; for + * everything else it is the caller args unchanged. */ + args: Record; + } + | { + outcome: 'reject'; + /** 'execution_denied' → the call may not run at all; + * 'invalid_params' → a parameter-level error the model can fix. */ + reason: 'execution_denied' | 'invalid_params'; + message: string; + }; + +/** + * Execution-time gate applied by CoreToolScheduler and ACP Session.runTool + * before an invocation is built: + * + * - a `fixed_policy` origin on a NON-media-policy tool is rejected + * (defense in depth — origins are never deserialized, but a forged + * origin must not become a permission bypass for Shell/Edit/MCP); + * - a `fixed_policy` origin on a media-policy tool passes untouched (the + * orchestrator already resolved its own `arguments`; modelAccess does + * not apply to fixed calls); + * - a model/client-origin call of a media-policy tool requires + * `modelAccess.enabled`, must not name any lockedArguments key + * explicitly, and gets defaults + lockedArguments merged in; + * - everything else passes untouched. + * + * A missing origin fails closed as `{ kind: 'model' }`. + */ +export function evaluateMediaPolicyToolCall(params: { + config: MediaPolicyConfigView; + tool: { name: string; mediaPolicyDescriptor?: MediaPolicyToolDescriptor }; + args: Record; + executionOrigin: ToolExecutionOrigin | undefined; +}): MediaPolicyCallGateResult { + const { config, tool, args } = params; + const origin = params.executionOrigin ?? { kind: 'model' }; + + if (origin.kind === 'fixed_policy') { + if (!tool.mediaPolicyDescriptor) { + return { + outcome: 'reject', + reason: 'execution_denied', + message: + `Tool "${tool.name}" cannot run with a fixed-policy execution ` + + `origin: it is not a media policy tool.`, + }; + } + return { outcome: 'pass', args }; + } + + if (!tool.mediaPolicyDescriptor) { + return { outcome: 'pass', args }; + } + + const access = resolveMediaPolicyModelAccess(config, tool.name); + if (!access.enabled) { + return { + outcome: 'reject', + reason: 'execution_denied', + message: + `Tool "${tool.name}" is an omni media policy tool reserved for ` + + `fixed-policy orchestration. Direct calls require ` + + `"omni.processing.policyTools.${tool.name}.modelAccess.enabled": true.`, + }; + } + + const lockedKeys = Object.keys(access.lockedArguments); + const violations = lockedKeys.filter((key) => + Object.prototype.hasOwnProperty.call(args, key), + ); + if (violations.length > 0) { + return { + outcome: 'reject', + reason: 'invalid_params', + message: + `Invalid parameters for tool "${tool.name}": ` + + `${violations.map((k) => `"${k}"`).join(', ')} ` + + `${violations.length === 1 ? 'is' : 'are'} locked by configuration ` + + `and must not be provided. Remove ${ + violations.length === 1 ? 'it' : 'them' + } and retry.`, + }; + } + + return { + outcome: 'pass', + args: { ...access.defaultArguments, ...args, ...access.lockedArguments }, + }; +} diff --git a/packages/core/src/omni/policy/types.ts b/packages/core/src/omni/policy/types.ts new file mode 100644 index 00000000000..247f7004788 --- /dev/null +++ b/packages/core/src/omni/policy/types.ts @@ -0,0 +1,66 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Omni policy-pipeline protocol types. + * + * The wire-facing pieces live where their consumers already are — + * `ToolExecutionOrigin` / `PolicyArtifactBatch` next to the scheduler + * protocol in core/turn.ts, `MediaPolicyToolDescriptor` next to the tool + * framework in tools/tools.ts — and are re-exported here so omni code can + * import everything policy-related from one place. + */ + +export type { + ToolExecutionOrigin, + PolicyArtifactBatch, +} from '../../core/turn.js'; +export type { + MediaPolicyToolDescriptor, + MediaPolicyToolOutputSpec, +} from '../../tools/tools.js'; + +/** + * Raw (pre-normalization) shape of one + * `omni.processing.policyTools.` settings entry. Full semantic + * validation happens in the config-normalization pass; these types only + * capture the structure the lenient readers navigate. + */ +export interface OmniPolicyToolModelAccessSettings { + /** Whether the model (and direct client calls) may invoke the tool. + * Default: false — media-policy tools are fixed-policy-only unless + * explicitly opened up. */ + enabled?: boolean; + /** Overrides the tool description the model sees. */ + description?: string; + /** Filled in when the model omits them. */ + defaultArguments?: Record; + /** Harness-injected arguments, hidden from the model's schema; a model + * call that passes any of these keys explicitly is a parameter error. */ + lockedArguments?: Record; + /** Narrowing-only projection over the tool's native schema. */ + parameterSchema?: Record; + /** Artifact behavior for model-origin calls (Stage B). */ + output?: Record; +} + +/** Raw shape of one `omni.processing.policyTools.` entry. */ +export interface OmniPolicyToolSettings { + /** Tool-level settings validated against the descriptor's settingsSchema. */ + settings?: Record; + /** Per-tool runtime limits (timeoutMs, maxConcurrency). */ + runtime?: Record; + /** Model-callability gate and argument projection. */ + modelAccess?: OmniPolicyToolModelAccessSettings; +} + +/** Raw `omni.processing.policyTools` map as loaded from settings. Values + * may be null (scope-merge tombstones) or malformed — readers must treat + * anything non-conforming as absent (fail closed). */ +export type OmniPolicyToolsSettings = Record< + string, + OmniPolicyToolSettings | null +>; diff --git a/packages/core/src/tools/tool-registry.test.ts b/packages/core/src/tools/tool-registry.test.ts index 74ab9f712b0..46524364c80 100644 --- a/packages/core/src/tools/tool-registry.test.ts +++ b/packages/core/src/tools/tool-registry.test.ts @@ -17,6 +17,7 @@ import { mcpToTool } from '@google/genai'; import { spawn } from 'node:child_process'; import fs from 'node:fs'; import { MockTool } from '../test-utils/mock-tool.js'; +import type { MediaPolicyToolDescriptor } from './tools.js'; import { CHARS_PER_TOKEN } from '../services/tokenEstimation.js'; import { McpClientManager } from './mcp-client-manager.js'; @@ -384,6 +385,78 @@ describe('ToolRegistry', () => { }); }); + describe('media-policy tool visibility', () => { + class MockMediaPolicyTool extends MockTool { + override get mediaPolicyDescriptor(): MediaPolicyToolDescriptor { + return { + kind: 'media_policy', + inputMediaTypes: ['image'], + outputs: [{ kind: 'media', required: true }], + }; + } + } + + const enabledConfig = () => + new Config({ + ...baseConfigParams, + omniPolicyTools: { + omni_compress_image: { modelAccess: { enabled: true } }, + }, + }); + + it('excludes media-policy tools from getFunctionDeclarations by default', () => { + toolRegistry.registerTool(new MockTool({ name: 'visible' })); + toolRegistry.registerTool( + new MockMediaPolicyTool({ name: 'omni_compress_image' }), + ); + + const names = toolRegistry.getFunctionDeclarations().map((d) => d.name); + expect(names).toEqual(['visible']); + }); + + it('keeps media-policy tools hidden even with includeDeferred: true', () => { + // agent-core's wildcard/default branches call + // getFunctionDeclarations({ includeDeferred: true }); the media-policy + // filter must hold there too. + toolRegistry.registerTool( + new MockMediaPolicyTool({ name: 'omni_compress_image' }), + ); + + const names = toolRegistry + .getFunctionDeclarations({ includeDeferred: true }) + .map((d) => d.name); + expect(names).toEqual([]); + }); + + it('excludes media-policy tools from getFunctionDeclarationsFiltered even when named explicitly', () => { + toolRegistry.registerTool(new MockTool({ name: 'visible' })); + toolRegistry.registerTool( + new MockMediaPolicyTool({ name: 'omni_compress_image' }), + ); + + const names = toolRegistry + .getFunctionDeclarationsFiltered(['visible', 'omni_compress_image']) + .map((d) => d.name); + expect(names).toEqual(['visible']); + }); + + it('declares media-policy tools when modelAccess.enabled is true', () => { + const registry = new ToolRegistry(enabledConfig()); + registry.registerTool( + new MockMediaPolicyTool({ name: 'omni_compress_image' }), + ); + + expect(registry.getFunctionDeclarations().map((d) => d.name)).toEqual([ + 'omni_compress_image', + ]); + expect( + registry + .getFunctionDeclarationsFiltered(['omni_compress_image']) + .map((d) => d.name), + ).toEqual(['omni_compress_image']); + }); + }); + describe('deferred tool filtering', () => { it('sorts visible function declarations by canonical name', () => { toolRegistry.registerTool(new MockTool({ name: 'zeta' })); diff --git a/packages/core/src/tools/tool-registry.ts b/packages/core/src/tools/tool-registry.ts index 2d6dc129ba9..06cd0a2978f 100644 --- a/packages/core/src/tools/tool-registry.ts +++ b/packages/core/src/tools/tool-registry.ts @@ -13,6 +13,7 @@ import type { } from './tools.js'; import { Kind, BaseDeclarativeTool, BaseToolInvocation } from './tools.js'; import { type Config, matchesAnyServerPattern } from '../config/config.js'; +import { isMediaPolicyToolHiddenFromModel } from '../omni/policy/model-access.js'; import { spawn } from 'node:child_process'; import { StringDecoder } from 'node:string_decoder'; import type { SendSdkMcpMessage } from './mcp-client.js'; @@ -734,16 +735,22 @@ export class ToolRegistry { includeDeferred?: boolean; }): FunctionDeclaration[] { const includeDeferred = options?.includeDeferred === true; - return Array.from(this.tools.values()) - .filter( - (tool) => - includeDeferred || - !tool.shouldDefer || - tool.alwaysLoad || - !this.isDeferredAndHidden(tool.name), - ) - .sort(ToolRegistry.compareToolsByDeclarationName) - .map((tool) => tool.schema); + return ( + Array.from(this.tools.values()) + .filter( + (tool) => + includeDeferred || + !tool.shouldDefer || + tool.alwaysLoad || + !this.isDeferredAndHidden(tool.name), + ) + // Omni media-policy tools without modelAccess.enabled are registered + // (the fixed-policy orchestrator needs them) but never declared to + // the model — including for subagents, which force includeDeferred. + .filter((tool) => !isMediaPolicyToolHiddenFromModel(this.config, tool)) + .sort(ToolRegistry.compareToolsByDeclarationName) + .map((tool) => tool.schema) + ); } /** @@ -903,7 +910,10 @@ export class ToolRegistry { const declarations: FunctionDeclaration[] = []; for (const name of toolNames) { const tool = this.tools.get(name); - if (tool) { + // Same modelAccess gate as getFunctionDeclarations: an explicit + // subagent tool list must not become a leak path for media-policy + // tools the model can't call. + if (tool && !isMediaPolicyToolHiddenFromModel(this.config, tool)) { declarations.push(tool.schema); } } diff --git a/packages/core/src/tools/tool-search.test.ts b/packages/core/src/tools/tool-search.test.ts index 065d7480767..92dd3f7582e 100644 --- a/packages/core/src/tools/tool-search.test.ts +++ b/packages/core/src/tools/tool-search.test.ts @@ -12,7 +12,7 @@ import { ToolRegistry } from './tool-registry.js'; import { DiscoveredMCPTool } from './mcp-tool.js'; import { MockTool } from '../test-utils/mock-tool.js'; import { ToolSearchTool, scoreTool, tokenize } from './tool-search.js'; -import type { ToolResult } from './tools.js'; +import type { MediaPolicyToolDescriptor, ToolResult } from './tools.js'; import { CronCreateTool } from './cron-create.js'; import { CronDeleteTool } from './cron-delete.js'; import { CronListTool } from './cron-list.js'; @@ -252,6 +252,95 @@ describe('ToolSearchTool', () => { expect(registry.isDeferredToolRevealed('bravo')).toBe(true); }); + describe('media-policy tool hiding', () => { + class MockMediaPolicyTool extends MockTool { + override get mediaPolicyDescriptor(): MediaPolicyToolDescriptor { + return { + kind: 'media_policy', + inputMediaTypes: ['image'], + outputs: [{ kind: 'media', required: true }], + }; + } + } + + const registerPolicyTool = (reg: ToolRegistry) => { + reg.registerTool( + new MockMediaPolicyTool({ + name: 'omni_compress_image', + description: 'compress an image to a target size', + shouldDefer: true, + }), + ); + }; + + function makeEnabledConfigWithRegistry(): { + config: Config; + registry: ToolRegistry; + } { + const enabledConfig = new Config({ + ...baseConfigParams, + omniPolicyTools: { + omni_compress_image: { modelAccess: { enabled: true } }, + }, + }); + const enabledRegistry = new ToolRegistry(enabledConfig); + vi.spyOn(enabledConfig, 'getToolRegistry').mockReturnValue( + enabledRegistry, + ); + vi.spyOn(enabledConfig, 'getGeminiClient').mockReturnValue({ + setTools: vi.fn().mockResolvedValue(undefined), + } as never); + return { config: enabledConfig, registry: enabledRegistry }; + } + + it('keyword search never surfaces a hidden media-policy tool', async () => { + registerPolicyTool(registry); + + const tool = new ToolSearchTool(config); + const result = await tool + .build({ query: 'compress image' }) + .execute(new AbortController().signal); + + expect(String(result.llmContent)).toContain('No tools found'); + expect(String(result.llmContent)).not.toContain('omni_compress_image'); + }); + + it('select: mode blocks a hidden media-policy tool without revealing it', async () => { + registerPolicyTool(registry); + + const tool = new ToolSearchTool(config); + const result = await tool + .build({ query: 'select:omni_compress_image' }) + .execute(new AbortController().signal); + + const content = String(result.llmContent); + expect(content).toContain('media policy tool'); + expect(content).not.toContain(''); + expect(result.error?.message).toContain('media policy tool'); + expect(registry.isDeferredToolRevealed('omni_compress_image')).toBe( + false, + ); + }); + + it('surfaces the tool in both modes once modelAccess.enabled is true', async () => { + const { config: enabledConfig, registry: enabledRegistry } = + makeEnabledConfigWithRegistry(); + registerPolicyTool(enabledRegistry); + + const tool = new ToolSearchTool(enabledConfig); + const keywordResult = await tool + .build({ query: 'compress image' }) + .execute(new AbortController().signal); + expect(String(keywordResult.llmContent)).toContain( + '"name":"omni_compress_image"', + ); + + expect( + enabledRegistry.isDeferredToolRevealed('omni_compress_image'), + ).toBe(true); + }); + }); + it('keyword search returns top-N ranked tools', async () => { registry.registerTool( new MockTool({ diff --git a/packages/core/src/tools/tool-search.ts b/packages/core/src/tools/tool-search.ts index e923b4ab876..a788ac6418d 100644 --- a/packages/core/src/tools/tool-search.ts +++ b/packages/core/src/tools/tool-search.ts @@ -36,6 +36,7 @@ import { isLeaderOnlyToolUnavailableInSubagent, isPlanLifecycleToolUnavailableInSubagent, } from '../agents/runtime/subagent-plan-tool-policy.js'; +import { isMediaPolicyToolHiddenFromModel } from '../omni/policy/model-access.js'; const debugLogger = createDebugLogger('TOOL_SEARCH'); @@ -246,9 +247,13 @@ class ToolSearchInvocation extends BaseToolInvocation< */ private collectCandidates(): AnyDeclarativeTool[] { const registry = this.config.getToolRegistry(); - return registry - .getAllTools() - .filter((t) => registry.isDeferredAndHidden(t.name)); + return registry.getAllTools().filter( + (t) => + registry.isDeferredAndHidden(t.name) && + // Media-policy tools without modelAccess.enabled must never be + // surfaced to the model — not even via keyword discovery. + !isMediaPolicyToolHiddenFromModel(this.config, t), + ); } private async loadAndReturnSchemas( @@ -320,6 +325,14 @@ class ToolSearchInvocation extends BaseToolInvocation< missing.push(requested); continue; } + // Hidden media-policy tools cannot be revealed by exact-name lookup + // either: modelAccess.enabled is the only switch that exposes them. + // Blocking here (after ensureTool, which is where the descriptor + // becomes inspectable) guarantees no schema reveal happens below. + if (isMediaPolicyToolHiddenFromModel(this.config, tool)) { + blocked.push(canonical); + continue; + } // Only reveal + count toward the setTools() trigger when the tool // is actually deferred. `select:` mode also accepts already-loaded // / alwaysLoad tools (the model may use it to re-inspect a schema) @@ -424,7 +437,9 @@ class ToolSearchInvocation extends BaseToolInvocation< const blockedMessages = blocked.map((name) => isLeaderOnlyToolUnavailableInSubagent(name) ? getLeaderOnlyToolUnavailableMessage(name) - : getSubagentPlanToolUnavailableMessage(name), + : isPlanLifecycleToolUnavailableInSubagent(name) + ? getSubagentPlanToolUnavailableMessage(name) + : `Tool "${name}" is a media policy tool and is not available to the model.`, ); blockedErrorMessage = blockedMessages.join('\n'); const header = llmContent ? '\n\n' : ''; diff --git a/packages/core/src/tools/tools.ts b/packages/core/src/tools/tools.ts index 23d0e8efb0c..0e2216aa82c 100644 --- a/packages/core/src/tools/tools.ts +++ b/packages/core/src/tools/tools.ts @@ -152,6 +152,40 @@ export abstract class BaseToolInvocation< */ export type AnyToolInvocation = ToolInvocation; +/** One declared output of a media-policy tool (see + * {@link MediaPolicyToolDescriptor}). */ +export interface MediaPolicyToolOutputSpec { + /** What the output is: a derived media artifact or a disclosure text. */ + kind: 'media' | 'text'; + /** Role label for text outputs (e.g. 'disclosure'). */ + role?: string; + /** MIME types the output may carry (media outputs). */ + mimeTypes?: string[]; + /** Whether a successful run MUST produce this output. */ + required: boolean; + /** Whether the output is a lossy transformation of its input. A lossy + * media output obligates a disclosure text alongside it. */ + lossy?: boolean; +} + +/** + * Code-registration fact marking a tool as an omni media-policy tool — + * declared by the tool class itself, immutable at runtime, and never + * configurable. Its presence is what the scheduler's modelAccess gate, + * the declaration surfaces, and the fixed-policy orchestrator key off: + * config can never turn an ordinary tool into a policy tool (or the + * reverse). + */ +export interface MediaPolicyToolDescriptor { + kind: 'media_policy'; + /** Media modalities the tool accepts as input. */ + inputMediaTypes: Array<'image' | 'audio' | 'video'>; + /** Outputs a successful run may/must produce. */ + outputs: MediaPolicyToolOutputSpec[]; + /** JSON schema for `omni.processing.policyTools..settings`. */ + settingsSchema?: object; +} + /** * Interface for a tool builder that validates parameters and creates invocations. */ @@ -248,6 +282,16 @@ export abstract class DeclarativeTool< }; } + /** + * Present iff this tool is an omni media-policy tool. A code-level fact + * of the tool class (not configuration): the scheduler's modelAccess + * gate, the declaration surfaces, and the fixed-policy orchestrator all + * key off it. Default: not a media-policy tool. + */ + get mediaPolicyDescriptor(): MediaPolicyToolDescriptor | undefined { + return undefined; + } + /** * Max model-facing characters for this tool's output before the scheduler * spills it to disk (mirrors Claude Code's per-tool `maxResultSizeChars`). From edd3517bd50fbe6448e16bf4b67edabee07e03fb Mon Sep 17 00:00:00 2001 From: LaZzyMan Date: Fri, 7 Aug 2026 00:27:59 +0800 Subject: [PATCH 04/62] feat(omni): add staging/quarantine storage areas and recovery sweeps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Storage design §4.3/§4.4/§6.1 for the policy pipeline: - OmniObjectStore gains staging/ and quarantine/ areas (0o700, symlink refusal in ensureLayout) plus the invocation lifecycle: exclusive createStagingDir (16-hex id validation, no silent reuse), removeStagingDir, and quarantineInvocation — which writes reason.json (policyId/toolName/reason/failedAt) into the staging directory before a single atomic rename into quarantine//. - Startup recovery deletes everything under staging/ (entries belong to invocations that never committed) and trims quarantine/ to a retention window (default 7 days) and size budget (default 5 GiB, oldest-first), following the existing isRealDirectory containment convention: symlinked roots and entries are never traversed, sized, or deleted through. --- packages/core/src/omni/recovery.test.ts | 161 ++++++++++++++++++++++++ packages/core/src/omni/recovery.ts | 144 ++++++++++++++++++++- packages/core/src/omni/storage.test.ts | 122 ++++++++++++++++++ packages/core/src/omni/storage.ts | 112 ++++++++++++++++- 4 files changed, 534 insertions(+), 5 deletions(-) diff --git a/packages/core/src/omni/recovery.test.ts b/packages/core/src/omni/recovery.test.ts index 760d24cbef6..56ae2830f4d 100644 --- a/packages/core/src/omni/recovery.test.ts +++ b/packages/core/src/omni/recovery.test.ts @@ -241,6 +241,167 @@ describe('runStartupRecoveryOnce', () => { await expect(runStartupRecoveryOnce(store)).resolves.toBeUndefined(); }); + describe('staging sweep (storage design §6.1: uncommitted work is deleted)', () => { + it('deletes every staging entry, including nested artifact trees and stray files', async () => { + const stagingDir = store.getStagingDir(); + const invocationDir = path.join(stagingDir, '0123456789abcdef'); + await fs.mkdir(path.join(invocationDir, 'nested'), { recursive: true }); + await fs.writeFile( + path.join(invocationDir, 'nested', 'artifact.webp'), + 'half-written', + ); + await fs.writeFile(path.join(stagingDir, 'stray.tmp'), 'stray'); + + await runStartupRecoveryOnce(store); + + await expect(fs.readdir(stagingDir)).resolves.toEqual([]); + }); + + it('a symlinked staging ROOT is never swept', async () => { + const outside = await fs.mkdtemp(path.join(os.tmpdir(), 'omni-stage-')); + const victim = path.join(outside, 'victim.bin'); + await fs.writeFile(victim, 'external'); + try { + const stagingDir = store.getStagingDir(); + await fs.rm(stagingDir, { recursive: true, force: true }); + await fs.symlink(outside, stagingDir); + + await runStartupRecoveryOnce(store); + + await expect(fs.readFile(victim, 'utf8')).resolves.toBe('external'); + expect((await fs.lstat(stagingDir)).isSymbolicLink()).toBe(true); + } finally { + await fs.rm(outside, { recursive: true, force: true }); + } + }); + }); + + describe('quarantine sweep (retention window + size budget)', () => { + async function makeQuarantineEntry( + name: string, + content: string, + ageMs: number, + ): Promise { + const dir = path.join(store.getQuarantineDir(), name); + await fs.mkdir(dir, { recursive: true }); + await fs.writeFile(path.join(dir, 'artifact.bin'), content); + await fs.writeFile(path.join(dir, 'reason.json'), '{}'); + const when = new Date(Date.now() - ageMs); + await fs.utimes(dir, when, when); + return dir; + } + + it('removes entries past the retention window, keeps younger ones', async () => { + const expired = await makeQuarantineEntry( + 'aaaaaaaaaaaaaaaa', + 'old', + 8 * 86_400_000, + ); + const fresh = await makeQuarantineEntry( + 'bbbbbbbbbbbbbbbb', + 'new', + 1 * 86_400_000, + ); + + await runStartupRecoveryOnce(store, undefined, { + quarantineRetentionDays: 7, + }); + + await expect(fs.lstat(expired)).rejects.toThrow(); + await expect(fs.lstat(fresh)).resolves.toBeDefined(); + }); + + it('removes oldest entries first when over the size budget', async () => { + const oldest = await makeQuarantineEntry( + 'aaaaaaaaaaaaaaaa', + 'x'.repeat(100), + 3 * 3600_000, + ); + const middle = await makeQuarantineEntry( + 'bbbbbbbbbbbbbbbb', + 'y'.repeat(100), + 2 * 3600_000, + ); + const newest = await makeQuarantineEntry( + 'cccccccccccccccc', + 'z'.repeat(100), + 1 * 3600_000, + ); + + // ~300 bytes of artifacts (+ reason.json) against a 250-byte budget: + // dropping the single oldest entry brings the area back under. + await runStartupRecoveryOnce(store, undefined, { + quarantineMaxBytes: 250, + }); + + await expect(fs.lstat(oldest)).rejects.toThrow(); + await expect(fs.lstat(middle)).resolves.toBeDefined(); + await expect(fs.lstat(newest)).resolves.toBeDefined(); + }); + + it('keeps everything when under both retention and budget', async () => { + const a = await makeQuarantineEntry('aaaaaaaaaaaaaaaa', 'a', 3600_000); + const b = await makeQuarantineEntry('bbbbbbbbbbbbbbbb', 'b', 7200_000); + + await runStartupRecoveryOnce(store); + + await expect(fs.lstat(a)).resolves.toBeDefined(); + await expect(fs.lstat(b)).resolves.toBeDefined(); + }); + + it('a symlinked quarantine ENTRY is never traversed, sized, or deleted', async () => { + const outside = await fs.mkdtemp(path.join(os.tmpdir(), 'omni-quar-')); + const victim = path.join(outside, 'victim.bin'); + await fs.writeFile(victim, 'x'.repeat(10_000)); + const old = new Date(Date.now() - 30 * 86_400_000); + await fs.utimes(outside, old, old); + await fs.utimes(victim, old, old); + try { + const link = path.join(store.getQuarantineDir(), 'dddddddddddddddd'); + await fs.symlink(outside, link); + + // Aggressive limits: if the sweep treated the link as an entry it + // would be expired AND over budget — external bytes must survive. + await runStartupRecoveryOnce(store, undefined, { + quarantineRetentionDays: 1, + quarantineMaxBytes: 1, + }); + + await expect(fs.readFile(victim, 'utf8')).resolves.toBe( + 'x'.repeat(10_000), + ); + expect((await fs.lstat(link)).isSymbolicLink()).toBe(true); + } finally { + await fs.rm(outside, { recursive: true, force: true }); + } + }); + + it('a symlinked quarantine ROOT is never swept', async () => { + const outside = await fs.mkdtemp(path.join(os.tmpdir(), 'omni-quar-')); + const victimDir = path.join(outside, 'eeeeeeeeeeeeeeee'); + await fs.mkdir(victimDir); + await fs.writeFile(path.join(victimDir, 'victim.bin'), 'external'); + const old = new Date(Date.now() - 30 * 86_400_000); + await fs.utimes(victimDir, old, old); + try { + const quarantineDir = store.getQuarantineDir(); + await fs.rm(quarantineDir, { recursive: true, force: true }); + await fs.symlink(outside, quarantineDir); + + await runStartupRecoveryOnce(store, undefined, { + quarantineRetentionDays: 1, + }); + + await expect( + fs.readFile(path.join(victimDir, 'victim.bin'), 'utf8'), + ).resolves.toBe('external'); + expect((await fs.lstat(quarantineDir)).isSymbolicLink()).toBe(true); + } finally { + await fs.rm(outside, { recursive: true, force: true }); + } + }); + }); + describe('symlink containment (recovery must never leave the omni root)', () => { /** External dir with a victim file whose NAME makes recovery want to * delete it through every code path: hash-mismatched "object", expired diff --git a/packages/core/src/omni/recovery.ts b/packages/core/src/omni/recovery.ts index 852b03983de..0362c63b53b 100644 --- a/packages/core/src/omni/recovery.ts +++ b/packages/core/src/omni/recovery.ts @@ -31,12 +31,21 @@ const SAMPLE_VERIFY_MAX_BYTES = 64 * 1024 * 1024; * belong to a promotion in flight in ANOTHER process — deleting it would * fail that process's rename. Older survivors are crash leftovers. */ const TMP_GRACE_MS = 3600_000; +/** Default retention for quarantined invocations (storage design §7). */ +const QUARANTINE_RETENTION_DAYS = 7; +/** Default size budget for the quarantine area (storage design §7). */ +const QUARANTINE_MAX_BYTES = 5 * 1024 * 1024 * 1024; /** Tunables for {@link runStartupRecoveryOnce}; production callers use * the defaults, tests inject small values. */ export interface StartupRecoveryOptions { sampleVerifyLimit?: number; sampleVerifyMaxBytes?: number; + /** Quarantined invocations older than this are removed. */ + quarantineRetentionDays?: number; + /** Above this total size, quarantined invocations are removed + * oldest-first until the area fits. */ + quarantineMaxBytes?: number; } /** One latch per omni root: distinct stores in one process (multi-project @@ -91,6 +100,124 @@ async function sweepDownloads(downloadsDir: string): Promise { } } +/** + * Delete EVERYTHING under `staging/`. Staging entries belong to policy + * invocations that never committed (a successful commit deletes its own + * staging directory first), so at startup there is nothing to keep + * (storage design §6.1). The staging root itself must be a real directory + * — a symlinked root would redirect the recursive deletes outside the + * omni root. + */ +async function sweepStaging(stagingDir: string): Promise { + if (!(await isRealDirectory(stagingDir))) return; + let names: string[]; + try { + names = await fs.readdir(stagingDir); + } catch { + return; + } + for (const name of names) { + try { + // rm on a symlink entry removes the link itself without following + // it, so no containment check is needed per entry. + await fs.rm(path.join(stagingDir, name), { + recursive: true, + force: true, + }); + debugLogger.debug(`recovery: removed uncommitted staging ${name}`); + } catch { + // Best-effort sweep. + } + } +} + +/** Recursively sum the sizes of regular files under a REAL directory, + * never following symlinks (neither directory nor file entries). */ +async function directorySizeBytes(dir: string): Promise { + let total = 0; + let names: string[]; + try { + names = await fs.readdir(dir); + } catch { + return total; + } + for (const name of names) { + const p = path.join(dir, name); + try { + const st = await fs.lstat(p); + if (st.isFile()) { + total += st.size; + } else if (st.isDirectory()) { + total += await directorySizeBytes(p); + } + } catch { + // Unreadable entry contributes nothing. + } + } + return total; +} + +/** + * Enforce the quarantine retention window and size budget (storage design + * §4.4/§6.1): entries older than `retentionMs` are removed; if the + * remainder still exceeds `maxBytes`, the oldest entries are removed + * first until the area fits. Only REAL directories are treated as + * quarantine entries — symlinks are never traversed, sized, or deleted. + */ +async function sweepQuarantine( + quarantineDir: string, + retentionMs: number, + maxBytes: number, +): Promise { + if (!(await isRealDirectory(quarantineDir))) return; + let names: string[]; + try { + names = await fs.readdir(quarantineDir); + } catch { + return; + } + const entries: Array<{ name: string; mtimeMs: number; sizeBytes: number }> = + []; + const cutoff = Date.now() - retentionMs; + for (const name of names) { + const p = path.join(quarantineDir, name); + if (!(await isRealDirectory(p))) continue; + try { + const st = await fs.lstat(p); + if (st.mtimeMs < cutoff) { + await fs.rm(p, { recursive: true, force: true }); + debugLogger.debug(`recovery: removed expired quarantine ${name}`); + continue; + } + entries.push({ + name, + mtimeMs: st.mtimeMs, + sizeBytes: await directorySizeBytes(p), + }); + } catch { + // Best-effort sweep. + } + } + let total = entries.reduce((sum, e) => sum + e.sizeBytes, 0); + if (total <= maxBytes) return; + entries.sort((a, b) => a.mtimeMs - b.mtimeMs); + for (const entry of entries) { + if (total <= maxBytes) break; + try { + await fs.rm(path.join(quarantineDir, entry.name), { + recursive: true, + force: true, + }); + total -= entry.sizeBytes; + debugLogger.debug( + `recovery: removed quarantine ${entry.name} (over size budget)`, + ); + } catch { + // Best-effort sweep. + } + } +} + async function sweepTmpFiles(objectsDir: string): Promise { if (!(await isRealDirectory(objectsDir))) return; let shards: string[]; @@ -201,10 +328,14 @@ async function sampleVerifyObjects( * lazily the first time the omni pipeline is touched — zero cost when * omni is unused. * - * 1. crash-orphaned `downloads/*.part` older than the 48h debugging + * 1. everything under `staging/` is deleted — staging entries belong to + * policy invocations that never committed (storage design §6.1); + * 2. crash-orphaned `downloads/*.part` older than the 48h debugging * retention window are removed; - * 2. `objects/…/.tmp-*` promotion orphans are removed (crash leftovers); - * 3. a small sample of objects is hash-verified; corrupt objects are + * 3. `quarantine/` is trimmed to its retention window and size budget + * (oldest-first once over budget); + * 4. `objects/…/.tmp-*` promotion orphans are removed (crash leftovers); + * 5. a small sample of objects is hash-verified; corrupt objects are * deleted with their upload-cache entries cascaded. * * Never throws: recovery is hygiene, not a gate. That covers the latch @@ -236,7 +367,14 @@ export function runStartupRecoveryOnce( // managed tree. (An absent directory fails the check too, which is // fine — there is nothing to sweep beneath it.) if (!(await isRealDirectory(root))) return; + await sweepStaging(path.join(root, 'staging')); await sweepDownloads(path.join(root, 'downloads')); + await sweepQuarantine( + path.join(root, 'quarantine'), + (options?.quarantineRetentionDays ?? QUARANTINE_RETENTION_DAYS) * + 86_400_000, + options?.quarantineMaxBytes ?? QUARANTINE_MAX_BYTES, + ); if (!(await isRealDirectory(path.join(root, 'objects')))) return; await sweepTmpFiles(store.getObjectsDir()); await sampleVerifyObjects( diff --git a/packages/core/src/omni/storage.test.ts b/packages/core/src/omni/storage.test.ts index bff12275c17..eb4b152c3af 100644 --- a/packages/core/src/omni/storage.test.ts +++ b/packages/core/src/omni/storage.test.ts @@ -159,4 +159,126 @@ describe('OmniObjectStore', () => { await fs.rm(linkedQwen, { recursive: true, force: true }); } }); + + describe('staging and quarantine areas', () => { + const INVOCATION_ID = '0123456789abcdef'; + + it('ensureLayout creates staging/ and quarantine/ with 0o700', async () => { + await store.ensureLayout(); + for (const dir of [store.getStagingDir(), store.getQuarantineDir()]) { + const st = await fs.stat(dir); + expect(st.isDirectory()).toBe(true); + if (process.platform !== 'win32') { + expect(st.mode & 0o777).toBe(0o700); + } + } + expect(store.getStagingDir()).toBe(path.join(qwenDir, 'omni', 'staging')); + expect(store.getQuarantineDir()).toBe( + path.join(qwenDir, 'omni', 'quarantine'), + ); + }); + + it('creates an exclusive per-invocation staging directory', async () => { + const dir = await store.createStagingDir(INVOCATION_ID); + expect(dir).toBe(path.join(store.getStagingDir(), INVOCATION_ID)); + const st = await fs.stat(dir); + expect(st.isDirectory()).toBe(true); + if (process.platform !== 'win32') { + expect(st.mode & 0o777).toBe(0o700); + } + // A second create with the same id must fail, never silently reuse. + await expect(store.createStagingDir(INVOCATION_ID)).rejects.toThrow(); + }); + + it.each([ + ['path traversal', '../../escape00'], + ['uppercase hex', '0123456789ABCDEF'], + ['wrong length', '0123456789abcde'], + ['separator smuggling', '0123456789abcde/'], + ])('rejects an invalid invocation id: %s', async (_label, id) => { + await expect(store.createStagingDir(id)).rejects.toThrow( + /Invalid omni policy invocation id/, + ); + await expect(store.removeStagingDir(id)).rejects.toThrow( + /Invalid omni policy invocation id/, + ); + await expect( + store.quarantineInvocation(id, { + policyId: 'p', + toolName: 't', + reason: 'r', + }), + ).rejects.toThrow(/Invalid omni policy invocation id/); + }); + + it('removeStagingDir deletes the invocation directory recursively', async () => { + const dir = await store.createStagingDir(INVOCATION_ID); + await fs.mkdir(path.join(dir, 'nested')); + await fs.writeFile(path.join(dir, 'nested', 'artifact.webp'), 'bytes'); + await store.removeStagingDir(INVOCATION_ID); + await expect(fs.lstat(dir)).rejects.toThrow(); + // Idempotent on a missing directory. + await expect( + store.removeStagingDir(INVOCATION_ID), + ).resolves.toBeUndefined(); + }); + + it('quarantineInvocation moves artifacts and writes reason.json', async () => { + const dir = await store.createStagingDir(INVOCATION_ID); + await fs.writeFile(path.join(dir, 'partial.mp4'), 'half-transcoded'); + const quarantineDir = await store.quarantineInvocation(INVOCATION_ID, { + policyId: 'video-downscale-v1', + toolName: 'omni_downscale_video', + reason: 'required output missing', + }); + + expect(quarantineDir).toBe( + path.join(store.getQuarantineDir(), INVOCATION_ID), + ); + // Staging entry is gone; artifacts moved with original names. + await expect(fs.lstat(dir)).rejects.toThrow(); + await expect( + fs.readFile(path.join(quarantineDir, 'partial.mp4'), 'utf8'), + ).resolves.toBe('half-transcoded'); + const reason = JSON.parse( + await fs.readFile(path.join(quarantineDir, 'reason.json'), 'utf8'), + ); + expect(reason).toMatchObject({ + policyId: 'video-downscale-v1', + toolName: 'omni_downscale_video', + reason: 'required output missing', + }); + expect(new Date(reason.failedAt).getTime()).not.toBeNaN(); + }); + + it('quarantineInvocation fails when the staging directory is missing', async () => { + await store.ensureLayout(); + await expect( + store.quarantineInvocation(INVOCATION_ID, { + policyId: 'p', + toolName: 't', + reason: 'r', + }), + ).rejects.toThrow(); + }); + + it('quarantineInvocation refuses a symlinked staging entry', async () => { + await store.ensureLayout(); + const outside = path.join(qwenDir, 'outside-staging'); + await fs.mkdir(outside); + await fs.symlink( + outside, + path.join(store.getStagingDir(), INVOCATION_ID), + ); + await expect( + store.quarantineInvocation(INVOCATION_ID, { + policyId: 'p', + toolName: 't', + reason: 'r', + }), + ).rejects.toThrow(/not a real directory/); + // Nothing was written through the link. + await expect(fs.readdir(outside)).resolves.toEqual([]); + }); + }); }); diff --git a/packages/core/src/omni/storage.ts b/packages/core/src/omni/storage.ts index 56e875157a2..7a94b078435 100644 --- a/packages/core/src/omni/storage.ts +++ b/packages/core/src/omni/storage.ts @@ -19,6 +19,28 @@ export interface PutObjectResult { deduped: boolean; } +/** Why a policy invocation's staging directory was quarantined; persisted + * as `reason.json` beside the failed artifacts for debugging. */ +export interface QuarantineReason { + /** The fixed policy whose invocation failed. */ + policyId: string; + /** The media policy tool that ran (or failed to run). */ + toolName: string; + /** Human-readable failure description. */ + reason: string; +} + +/** Policy invocation IDs are orchestrator-generated 16-hex tokens; anything + * else (path separators, dots, uppercase) is refused before touching the + * filesystem so staging/quarantine paths can never escape their area. */ +const INVOCATION_ID_RE = /^[0-9a-f]{16}$/; + +function assertInvocationId(invocationId: string): void { + if (!INVOCATION_ID_RE.test(invocationId)) { + throw new Error(`Invalid omni policy invocation id: ${invocationId}`); + } +} + /** Reject paths that exist but are not what the store expects (symlinks, * devices, …). The store never follows symlinks for its own entries. */ async function assertRealDirIfExists(p: string): Promise { @@ -38,11 +60,13 @@ async function assertRealDirIfExists(p: string): Promise { /** * Content-addressed, immutable object store under `/.qwen/omni/`. * - * S1 scope: only the `objects/` area exists. Layout: + * Layout (storage design §4): * * .qwen/omni/ * ├── .gitignore # "*" — self-ignoring - * └── objects/sha256// + * ├── objects/sha256// + * ├── staging// # policy tool work dirs (pre-commit) + * └── quarantine// # failed invocations + reason.json * * Write protocol: stream-copy to a sibling `.tmp-*` file in the final * directory while re-computing the content hash, verify it matches the @@ -68,6 +92,18 @@ export class OmniObjectStore { return path.join(this.omniRoot, 'objects', 'sha256'); } + /** Root of the policy-invocation work area (deleted wholesale by + * startup recovery — anything here belongs to an uncommitted run). */ + getStagingDir(): string { + return path.join(this.omniRoot, 'staging'); + } + + /** Root of the failed-invocation debris area, kept for debugging under + * a retention/size budget and never re-entering recognition/delivery. */ + getQuarantineDir(): string { + return path.join(this.omniRoot, 'quarantine'); + } + /** Compute the final object path for a content hash + extension. */ objectPathFor(sha256: string, extension: string): string { return path.join( @@ -87,7 +123,14 @@ export class OmniObjectStore { await assertRealDirIfExists(this.omniRoot); await assertRealDirIfExists(path.join(this.omniRoot, 'objects')); await assertRealDirIfExists(this.getObjectsDir()); + await assertRealDirIfExists(this.getStagingDir()); + await assertRealDirIfExists(this.getQuarantineDir()); await fs.mkdir(this.getObjectsDir(), { recursive: true, mode: 0o700 }); + await fs.mkdir(this.getStagingDir(), { recursive: true, mode: 0o700 }); + await fs.mkdir(this.getQuarantineDir(), { + recursive: true, + mode: 0o700, + }); const gitignorePath = path.join(this.omniRoot, '.gitignore'); try { // 'wx' fails when the file already exists — atomic create-once, @@ -106,6 +149,71 @@ export class OmniObjectStore { return this.layoutReady; } + /** + * Create the exclusive work directory for one policy invocation and + * return its absolute path. The directory is the ONLY location the + * policy tool is allowed to write to (storage design §4.3). Creation is + * non-recursive and exclusive: a pre-existing entry (id collision or a + * planted path) fails instead of being silently reused. + */ + async createStagingDir(invocationId: string): Promise { + assertInvocationId(invocationId); + await this.ensureLayout(); + const dir = path.join(this.getStagingDir(), invocationId); + await fs.mkdir(dir, { mode: 0o700 }); + return dir; + } + + /** + * Delete one invocation's staging directory (after a successful commit, + * or as the failure path while quarantine is not involved). + */ + async removeStagingDir(invocationId: string): Promise { + assertInvocationId(invocationId); + await fs.rm(path.join(this.getStagingDir(), invocationId), { + recursive: true, + force: true, + }); + } + + /** + * Move a failed invocation's staging directory into + * `quarantine//`, preserving the artifact files and adding + * a `reason.json` (storage design §4.4). The reason file is written into + * the staging directory BEFORE the rename so the quarantine entry appears + * complete in one atomic step; a crash in between leaves it in staging, + * which startup recovery deletes wholesale. + */ + async quarantineInvocation( + invocationId: string, + reason: QuarantineReason, + ): Promise { + assertInvocationId(invocationId); + await this.ensureLayout(); + const stagingDir = path.join(this.getStagingDir(), invocationId); + // The rename source must be a real directory: a symlink here would + // make the reason.json write (and the quarantined "content") point + // outside the omni root. + const st = await fs.lstat(stagingDir); + if (st.isSymbolicLink() || !st.isDirectory()) { + throw new Error( + `Staging path is not a real directory (symlink or special file refused): ${stagingDir}`, + ); + } + await fs.writeFile( + path.join(stagingDir, 'reason.json'), + JSON.stringify( + { ...reason, failedAt: new Date().toISOString() }, + null, + 2, + ), + { mode: 0o600 }, + ); + const quarantineDir = path.join(this.getQuarantineDir(), invocationId); + await fs.rename(stagingDir, quarantineDir); + return quarantineDir; + } + /** * Promote a local file into the object store under its content hash. * The bytes are re-hashed while copying and verified against `sha256`, From 9a75fcd2532ced6df3d321548293510254beea8b Mon Sep 17 00:00:00 2001 From: LaZzyMan Date: Fri, 7 Aug 2026 00:43:24 +0800 Subject: [PATCH 05/62] feat(omni): add fixed-policy when-condition DSL evaluator and validator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implement the restricted when-condition DSL from the policy design (§8.3): recursive all/any combinators over gt|gte|lt|lte|eq comparisons, with operands drawn from three read-only namespaces (resource.*, request.*, session.*) or literals. No arbitrary code, no JSONPath. Evaluation is three-valued. A comparison over an unresolvable field yields `unavailable` with the missing fields recorded — never a silent false — so the caller can apply the policy's onConditionUnavailable behavior and surface the fields in the run record. Combinators use strong Kleene logic: `all` with a false branch is a determinate no_match regardless of unavailable siblings; `any` with a true branch is a determinate match. Vacuous semantics: all [] → match, any [] → no_match. The evaluator is a total function that never throws — structurally malformed nodes degrade to `unavailable`, the fail-safe outcome. The separate structural validator (for the startup config-normalization pass, policy design §13 #5) rejects malformed conditions with path-annotated errors: exactly-one-of comparison/all/any, non-empty combinator arrays, exactly-one-of field/value per operand, known-field membership, and finite numeric literals for ordering operators. Condition types are re-exported from omni/policy/types.ts alongside the rest of the policy protocol surface. --- .../core/src/omni/policy/conditions.test.ts | 336 +++++++++++++++++ packages/core/src/omni/policy/conditions.ts | 345 ++++++++++++++++++ packages/core/src/omni/policy/types.ts | 9 + 3 files changed, 690 insertions(+) create mode 100644 packages/core/src/omni/policy/conditions.test.ts create mode 100644 packages/core/src/omni/policy/conditions.ts diff --git a/packages/core/src/omni/policy/conditions.test.ts b/packages/core/src/omni/policy/conditions.test.ts new file mode 100644 index 00000000000..af934353aab --- /dev/null +++ b/packages/core/src/omni/policy/conditions.test.ts @@ -0,0 +1,336 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { describe, expect, it } from 'vitest'; +import type { + FixedPolicyCondition, + FixedPolicyConditionContext, +} from './conditions.js'; +import { + evaluateFixedPolicyCondition, + validateFixedPolicyCondition, +} from './conditions.js'; + +const CONTEXT: FixedPolicyConditionContext = { + resource: { + sizeBytes: 8_200_000, + width: 4096, + height: 3072, + estimatedTokenCount: 150_000, + }, + request: { totalEstimatedMediaTokens: 180_000 }, + session: { + contextWindowTokens: 131_072, + promptTokenCount: 20_000, + reservedOutputTokens: 8_192, + availableContextTokens: 102_880, + }, +}; + +const cmp = ( + left: object, + operator: string, + right: object, +): FixedPolicyCondition => + ({ left, operator, right }) as unknown as FixedPolicyCondition; + +describe('evaluateFixedPolicyCondition — comparisons', () => { + it.each([ + // [operator, right literal, expected outcome] against width=4096 + ['gt', 4095, 'match'], + ['gt', 4096, 'no_match'], + ['gte', 4096, 'match'], + ['gte', 4097, 'no_match'], + ['lt', 4097, 'match'], + ['lt', 4096, 'no_match'], + ['lte', 4096, 'match'], + ['lte', 4095, 'no_match'], + ['eq', 4096, 'match'], + ['eq', 4095, 'no_match'], + ] as const)('width %s %d → %s', (operator, right, outcome) => { + const result = evaluateFixedPolicyCondition( + cmp({ field: 'resource.width' }, operator, { value: right }), + CONTEXT, + ); + expect(result.outcome).toBe(outcome); + }); + + it('compares field to field (the §8.3 keyframe-extraction example)', () => { + const result = evaluateFixedPolicyCondition( + cmp({ field: 'resource.estimatedTokenCount' }, 'gt', { + field: 'session.availableContextTokens', + }), + CONTEXT, + ); + expect(result).toEqual({ outcome: 'match' }); + }); + + it('compares literal to literal', () => { + expect( + evaluateFixedPolicyCondition( + cmp({ value: 2 }, 'lt', { value: 3 }), + CONTEXT, + ).outcome, + ).toBe('match'); + }); + + it('eq supports strict string/boolean equality; type mismatch is a determinate no_match', () => { + expect( + evaluateFixedPolicyCondition( + cmp({ value: 'aac' }, 'eq', { value: 'aac' }), + CONTEXT, + ).outcome, + ).toBe('match'); + expect( + evaluateFixedPolicyCondition( + cmp({ value: '3' }, 'eq', { value: 3 }), + CONTEXT, + ).outcome, + ).toBe('no_match'); + }); + + it('an absent field is unavailable, never false', () => { + const result = evaluateFixedPolicyCondition( + cmp({ field: 'resource.durationMs' }, 'gt', { value: 0 }), + CONTEXT, + ); + expect(result).toEqual({ + outcome: 'unavailable', + missingFields: ['resource.durationMs'], + }); + }); + + it('an unknown field name is unavailable and named', () => { + const result = evaluateFixedPolicyCondition( + cmp({ field: 'resource.doesNotExist' }, 'gt', { value: 0 }), + CONTEXT, + ); + expect(result).toMatchObject({ + outcome: 'unavailable', + missingFields: ['resource.doesNotExist'], + }); + }); + + it('both operands missing → both fields recorded', () => { + const result = evaluateFixedPolicyCondition( + cmp({ field: 'resource.bitRate' }, 'gt', { + field: 'resource.sampleRateHz', + }), + CONTEXT, + ); + expect(result).toEqual({ + outcome: 'unavailable', + missingFields: ['resource.bitRate', 'resource.sampleRateHz'], + }); + }); + + it('ordering over a non-numeric literal is unavailable, not false', () => { + const result = evaluateFixedPolicyCondition( + cmp({ field: 'resource.width' }, 'gt', { value: 'wide' }), + CONTEXT, + ); + expect(result).toMatchObject({ outcome: 'unavailable' }); + }); +}); + +describe('evaluateFixedPolicyCondition — combinators (strong Kleene)', () => { + const TRUE = cmp({ value: 1 }, 'eq', { value: 1 }); + const FALSE = cmp({ value: 1 }, 'eq', { value: 2 }); + const UNAVAILABLE = cmp({ field: 'resource.durationMs' }, 'gt', { + value: 0, + }); + + it('all: every branch true → match', () => { + expect( + evaluateFixedPolicyCondition({ all: [TRUE, TRUE] }, CONTEXT).outcome, + ).toBe('match'); + }); + + it('all: a false branch dominates an unavailable sibling', () => { + expect( + evaluateFixedPolicyCondition({ all: [UNAVAILABLE, FALSE] }, CONTEXT) + .outcome, + ).toBe('no_match'); + }); + + it('all: true + unavailable → unavailable with the missing field', () => { + expect( + evaluateFixedPolicyCondition({ all: [TRUE, UNAVAILABLE] }, CONTEXT), + ).toEqual({ + outcome: 'unavailable', + missingFields: ['resource.durationMs'], + }); + }); + + it('any: a true branch dominates an unavailable sibling', () => { + expect( + evaluateFixedPolicyCondition({ any: [UNAVAILABLE, TRUE] }, CONTEXT) + .outcome, + ).toBe('match'); + }); + + it('any: every branch false → no_match', () => { + expect( + evaluateFixedPolicyCondition({ any: [FALSE, FALSE] }, CONTEXT).outcome, + ).toBe('no_match'); + }); + + it('any: false + unavailable → unavailable', () => { + expect( + evaluateFixedPolicyCondition({ any: [FALSE, UNAVAILABLE] }, CONTEXT), + ).toEqual({ + outcome: 'unavailable', + missingFields: ['resource.durationMs'], + }); + }); + + it('nests recursively and dedups missing fields', () => { + const result = evaluateFixedPolicyCondition( + { + any: [ + { all: [UNAVAILABLE, TRUE] }, + cmp({ field: 'resource.durationMs' }, 'lt', { value: 100 }), + ], + }, + CONTEXT, + ); + expect(result).toEqual({ + outcome: 'unavailable', + missingFields: ['resource.durationMs'], + }); + }); + + it('vacuous combinators: all [] → match, any [] → no_match', () => { + expect(evaluateFixedPolicyCondition({ all: [] }, CONTEXT).outcome).toBe( + 'match', + ); + expect(evaluateFixedPolicyCondition({ any: [] }, CONTEXT).outcome).toBe( + 'no_match', + ); + }); + + it('never throws on malformed nodes — degrades to unavailable', () => { + for (const bad of [ + null, + 42, + 'gt', + {}, + { all: 'not-an-array' }, + { left: { field: 'resource.width' } }, // no operator/right + { left: {}, operator: 'between', right: {} }, + ]) { + const result = evaluateFixedPolicyCondition( + bad as unknown as FixedPolicyCondition, + CONTEXT, + ); + expect(result.outcome).toBe('unavailable'); + } + }); +}); + +describe('validateFixedPolicyCondition', () => { + it('accepts the §8.3 documentation example', () => { + expect( + validateFixedPolicyCondition({ + all: [ + { + left: { field: 'resource.estimatedTokenCount' }, + operator: 'gt', + right: { field: 'session.availableContextTokens' }, + }, + { + left: { field: 'session.contextWindowTokens' }, + operator: 'gte', + right: { value: 131072 }, + }, + ], + }), + ).toEqual([]); + }); + + it.each([ + ['non-object root', 7, /must be an object/], + ['empty object', {}, /exactly one of/], + ['both all and any', { all: [], any: [] }, /exactly one of/], + ['empty all', { all: [] }, /non-empty array/], + ['non-array any', { any: {} }, /non-empty array/], + [ + 'unknown operator', + { + left: { value: 1 }, + operator: 'between', + right: { value: 2 }, + }, + /operator/, + ], + [ + 'unknown field', + { + left: { field: 'resource.nope' }, + operator: 'gt', + right: { value: 1 }, + }, + /unknown field/, + ], + [ + 'operand with both field and value', + { + left: { field: 'resource.width', value: 1 }, + operator: 'gt', + right: { value: 1 }, + }, + /exactly one of "field"\/"value"/, + ], + [ + 'operand with neither field nor value', + { left: {}, operator: 'gt', right: { value: 1 } }, + /exactly one of "field"\/"value"/, + ], + [ + 'ordering operator with a string literal', + { + left: { field: 'resource.width' }, + operator: 'gt', + right: { value: 'wide' }, + }, + /requires a finite numeric literal/, + ], + [ + 'non-primitive literal', + { + left: { value: { nested: true } }, + operator: 'eq', + right: { value: 1 }, + }, + /number, string, or boolean/, + ], + ])('rejects %s', (_label, raw, pattern) => { + const errors = validateFixedPolicyCondition(raw); + expect(errors.length).toBeGreaterThan(0); + expect(errors.join('\n')).toMatch(pattern); + }); + + it('eq allows string and boolean literals', () => { + expect( + validateFixedPolicyCondition({ + left: { value: true }, + operator: 'eq', + right: { value: 'x' }, + }), + ).toEqual([]); + }); + + it('reports nested paths for errors inside combinators', () => { + const errors = validateFixedPolicyCondition({ + any: [ + { + all: [{ left: { value: 1 }, operator: 'nope', right: { value: 2 } }], + }, + ], + }); + expect(errors.join('\n')).toContain('when.any[0].all[0].operator'); + }); +}); diff --git a/packages/core/src/omni/policy/conditions.ts b/packages/core/src/omni/policy/conditions.ts new file mode 100644 index 00000000000..7bac96e0073 --- /dev/null +++ b/packages/core/src/omni/policy/conditions.ts @@ -0,0 +1,345 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Restricted `when` condition DSL for fixed policies (policy design §8.3). + * + * Conditions are recursive compositions of `all` / `any` over comparisons; + * comparisons support `gt|gte|lt|lte|eq` with fields or literals on either + * side. No arbitrary code, no JSONPath. Evaluation is three-valued: a + * comparison over a field that cannot be resolved yields `unavailable`, + * which must NEVER be silently treated as false — the caller applies the + * policy's `onConditionUnavailable` behavior (default: skip) and the run + * record names the missing fields. + */ + +/** Readable fields, grouped in their three natural namespaces. */ +export const RESOURCE_CONDITION_FIELDS = [ + 'sizeBytes', + 'durationMs', + 'width', + 'height', + 'maxWidth', + 'maxHeight', + 'frameRate', + 'frameCount', + 'bitRate', + 'sampleRateHz', + 'channels', + 'estimatedTokenCount', +] as const; +export const REQUEST_CONDITION_FIELDS = ['totalEstimatedMediaTokens'] as const; +export const SESSION_CONDITION_FIELDS = [ + 'contextWindowTokens', + 'promptTokenCount', + 'reservedOutputTokens', + 'availableContextTokens', +] as const; + +export type ResourceConditionField = (typeof RESOURCE_CONDITION_FIELDS)[number]; +export type RequestConditionField = (typeof REQUEST_CONDITION_FIELDS)[number]; +export type SessionConditionField = (typeof SESSION_CONDITION_FIELDS)[number]; + +/** Fully-qualified field name, e.g. `resource.sizeBytes`. */ +export type FixedPolicyField = + | `resource.${ResourceConditionField}` + | `request.${RequestConditionField}` + | `session.${SessionConditionField}`; + +export type ConditionOperand = + | { field: FixedPolicyField } + | { value: number | string | boolean }; + +export type ComparisonOperator = 'gt' | 'gte' | 'lt' | 'lte' | 'eq'; + +export interface ComparisonCondition { + left: ConditionOperand; + operator: ComparisonOperator; + right: ConditionOperand; +} + +export type FixedPolicyCondition = + | ComparisonCondition + | { all: FixedPolicyCondition[] } + | { any: FixedPolicyCondition[] }; + +/** Values feeding field resolution. All defined fields are numeric; an + * absent entry means the field could not be obtained for this resource + * (e.g. `durationMs` for an image, or a probe that returned nothing). */ +export interface FixedPolicyConditionContext { + resource?: Partial>; + request?: Partial>; + session?: Partial>; +} + +export type ConditionEvaluation = + | { outcome: 'match' } + | { outcome: 'no_match' } + | { + outcome: 'unavailable'; + /** Field names (or operand descriptions) that made the result + * undecidable — surfaced in the policy run record. */ + missingFields: string[]; + }; + +const MATCH: ConditionEvaluation = { outcome: 'match' }; +const NO_MATCH: ConditionEvaluation = { outcome: 'no_match' }; + +function unavailable(missingFields: string[]): ConditionEvaluation { + return { outcome: 'unavailable', missingFields: [...new Set(missingFields)] }; +} + +function isPlainObject(v: unknown): v is Record { + return typeof v === 'object' && v !== null && !Array.isArray(v); +} + +const OPERATORS: readonly ComparisonOperator[] = [ + 'gt', + 'gte', + 'lt', + 'lte', + 'eq', +]; + +const KNOWN_FIELDS: ReadonlySet = new Set([ + ...RESOURCE_CONDITION_FIELDS.map((f) => `resource.${f}`), + ...REQUEST_CONDITION_FIELDS.map((f) => `request.${f}`), + ...SESSION_CONDITION_FIELDS.map((f) => `session.${f}`), +]); + +type ResolvedOperand = + | { ok: true; value: number | string | boolean; describe: string } + | { ok: false; missing: string }; + +function resolveOperand( + operand: ConditionOperand, + context: FixedPolicyConditionContext, +): ResolvedOperand { + if ('value' in operand) { + return { ok: true, value: operand.value, describe: 'value' }; + } + const field = operand.field; + if (!KNOWN_FIELDS.has(field)) { + return { ok: false, missing: field }; + } + const [namespace, name] = field.split('.') as [ + 'resource' | 'request' | 'session', + string, + ]; + const value = ( + context[namespace] as Record | undefined + )?.[name]; + if (typeof value !== 'number' || Number.isNaN(value)) { + return { ok: false, missing: field }; + } + return { ok: true, value, describe: field }; +} + +function evaluateComparison( + condition: ComparisonCondition, + context: FixedPolicyConditionContext, +): ConditionEvaluation { + const left = resolveOperand(condition.left, context); + const right = resolveOperand(condition.right, context); + if (!left.ok || !right.ok) { + const missing: string[] = []; + if (!left.ok) missing.push(left.missing); + if (!right.ok) missing.push(right.missing); + return unavailable(missing); + } + if (condition.operator === 'eq') { + // Strict equality: a type mismatch between two AVAILABLE values is a + // determinate not-equal, not an unavailability. + return left.value === right.value ? MATCH : NO_MATCH; + } + // Ordering requires two finite numbers; anything else cannot be ordered + // and must not silently collapse to false. + if ( + typeof left.value !== 'number' || + typeof right.value !== 'number' || + !Number.isFinite(left.value) || + !Number.isFinite(right.value) + ) { + const missing: string[] = []; + if (typeof left.value !== 'number' || !Number.isFinite(left.value)) { + missing.push(`${left.describe} (not orderable)`); + } + if (typeof right.value !== 'number' || !Number.isFinite(right.value)) { + missing.push(`${right.describe} (not orderable)`); + } + return unavailable(missing); + } + switch (condition.operator) { + case 'gt': + return left.value > right.value ? MATCH : NO_MATCH; + case 'gte': + return left.value >= right.value ? MATCH : NO_MATCH; + case 'lt': + return left.value < right.value ? MATCH : NO_MATCH; + case 'lte': + return left.value <= right.value ? MATCH : NO_MATCH; + default: { + const exhaustive: never = condition.operator; + return unavailable([`unknown operator ${String(exhaustive)}`]); + } + } +} + +/** + * Evaluate a `when` condition against a context snapshot. Total function — + * never throws, even on structurally malformed input (which startup + * validation rejects; anything that slips through degrades to + * `unavailable`, the fail-safe outcome). + * + * Combinators use strong Kleene logic so `unavailable` propagates only + * when it is actually decisive: `all` with a false branch is false + * regardless of an unavailable sibling; `any` with a true branch is true. + */ +export function evaluateFixedPolicyCondition( + condition: FixedPolicyCondition, + context: FixedPolicyConditionContext, +): ConditionEvaluation { + // Deliberately typed as a loose record: the type predicate would narrow + // `condition` itself into a `never` after the combinator checks, and this + // total function must handle malformed nodes anyway. + const node: unknown = condition; + if (!isPlainObject(node)) { + return unavailable(['']); + } + if ('all' in node || 'any' in node) { + const isAll = 'all' in node; + const children = isAll ? node['all'] : node['any']; + if (!Array.isArray(children)) { + return unavailable(['']); + } + const missing: string[] = []; + let sawUnavailable = false; + for (const child of children) { + const result = evaluateFixedPolicyCondition( + child as FixedPolicyCondition, + context, + ); + if (result.outcome === 'unavailable') { + sawUnavailable = true; + missing.push(...result.missingFields); + continue; + } + // Dominant outcomes short-circuit: false for `all`, true for `any`. + if (isAll && result.outcome === 'no_match') return NO_MATCH; + if (!isAll && result.outcome === 'match') return MATCH; + } + if (sawUnavailable) return unavailable(missing); + return isAll ? MATCH : NO_MATCH; + } + if ( + 'left' in node && + 'operator' in node && + 'right' in node && + isPlainObject(node['left']) && + isPlainObject(node['right']) && + OPERATORS.includes(node['operator'] as ComparisonOperator) + ) { + return evaluateComparison(node as unknown as ComparisonCondition, context); + } + return unavailable(['']); +} + +function validateOperand( + raw: unknown, + operator: ComparisonOperator | undefined, + where: string, + errors: string[], +): void { + if (!isPlainObject(raw)) { + errors.push(`${where}: operand must be an object`); + return; + } + const hasField = 'field' in raw; + const hasValue = 'value' in raw; + if (hasField === hasValue) { + errors.push(`${where}: operand must have exactly one of "field"/"value"`); + return; + } + if (hasField) { + if (typeof raw['field'] !== 'string' || !KNOWN_FIELDS.has(raw['field'])) { + errors.push(`${where}: unknown field ${JSON.stringify(raw['field'])}`); + } + return; + } + const value = raw['value']; + if ( + typeof value !== 'number' && + typeof value !== 'string' && + typeof value !== 'boolean' + ) { + errors.push(`${where}: literal must be a number, string, or boolean`); + return; + } + if (operator !== 'eq' && operator !== undefined) { + if (typeof value !== 'number' || !Number.isFinite(value)) { + errors.push( + `${where}: operator "${operator}" requires a finite numeric literal`, + ); + } + } +} + +/** + * Structural validation for a raw `when` condition (policy design §13 #5), + * run at config-normalization time. Returns a list of human-readable + * errors; empty means valid. + */ +export function validateFixedPolicyCondition( + raw: unknown, + where = 'when', +): string[] { + const errors: string[] = []; + if (!isPlainObject(raw)) { + errors.push(`${where}: condition must be an object`); + return errors; + } + const keys = ['all', 'any', 'left'].filter((k) => k in raw); + if (keys.length !== 1) { + errors.push( + `${where}: condition must be exactly one of a comparison, "all", or "any"`, + ); + return errors; + } + if ('all' in raw || 'any' in raw) { + const key = 'all' in raw ? 'all' : 'any'; + const children = raw[key]; + if (!Array.isArray(children) || children.length === 0) { + errors.push(`${where}.${key}: must be a non-empty array of conditions`); + return errors; + } + children.forEach((child, i) => { + errors.push( + ...validateFixedPolicyCondition(child, `${where}.${key}[${i}]`), + ); + }); + return errors; + } + const operator = raw['operator']; + const knownOperator = OPERATORS.includes(operator as ComparisonOperator); + if (!knownOperator) { + errors.push( + `${where}.operator: must be one of ${OPERATORS.join(', ')} (got ${JSON.stringify(operator)})`, + ); + } + validateOperand( + raw['left'], + knownOperator ? (operator as ComparisonOperator) : undefined, + `${where}.left`, + errors, + ); + validateOperand( + raw['right'], + knownOperator ? (operator as ComparisonOperator) : undefined, + `${where}.right`, + errors, + ); + return errors; +} diff --git a/packages/core/src/omni/policy/types.ts b/packages/core/src/omni/policy/types.ts index 247f7004788..8652c476353 100644 --- a/packages/core/src/omni/policy/types.ts +++ b/packages/core/src/omni/policy/types.ts @@ -22,6 +22,15 @@ export type { MediaPolicyToolDescriptor, MediaPolicyToolOutputSpec, } from '../../tools/tools.js'; +export type { + ComparisonCondition, + ComparisonOperator, + ConditionEvaluation, + ConditionOperand, + FixedPolicyCondition, + FixedPolicyConditionContext, + FixedPolicyField, +} from './conditions.js'; /** * Raw (pre-normalization) shape of one From 7a66122f56e77e4b67521d9277acc33fcdf7374c Mon Sep 17 00:00:00 2001 From: LaZzyMan Date: Fri, 7 Aug 2026 10:08:43 +0800 Subject: [PATCH 06/62] feat(omni): add three-modality degradation media-policy tools MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Commit 6 of the S4 policy pipeline: the three built-in degradation tools (mapping doc §6), each a real BaseDeclarativeTool carrying a media_policy descriptor so the orchestrator and the modelAccess gate can key off code-level facts. - omni/policy/tools/media-policy-tool.ts: shared base — validates against the NATIVE parameter schema (never the model-visible projection Stage B will narrow), async io assertions (lstat: symlink input refused, real output directory required), per-tool policyTools..runtime.timeoutMs resolution (default 600s), uniform success/error ToolResults; success emits exactly one lossy artifact whose metadata.omniDisclosure carries the D8 disclosure and whose workspacePath is staging-relative. - omni_downsample_image (sharp, lazy-loaded per D9; load failure is an execution failure): fit 1568px / JPEG q75, EXIF orientation baked in, animated inputs refused outright (sharp would silently keep only the first frame). - omni_downscale_video (ffmpeg): scale to even target height computed in JS (no filtergraph expressions), fps 10, x264 crf 28 veryfast; audio stream-copy with AAC 64k fallback. - omni_downsample_audio (ffmpeg): AAC 64kbps/16kHz/mono, -vn strips cover art. - omni/ffmpeg.ts: runFfmpeg — never rejects, callers branch on the exit code and must check signal.aborted explicitly. - Registered lazily behind isOmniEnabled(); the commit-3 modelAccess gate keeps them hidden from model surfaces by default. --- packages/core/src/config/config.ts | 26 ++ packages/core/src/omni/ffmpeg.test.ts | 49 ++++ packages/core/src/omni/ffmpeg.ts | 29 ++ .../policy/tools/downsample-audio.test.ts | 222 +++++++++++++++ .../src/omni/policy/tools/downsample-audio.ts | 225 +++++++++++++++ .../policy/tools/downsample-image.test.ts | 213 ++++++++++++++ .../src/omni/policy/tools/downsample-image.ts | 242 ++++++++++++++++ .../omni/policy/tools/downscale-video.test.ts | 254 +++++++++++++++++ .../src/omni/policy/tools/downscale-video.ts | 260 ++++++++++++++++++ .../policy/tools/media-policy-tool.test.ts | 237 ++++++++++++++++ .../omni/policy/tools/media-policy-tool.ts | 201 ++++++++++++++ packages/core/src/tools/tool-names.ts | 8 + 12 files changed, 1966 insertions(+) create mode 100644 packages/core/src/omni/policy/tools/downsample-audio.test.ts create mode 100644 packages/core/src/omni/policy/tools/downsample-audio.ts create mode 100644 packages/core/src/omni/policy/tools/downsample-image.test.ts create mode 100644 packages/core/src/omni/policy/tools/downsample-image.ts create mode 100644 packages/core/src/omni/policy/tools/downscale-video.test.ts create mode 100644 packages/core/src/omni/policy/tools/downscale-video.ts create mode 100644 packages/core/src/omni/policy/tools/media-policy-tool.test.ts create mode 100644 packages/core/src/omni/policy/tools/media-policy-tool.ts diff --git a/packages/core/src/config/config.ts b/packages/core/src/config/config.ts index 7698ec5a460..903a55beea7 100644 --- a/packages/core/src/config/config.ts +++ b/packages/core/src/config/config.ts @@ -8053,6 +8053,32 @@ export class Config { } // Register monitor tool + // Omni media-policy tools: always registered when omni is enabled (the + // fixed-policy orchestrator must be able to find them), but hidden from + // every model-facing surface unless + // `omni.processing.policyTools..modelAccess.enabled` opens them up + // (see omni/policy/model-access.ts). + if (this.isOmniEnabled()) { + await registerLazy(ToolNames.OMNI_DOWNSAMPLE_IMAGE, async () => { + const { OmniDownsampleImageTool } = await import( + '../omni/policy/tools/downsample-image.js' + ); + return new OmniDownsampleImageTool(); + }); + await registerLazy(ToolNames.OMNI_DOWNSCALE_VIDEO, async () => { + const { OmniDownscaleVideoTool } = await import( + '../omni/policy/tools/downscale-video.js' + ); + return new OmniDownscaleVideoTool(this); + }); + await registerLazy(ToolNames.OMNI_DOWNSAMPLE_AUDIO, async () => { + const { OmniDownsampleAudioTool } = await import( + '../omni/policy/tools/downsample-audio.js' + ); + return new OmniDownsampleAudioTool(this); + }); + } + await registerLazy(ToolNames.MONITOR, async () => { const { MonitorTool } = await import('../tools/monitor.js'); return new MonitorTool(this); diff --git a/packages/core/src/omni/ffmpeg.test.ts b/packages/core/src/omni/ffmpeg.test.ts index dd6930efae3..78a380be7a7 100644 --- a/packages/core/src/omni/ffmpeg.test.ts +++ b/packages/core/src/omni/ffmpeg.test.ts @@ -17,6 +17,7 @@ import { isFfprobeAvailable, probeMediaMetadata, resetFfmpegCachesForTests, + runFfmpeg, } from './ffmpeg.js'; type ExecCallback = ( @@ -343,3 +344,51 @@ describe('probeMediaMetadata per-modality branches', () => { expect(result.codec).toBeUndefined(); }); }); + +describe('runFfmpeg', () => { + it('invokes ffmpeg with the given args and resolves code 0 on success', async () => { + mockExecResult(() => ({ stderr: 'frame= 100' })); + await expect( + runFfmpeg(['-y', '-i', '/in.mov', '/out.mp4']), + ).resolves.toEqual({ + code: 0, + stderr: 'frame= 100', + }); + const [command, args, options] = execFileMock.mock.calls[0]; + expect(command).toBe('ffmpeg'); + expect(args).toEqual(['-y', '-i', '/in.mov', '/out.mp4']); + expect(options).toMatchObject({ maxBuffer: 16 * 1024 * 1024 }); + expect(options).not.toHaveProperty('timeout'); + }); + + it('threads timeoutMs and signal into execFile options', async () => { + mockExecResult(() => ({})); + const signal = new AbortController().signal; + await runFfmpeg(['-version'], { signal, timeoutMs: 120_000 }); + const options = execFileMock.mock.calls[0][2]; + expect(options).toMatchObject({ timeout: 120_000, signal }); + }); + + it('never rejects: a failing run resolves with the exit code and stderr', async () => { + mockExecResult(() => ({ + error: Object.assign(new Error('exit 187'), { code: 187 }), + stderr: 'Conversion failed!', + })); + await expect(runFfmpeg(['-i', '/in.mov'])).resolves.toEqual({ + code: 187, + stderr: 'Conversion failed!', + }); + }); + + it('maps a non-numeric error code (e.g. ENOENT/abort kill) to 1', async () => { + mockExecResult(() => ({ + error: Object.assign(new Error('spawn ffmpeg ENOENT'), { + code: 'ENOENT', + }), + })); + await expect(runFfmpeg(['-version'])).resolves.toEqual({ + code: 1, + stderr: '', + }); + }); +}); diff --git a/packages/core/src/omni/ffmpeg.ts b/packages/core/src/omni/ffmpeg.ts index 289b09e6f34..295947f4643 100644 --- a/packages/core/src/omni/ffmpeg.ts +++ b/packages/core/src/omni/ffmpeg.ts @@ -121,6 +121,35 @@ export async function assertOmniRuntimeDependencies(): Promise { ); } +/** Outcome of one ffmpeg run (see {@link runFfmpeg}). */ +export interface FfmpegRunResult { + /** Process exit code (non-zero on failure, including timeout kill). */ + code: number; + /** Captured stderr (ffmpeg writes its diagnostics there). */ + stderr: string; +} + +/** + * Run ffmpeg with the given arguments. Never rejects — callers branch on + * the exit code, and MUST check `signal?.aborted` explicitly afterwards + * (an aborted run also surfaces as a non-zero code, but the two need + * different error messages). `timeoutMs` kills the process when exceeded, + * which likewise surfaces as a non-zero exit code. + */ +export async function runFfmpeg( + args: string[], + options?: { signal?: AbortSignal; timeoutMs?: number }, +): Promise { + const { code, stderr } = await execCommand('ffmpeg', args, { + // Transcodes are long-running; stderr carries progress lines, so give + // it more headroom than the probe calls. + maxBuffer: 16 * 1024 * 1024, + ...(options?.timeoutMs !== undefined && { timeout: options.timeoutMs }), + ...(options?.signal && { signal: options.signal }), + }); + return { code, stderr }; +} + /** Media metadata extracted via ffprobe (fields populated per modality). */ export interface MediaProbeResult { /** Container/format name reported by ffprobe (e.g. "mov,mp4,m4a,..."). */ diff --git a/packages/core/src/omni/policy/tools/downsample-audio.test.ts b/packages/core/src/omni/policy/tools/downsample-audio.test.ts new file mode 100644 index 00000000000..9e167e7b42d --- /dev/null +++ b/packages/core/src/omni/policy/tools/downsample-audio.test.ts @@ -0,0 +1,222 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import type { MediaProbeResult } from '../../ffmpeg.js'; +import type { ToolResult } from '../../../tools/tools.js'; +import { DEFAULT_POLICY_TOOL_TIMEOUT_MS } from './media-policy-tool.js'; +import { + DOWNSAMPLE_AUDIO_DEFAULTS, + OMNI_DOWNSAMPLE_AUDIO_TOOL_NAME, + OmniDownsampleAudioTool, +} from './downsample-audio.js'; + +const mocks = vi.hoisted(() => ({ + probeMediaMetadata: vi.fn(), + runFfmpeg: vi.fn(), +})); + +vi.mock('../../ffmpeg.js', () => ({ + probeMediaMetadata: mocks.probeMediaMetadata, + runFfmpeg: mocks.runFfmpeg, +})); + +const INPUT_SIZE = 1024 ** 2; // "1MB" +const OUTPUT_SIZE = 120 * 1024; // "120KB" + +describe('OmniDownsampleAudioTool', () => { + let root: string; + let inputPath: string; + let outputDir: string; + + const tool = new OmniDownsampleAudioTool({}); + + const probe = (result: Partial): void => { + mocks.probeMediaMetadata.mockResolvedValue(result as MediaProbeResult); + }; + + const run = async ( + params: Record = {}, + toolInstance: OmniDownsampleAudioTool = tool, + ): Promise<{ result: ToolResult; signal: AbortSignal }> => { + const invocation = toolInstance.build({ + inputPath, + outputDir, + ...params, + } as never); + const signal = new AbortController().signal; + return { result: await invocation.execute(signal), signal }; + }; + + beforeEach(async () => { + vi.clearAllMocks(); + root = await fs.mkdtemp(path.join(os.tmpdir(), 'omni-aud-')); + inputPath = path.join(root, 'track.wav'); + await fs.writeFile(inputPath, Buffer.alloc(INPUT_SIZE)); + outputDir = path.join(root, 'staging'); + await fs.mkdir(outputDir); + mocks.runFfmpeg.mockImplementation(async (args: string[]) => { + await fs.writeFile(args[args.length - 1], Buffer.alloc(OUTPUT_SIZE)); + return { code: 0, stderr: '' }; + }); + }); + + afterEach(async () => { + await fs.rm(root, { recursive: true, force: true }); + }); + + it('declares the media-policy descriptor and defaults', () => { + expect(tool.name).toBe(OMNI_DOWNSAMPLE_AUDIO_TOOL_NAME); + expect(tool.mediaPolicyDescriptor).toEqual({ + kind: 'media_policy', + inputMediaTypes: ['audio'], + outputs: [ + { + kind: 'media', + mimeTypes: ['audio/mp4'], + required: true, + lossy: true, + }, + ], + settingsSchema: expect.objectContaining({ type: 'object' }), + }); + expect(DOWNSAMPLE_AUDIO_DEFAULTS).toEqual({ + bitrateKbps: 64, + sampleRateHz: 16_000, + channels: 1, + }); + }); + + it('downsamples with the fixed-call defaults and disclosure (D8)', async () => { + probe({ bitRate: 320_000, sampleRateHz: 48_000, channels: 2 }); + const { result, signal } = await run(); + + expect(mocks.probeMediaMetadata).toHaveBeenCalledWith( + inputPath, + 'audio', + signal, + ); + const outputPath = path.join(outputDir, 'downsampled.m4a'); + expect(mocks.runFfmpeg).toHaveBeenCalledTimes(1); + expect(mocks.runFfmpeg).toHaveBeenCalledWith( + [ + '-y', + '-i', + inputPath, + '-vn', + '-c:a', + 'aac', + '-b:a', + '64k', + '-ar', + '16000', + '-ac', + '1', + outputPath, + ], + { signal, timeoutMs: DEFAULT_POLICY_TOOL_TIMEOUT_MS }, + ); + + expect(result.error).toBeUndefined(); + expect(result.artifacts).toEqual([ + { + kind: 'audio', + storage: 'workspace', + title: 'Downsampled audio', + workspacePath: 'downsampled.m4a', + mimeType: 'audio/mp4', + sizeBytes: OUTPUT_SIZE, + metadata: { + omniDisclosure: + '原 320kbps/48kHz 立体声 → 64kbps/16kHz 单声道，高频细节丢失', + }, + }, + ]); + }); + + it('falls back to input byte size when the probe lacks a bit rate', async () => { + probe({ sampleRateHz: 44_100 }); + const { result } = await run(); + expect(result.artifacts?.[0]?.metadata?.['omniDisclosure']).toBe( + '原 1MB/44kHz → 64kbps/16kHz 单声道，高频细节丢失', + ); + }); + + it('threads tunable overrides into the ffmpeg args and disclosure', async () => { + probe({ bitRate: 256_000, sampleRateHz: 48_000, channels: 6 }); + const { result } = await run({ + bitrateKbps: 96, + sampleRateHz: 24_000, + channels: 2, + }); + const args = mocks.runFfmpeg.mock.calls[0][0] as string[]; + expect(args.join(' ')).toContain('-b:a 96k -ar 24000 -ac 2'); + expect(result.artifacts?.[0]?.metadata?.['omniDisclosure']).toBe( + '原 256kbps/48kHz 6声道 → 96kbps/24kHz 立体声，高频细节丢失', + ); + }); + + it('threads policyTools..runtime.timeoutMs into runFfmpeg', async () => { + probe({ bitRate: 128_000 }); + const configured = new OmniDownsampleAudioTool({ + getOmniPolicyToolsSettings: () => ({ + [OMNI_DOWNSAMPLE_AUDIO_TOOL_NAME]: { + runtime: { timeoutMs: 90_000 }, + }, + }), + }); + await run({}, configured); + expect(mocks.runFfmpeg).toHaveBeenCalledWith( + expect.any(Array), + expect.objectContaining({ timeoutMs: 90_000 }), + ); + }); + + it('reports the ffmpeg error on a non-zero exit', async () => { + probe({ bitRate: 128_000 }); + mocks.runFfmpeg.mockResolvedValue({ + code: 1, + stderr: 'Invalid data found when processing input', + }); + const { result } = await run(); + expect(result.error?.message).toMatch(/ffmpeg failed \(exit 1\)/); + expect(result.error?.message).toContain('Invalid data found'); + expect(result.artifacts).toBeUndefined(); + }); + + it('reports an aborted run', async () => { + probe({ bitRate: 128_000 }); + const controller = new AbortController(); + mocks.runFfmpeg.mockImplementation(async () => { + controller.abort(); + return { code: 1, stderr: '' }; + }); + const invocation = tool.build({ inputPath, outputDir }); + const result = await invocation.execute(controller.signal); + expect(result.error?.message).toBe('audio downsampling aborted'); + }); + + it('returns an error result when the input is a symlink', async () => { + const link = path.join(root, 'link.wav'); + await fs.symlink(inputPath, link); + const { result } = await run({ inputPath: link }); + expect(result.error?.message).toMatch(/not a regular file/); + expect(mocks.runFfmpeg).not.toHaveBeenCalled(); + }); + + it.each([ + ['relative inputPath', { inputPath: 'track.wav' }], + ['unknown property', { loudness: 5 }], + ['channels out of range', { channels: 3 }], + ])('build rejects %s', (_label, overrides) => { + expect(() => + tool.build({ inputPath, outputDir, ...overrides } as never), + ).toThrow(); + }); +}); diff --git a/packages/core/src/omni/policy/tools/downsample-audio.ts b/packages/core/src/omni/policy/tools/downsample-audio.ts new file mode 100644 index 00000000000..9f0cf639fe2 --- /dev/null +++ b/packages/core/src/omni/policy/tools/downsample-audio.ts @@ -0,0 +1,225 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import fs from 'node:fs/promises'; +import path from 'node:path'; +import type { + MediaPolicyToolDescriptor, + ToolInvocation, + ToolResult, +} from '../../../tools/tools.js'; +import { BaseToolInvocation, Kind } from '../../../tools/tools.js'; +import { probeMediaMetadata, runFfmpeg } from '../../ffmpeg.js'; +import { + assertMediaPolicyIo, + BaseMediaPolicyTool, + formatBytesShort, + MEDIA_POLICY_IO_SCHEMA_PROPERTIES, + mediaPolicyToolError, + mediaPolicyToolSuccess, + resolvePolicyToolTimeoutMs, + validateMediaPolicyIoParams, + type MediaPolicyIoParams, + type MediaPolicyToolConfigView, +} from './media-policy-tool.js'; + +export const OMNI_DOWNSAMPLE_AUDIO_TOOL_NAME = 'omni_downsample_audio'; + +/** Fixed-call default parameters (mapping doc §6). */ +export const DOWNSAMPLE_AUDIO_DEFAULTS = { + bitrateKbps: 64, + sampleRateHz: 16_000, + channels: 1, +} as const; + +const OUTPUT_FILE_NAME = 'downsampled.m4a'; + +export interface DownsampleAudioParams extends MediaPolicyIoParams { + /** Output bit rate in kbit/s. */ + bitrateKbps?: number; + /** Output sample rate in Hz. */ + sampleRateHz?: number; + /** Output channel count. */ + channels?: number; +} + +const TUNABLE_SCHEMA_PROPERTIES = { + bitrateKbps: { + type: 'number', + description: 'Output bit rate in kbit/s. Default 64.', + minimum: 8, + }, + sampleRateHz: { + type: 'number', + description: 'Output sample rate in Hz. Default 16000.', + minimum: 8000, + }, + channels: { + type: 'number', + description: 'Output channel count. Default 1 (mono).', + minimum: 1, + maximum: 2, + }, +} as const; + +const DESCRIPTOR: MediaPolicyToolDescriptor = { + kind: 'media_policy', + inputMediaTypes: ['audio'], + outputs: [ + { + kind: 'media', + mimeTypes: ['audio/mp4'], + required: true, + lossy: true, + }, + ], + settingsSchema: { + type: 'object', + properties: TUNABLE_SCHEMA_PROPERTIES, + additionalProperties: false, + }, +}; + +/** "立体声" / "单声道" / "N声道" for the disclosure text. */ +function describeChannels(channels: number | undefined): string { + if (channels === undefined) return ''; + if (channels === 1) return ' 单声道'; + if (channels === 2) return ' 立体声'; + return ` ${channels}声道`; +} + +class DownsampleAudioInvocation extends BaseToolInvocation< + DownsampleAudioParams, + ToolResult +> { + constructor( + params: DownsampleAudioParams, + private readonly timeoutMs: number, + ) { + super(params); + } + + getDescription(): string { + const bitrateKbps = + this.params.bitrateKbps ?? DOWNSAMPLE_AUDIO_DEFAULTS.bitrateKbps; + return `Downsample ${path.basename(this.params.inputPath)} to ${bitrateKbps}kbps`; + } + + async execute(signal: AbortSignal): Promise { + const bitrateKbps = + this.params.bitrateKbps ?? DOWNSAMPLE_AUDIO_DEFAULTS.bitrateKbps; + const sampleRateHz = + this.params.sampleRateHz ?? DOWNSAMPLE_AUDIO_DEFAULTS.sampleRateHz; + const channels = this.params.channels ?? DOWNSAMPLE_AUDIO_DEFAULTS.channels; + try { + const { inputSizeBytes } = await assertMediaPolicyIo(this.params); + const probe = await probeMediaMetadata( + this.params.inputPath, + 'audio', + signal, + ); + + const outputPath = path.join(this.params.outputDir, OUTPUT_FILE_NAME); + const run = await runFfmpeg( + [ + '-y', + '-i', + this.params.inputPath, + // Audio-only output: a cover-art video stream would otherwise be + // carried along (and can even fail the m4a mux). + '-vn', + '-c:a', + 'aac', + '-b:a', + `${bitrateKbps}k`, + '-ar', + String(sampleRateHz), + '-ac', + String(channels), + outputPath, + ], + { signal, timeoutMs: this.timeoutMs }, + ); + if (signal.aborted) { + return mediaPolicyToolError('audio downsampling aborted'); + } + if (run.code !== 0) { + return mediaPolicyToolError( + `ffmpeg failed (exit ${run.code}) downsampling ${path.basename(this.params.inputPath)}: ${run.stderr.slice(-500)}`, + ); + } + + const outputSizeBytes = (await fs.stat(outputPath)).size; + const originalBitrate = + probe.bitRate !== undefined + ? `${Math.round(probe.bitRate / 1000)}kbps` + : formatBytesShort(inputSizeBytes); + const originalRate = + probe.sampleRateHz !== undefined + ? `/${Math.round(probe.sampleRateHz / 1000)}kHz` + : ''; + const disclosure = `原 ${originalBitrate}${originalRate}${describeChannels(probe.channels)} → ${bitrateKbps}kbps/${Math.round(sampleRateHz / 1000)}kHz${describeChannels(channels)}，高频细节丢失`; + + return mediaPolicyToolSuccess({ + outputDir: this.params.outputDir, + outputFileName: OUTPUT_FILE_NAME, + artifactKind: 'audio', + title: 'Downsampled audio', + mimeType: 'audio/mp4', + sizeBytes: outputSizeBytes, + disclosure, + }); + } catch (error) { + return mediaPolicyToolError( + error instanceof Error ? error.message : String(error), + ); + } + } +} + +/** + * `omni_downsample_audio` — lossy audio degradation (ffmpeg): re-encode + * to AAC at a low bit rate, sample rate, and channel count (mapping doc + * §6). + */ +export class OmniDownsampleAudioTool extends BaseMediaPolicyTool { + constructor(private readonly config: MediaPolicyToolConfigView) { + super( + OMNI_DOWNSAMPLE_AUDIO_TOOL_NAME, + 'DownsampleAudio', + 'Downsamples an audio file to a lower bit rate, sample rate, and channel count, producing a smaller lossy derivative with a disclosure of the degradation.', + Kind.Other, + { + type: 'object', + properties: { + ...MEDIA_POLICY_IO_SCHEMA_PROPERTIES, + ...TUNABLE_SCHEMA_PROPERTIES, + }, + required: ['inputPath', 'outputDir'], + additionalProperties: false, + }, + ); + } + + override get mediaPolicyDescriptor(): MediaPolicyToolDescriptor { + return DESCRIPTOR; + } + + protected override validateToolParamValues( + params: DownsampleAudioParams, + ): string | null { + return validateMediaPolicyIoParams(params); + } + + protected createInvocation( + params: DownsampleAudioParams, + ): ToolInvocation { + return new DownsampleAudioInvocation( + params, + resolvePolicyToolTimeoutMs(this.config, this.name), + ); + } +} diff --git a/packages/core/src/omni/policy/tools/downsample-image.test.ts b/packages/core/src/omni/policy/tools/downsample-image.test.ts new file mode 100644 index 00000000000..fdc9baf050a --- /dev/null +++ b/packages/core/src/omni/policy/tools/downsample-image.test.ts @@ -0,0 +1,213 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import type { MediaProbeResult } from '../../ffmpeg.js'; +import type { ToolResult } from '../../../tools/tools.js'; +import { + DOWNSAMPLE_IMAGE_DEFAULTS, + OMNI_DOWNSAMPLE_IMAGE_TOOL_NAME, + OmniDownsampleImageTool, +} from './downsample-image.js'; + +const mocks = vi.hoisted(() => ({ + probeMediaMetadata: vi.fn(), + runFfmpeg: vi.fn(), + sharpCreate: vi.fn(), +})); + +vi.mock('../../ffmpeg.js', () => ({ + probeMediaMetadata: mocks.probeMediaMetadata, + runFfmpeg: mocks.runFfmpeg, +})); + +vi.mock('sharp', () => ({ + default: (...args: unknown[]) => mocks.sharpCreate(...args), +})); + +const INPUT_SIZE = 2 * 1024 ** 2; // "2MB" +const OUTPUT_SIZE = 300 * 1024; // "300KB" + +describe('OmniDownsampleImageTool', () => { + let root: string; + let inputPath: string; + let outputDir: string; + let toFile: ReturnType; + let jpeg: ReturnType; + let resize: ReturnType; + let rotate: ReturnType; + + const tool = new OmniDownsampleImageTool(); + + const probe = (result: Partial): void => { + mocks.probeMediaMetadata.mockResolvedValue(result as MediaProbeResult); + }; + + const run = async ( + params: Record = {}, + ): Promise<{ result: ToolResult; signal: AbortSignal }> => { + const invocation = tool.build({ + inputPath, + outputDir, + ...params, + } as never); + const signal = new AbortController().signal; + return { result: await invocation.execute(signal), signal }; + }; + + beforeEach(async () => { + vi.clearAllMocks(); + root = await fs.mkdtemp(path.join(os.tmpdir(), 'omni-img-')); + inputPath = path.join(root, 'photo.png'); + await fs.writeFile(inputPath, Buffer.alloc(INPUT_SIZE)); + outputDir = path.join(root, 'staging'); + await fs.mkdir(outputDir); + + toFile = vi + .fn() + .mockResolvedValue({ width: 1568, height: 1176, size: OUTPUT_SIZE }); + jpeg = vi.fn(() => ({ toFile })); + resize = vi.fn(() => ({ jpeg })); + rotate = vi.fn(() => ({ resize })); + mocks.sharpCreate.mockReturnValue({ rotate }); + }); + + afterEach(async () => { + await fs.rm(root, { recursive: true, force: true }); + }); + + it('declares the media-policy descriptor and tool name', () => { + expect(tool.name).toBe(OMNI_DOWNSAMPLE_IMAGE_TOOL_NAME); + expect(tool.mediaPolicyDescriptor).toEqual({ + kind: 'media_policy', + inputMediaTypes: ['image'], + outputs: [ + { + kind: 'media', + mimeTypes: ['image/jpeg'], + required: true, + lossy: true, + }, + ], + settingsSchema: expect.objectContaining({ type: 'object' }), + }); + expect(DOWNSAMPLE_IMAGE_DEFAULTS).toEqual({ + maxDimension: 1568, + quality: 75, + }); + }); + + it('downsamples with the fixed-call defaults and disclosure (D8)', async () => { + probe({ width: 4096, height: 3072, frameCount: 1 }); + const { result, signal } = await run(); + + expect(mocks.probeMediaMetadata).toHaveBeenCalledWith( + inputPath, + 'image', + signal, + ); + expect(mocks.sharpCreate).toHaveBeenCalledWith(inputPath, { + failOn: 'error', + limitInputPixels: true, + }); + expect(rotate).toHaveBeenCalledOnce(); + expect(resize).toHaveBeenCalledWith({ + width: 1568, + height: 1568, + fit: 'inside', + withoutEnlargement: true, + }); + expect(jpeg).toHaveBeenCalledWith({ quality: 75 }); + expect(toFile).toHaveBeenCalledWith( + path.join(outputDir, 'downsampled.jpg'), + ); + + expect(result.error).toBeUndefined(); + expect(result.artifacts).toEqual([ + { + kind: 'image', + storage: 'workspace', + title: 'Downsampled image', + workspacePath: 'downsampled.jpg', + mimeType: 'image/jpeg', + sizeBytes: OUTPUT_SIZE, + metadata: { + omniDisclosure: + '原 4096×3072/2MB → 1568×1176/300KB，质量 75，细节与文字锐度受损', + }, + }, + ]); + expect(result.llmContent).toContain('Downsampled image'); + }); + + it('threads tunable overrides into sharp', async () => { + probe({ width: 4000, height: 3000, frameCount: 1 }); + await run({ maxDimension: 800, quality: 50 }); + expect(resize).toHaveBeenCalledWith( + expect.objectContaining({ width: 800, height: 800 }), + ); + expect(jpeg).toHaveBeenCalledWith({ quality: 50 }); + }); + + it('omits original dimensions from the disclosure when the probe lacks them', async () => { + probe({ frameCount: 1 }); + const { result } = await run(); + expect(result.artifacts?.[0]?.metadata?.['omniDisclosure']).toBe( + '原 2MB → 1568×1176/300KB，质量 75，细节与文字锐度受损', + ); + }); + + it('refuses animated images instead of silently keeping one frame', async () => { + probe({ width: 640, height: 480, frameCount: 12 }); + const { result } = await run(); + expect(result.error?.message).toMatch( + /animated image \(12 frames\) is not supported/, + ); + expect(mocks.sharpCreate).not.toHaveBeenCalled(); + expect(result.artifacts).toBeUndefined(); + }); + + it('returns an error result when the input file is missing', async () => { + await fs.rm(inputPath); + const { result } = await run(); + expect(result.error?.message).toMatch(/input file not found/); + }); + + it.each([ + ['relative inputPath', { inputPath: 'rel.png' }], + ['unknown property', { extra: true }], + ['quality out of range', { quality: 150 }], + ])('build rejects %s', (_label, overrides) => { + expect(() => + tool.build({ inputPath, outputDir, ...overrides } as never), + ).toThrow(); + }); + + it('returns an error result when sharp cannot be loaded (D9)', async () => { + vi.resetModules(); + vi.doMock('sharp', () => { + throw new Error("Cannot find module 'sharp'"); + }); + try { + const { OmniDownsampleImageTool: FreshTool } = await import( + './downsample-image.js' + ); + probe({ width: 100, height: 100, frameCount: 1 }); + const invocation = new FreshTool().build({ inputPath, outputDir }); + const result = await invocation.execute(new AbortController().signal); + expect(result.error?.message).toMatch( + /"sharp" image module could not be loaded/, + ); + expect(mocks.sharpCreate).not.toHaveBeenCalled(); + } finally { + vi.doUnmock('sharp'); + vi.resetModules(); + } + }); +}); diff --git a/packages/core/src/omni/policy/tools/downsample-image.ts b/packages/core/src/omni/policy/tools/downsample-image.ts new file mode 100644 index 00000000000..9b6bb6562a1 --- /dev/null +++ b/packages/core/src/omni/policy/tools/downsample-image.ts @@ -0,0 +1,242 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import path from 'node:path'; +import type { + MediaPolicyToolDescriptor, + ToolInvocation, + ToolResult, +} from '../../../tools/tools.js'; +import { BaseToolInvocation, Kind } from '../../../tools/tools.js'; +import { probeMediaMetadata } from '../../ffmpeg.js'; +import { + assertMediaPolicyIo, + BaseMediaPolicyTool, + formatBytesShort, + MEDIA_POLICY_IO_SCHEMA_PROPERTIES, + mediaPolicyToolError, + mediaPolicyToolSuccess, + validateMediaPolicyIoParams, + type MediaPolicyIoParams, +} from './media-policy-tool.js'; + +export const OMNI_DOWNSAMPLE_IMAGE_TOOL_NAME = 'omni_downsample_image'; + +/** Fixed-call default parameters (mapping doc §6). */ +export const DOWNSAMPLE_IMAGE_DEFAULTS = { + maxDimension: 1568, + quality: 75, +} as const; + +const OUTPUT_FILE_NAME = 'downsampled.jpg'; + +export interface DownsampleImageParams extends MediaPolicyIoParams { + /** Longest-edge ceiling in pixels; aspect ratio is preserved. */ + maxDimension?: number; + /** JPEG quality factor of the re-encode (1-100). */ + quality?: number; +} + +const TUNABLE_SCHEMA_PROPERTIES = { + maxDimension: { + type: 'number', + description: + 'Longest-edge ceiling in pixels (aspect ratio preserved). Default 1568.', + minimum: 1, + }, + quality: { + type: 'number', + description: 'JPEG quality factor of the re-encode (1-100). Default 75.', + minimum: 1, + maximum: 100, + }, +} as const; + +const DESCRIPTOR: MediaPolicyToolDescriptor = { + kind: 'media_policy', + inputMediaTypes: ['image'], + outputs: [ + { + kind: 'media', + mimeTypes: ['image/jpeg'], + required: true, + lossy: true, + }, + ], + settingsSchema: { + type: 'object', + properties: TUNABLE_SCHEMA_PROPERTIES, + additionalProperties: false, + }, +}; + +/** Minimal slice of the sharp module the tool uses. */ +type SharpModule = ( + input: string, + options?: object, +) => { + rotate(): SharpPipeline; +}; +interface SharpPipeline { + resize(options: { + width: number; + height: number; + fit: 'inside'; + withoutEnlargement: boolean; + }): SharpPipeline; + jpeg(options: { quality: number }): SharpPipeline; + toFile( + outputPath: string, + ): Promise<{ width: number; height: number; size: number }>; +} + +/** + * Load sharp lazily (decision D9: soft dependency, mirroring the + * image-view.ts convention). A load failure is an EXECUTION failure of + * this invocation — onFailure semantics take over — never a startup gate. + */ +async function loadSharp(): Promise { + // sharp is a CJS `export =` module, so the callable is on `.default` + // at runtime even though NodeNext types collapse that namespace away. + return ((await import('sharp')) as unknown as { default: SharpModule }) + .default; +} + +class DownsampleImageInvocation extends BaseToolInvocation< + DownsampleImageParams, + ToolResult +> { + getDescription(): string { + const maxDimension = + this.params.maxDimension ?? DOWNSAMPLE_IMAGE_DEFAULTS.maxDimension; + return `Downsample ${path.basename(this.params.inputPath)} to fit ${maxDimension}px`; + } + + async execute(signal: AbortSignal): Promise { + const maxDimension = + this.params.maxDimension ?? DOWNSAMPLE_IMAGE_DEFAULTS.maxDimension; + const quality = this.params.quality ?? DOWNSAMPLE_IMAGE_DEFAULTS.quality; + try { + const { inputSizeBytes } = await assertMediaPolicyIo(this.params); + + // Probe BEFORE decoding: the original dimensions feed the disclosure, + // and animated inputs must be refused outright — sharp would silently + // re-encode only the first frame, destroying the animation without + // any disclosure of that loss (decision D9: animated images are + // excluded from the image policy; the guard handles them). + const probe = await probeMediaMetadata( + this.params.inputPath, + 'image', + signal, + ); + if ((probe.frameCount ?? 1) > 1) { + return mediaPolicyToolError( + `animated image (${probe.frameCount} frames) is not supported by ${OMNI_DOWNSAMPLE_IMAGE_TOOL_NAME}`, + ); + } + + let sharp: SharpModule; + try { + sharp = await loadSharp(); + } catch { + return mediaPolicyToolError( + 'the "sharp" image module could not be loaded; image downsampling is unavailable', + ); + } + if (signal.aborted) { + return mediaPolicyToolError('image downsampling aborted'); + } + + const outputPath = path.join(this.params.outputDir, OUTPUT_FILE_NAME); + // `rotate()` bakes in the EXIF orientation so the resized pixels + // match what the user saw; `fit: 'inside'` preserves aspect ratio; + // `withoutEnlargement` keeps already-small originals at native size. + // PNG and other lossless inputs are re-encoded to JPEG too — the + // whole point of the policy is a smaller transport payload. + const info = await sharp(this.params.inputPath, { + failOn: 'error', + limitInputPixels: true, + }) + .rotate() + .resize({ + width: maxDimension, + height: maxDimension, + fit: 'inside', + withoutEnlargement: true, + }) + .jpeg({ quality }) + .toFile(outputPath); + if (signal.aborted) { + return mediaPolicyToolError('image downsampling aborted'); + } + + // Disclosure (decision D8): dimensions/bytes plus the OUTPUT quality + // parameter only — the original's JPEG quality factor is not stored + // in the bitstream, so no claim is made about it. + const original = + probe.width !== undefined && probe.height !== undefined + ? `${probe.width}×${probe.height}/${formatBytesShort(inputSizeBytes)}` + : formatBytesShort(inputSizeBytes); + const disclosure = `原 ${original} → ${info.width}×${info.height}/${formatBytesShort(info.size)}，质量 ${quality}，细节与文字锐度受损`; + + return mediaPolicyToolSuccess({ + outputDir: this.params.outputDir, + outputFileName: OUTPUT_FILE_NAME, + artifactKind: 'image', + title: 'Downsampled image', + mimeType: 'image/jpeg', + sizeBytes: info.size, + disclosure, + }); + } catch (error) { + return mediaPolicyToolError( + error instanceof Error ? error.message : String(error), + ); + } + } +} + +/** + * `omni_downsample_image` — lossy image degradation (sharp): scale to fit + * `maxDimension` and re-encode as JPEG at `quality` (mapping doc §6). + * Registered as a media-policy tool: fixed-policy-only unless modelAccess + * opens it up. + */ +export class OmniDownsampleImageTool extends BaseMediaPolicyTool { + constructor() { + super( + OMNI_DOWNSAMPLE_IMAGE_TOOL_NAME, + 'DownsampleImage', + 'Downsamples an image to fit a maximum dimension and re-encodes it as JPEG, producing a smaller lossy derivative with a disclosure of the degradation.', + Kind.Other, + { + type: 'object', + properties: { + ...MEDIA_POLICY_IO_SCHEMA_PROPERTIES, + ...TUNABLE_SCHEMA_PROPERTIES, + }, + required: ['inputPath', 'outputDir'], + additionalProperties: false, + }, + ); + } + + override get mediaPolicyDescriptor(): MediaPolicyToolDescriptor { + return DESCRIPTOR; + } + + protected override validateToolParamValues( + params: DownsampleImageParams, + ): string | null { + return validateMediaPolicyIoParams(params); + } + + protected createInvocation( + params: DownsampleImageParams, + ): ToolInvocation { + return new DownsampleImageInvocation(params); + } +} diff --git a/packages/core/src/omni/policy/tools/downscale-video.test.ts b/packages/core/src/omni/policy/tools/downscale-video.test.ts new file mode 100644 index 00000000000..31914e844f9 --- /dev/null +++ b/packages/core/src/omni/policy/tools/downscale-video.test.ts @@ -0,0 +1,254 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import type { MediaProbeResult } from '../../ffmpeg.js'; +import type { ToolResult } from '../../../tools/tools.js'; +import { DEFAULT_POLICY_TOOL_TIMEOUT_MS } from './media-policy-tool.js'; +import { + DOWNSCALE_VIDEO_DEFAULTS, + OMNI_DOWNSCALE_VIDEO_TOOL_NAME, + OmniDownscaleVideoTool, +} from './downscale-video.js'; + +const mocks = vi.hoisted(() => ({ + probeMediaMetadata: vi.fn(), + runFfmpeg: vi.fn(), +})); + +vi.mock('../../ffmpeg.js', () => ({ + probeMediaMetadata: mocks.probeMediaMetadata, + runFfmpeg: mocks.runFfmpeg, +})); + +const INPUT_SIZE = 2 * 1024 ** 2; // "2MB" +const OUTPUT_SIZE = 300 * 1024; // "300KB" + +describe('OmniDownscaleVideoTool', () => { + let root: string; + let inputPath: string; + let outputDir: string; + + const tool = new OmniDownscaleVideoTool({}); + + const probe = (result: Partial): void => { + mocks.probeMediaMetadata.mockResolvedValue(result as MediaProbeResult); + }; + + /** ffmpeg success: writes the output file (last arg) and exits 0. */ + const ffmpegSucceeds = (): void => { + mocks.runFfmpeg.mockImplementation(async (args: string[]) => { + await fs.writeFile(args[args.length - 1], Buffer.alloc(OUTPUT_SIZE)); + return { code: 0, stderr: '' }; + }); + }; + + const run = async ( + params: Record = {}, + toolInstance: OmniDownscaleVideoTool = tool, + ): Promise<{ result: ToolResult; signal: AbortSignal }> => { + const invocation = toolInstance.build({ + inputPath, + outputDir, + ...params, + } as never); + const signal = new AbortController().signal; + return { result: await invocation.execute(signal), signal }; + }; + + beforeEach(async () => { + vi.clearAllMocks(); + root = await fs.mkdtemp(path.join(os.tmpdir(), 'omni-vid-')); + inputPath = path.join(root, 'clip.mov'); + await fs.writeFile(inputPath, Buffer.alloc(INPUT_SIZE)); + outputDir = path.join(root, 'staging'); + await fs.mkdir(outputDir); + ffmpegSucceeds(); + }); + + afterEach(async () => { + await fs.rm(root, { recursive: true, force: true }); + }); + + it('declares the media-policy descriptor and defaults', () => { + expect(tool.name).toBe(OMNI_DOWNSCALE_VIDEO_TOOL_NAME); + expect(tool.mediaPolicyDescriptor).toEqual({ + kind: 'media_policy', + inputMediaTypes: ['video'], + outputs: [ + { + kind: 'media', + mimeTypes: ['video/mp4'], + required: true, + lossy: true, + }, + ], + settingsSchema: expect.objectContaining({ type: 'object' }), + }); + expect(DOWNSCALE_VIDEO_DEFAULTS).toEqual({ + maxHeight: 480, + fps: 10, + crf: 28, + preset: 'veryfast', + }); + }); + + it('downscales with the fixed-call defaults, audio stream-copied', async () => { + probe({ height: 1080, frameRate: 30 }); + const { result, signal } = await run(); + + expect(mocks.probeMediaMetadata).toHaveBeenCalledWith( + inputPath, + 'video', + signal, + ); + const outputPath = path.join(outputDir, 'downscaled.mp4'); + expect(mocks.runFfmpeg).toHaveBeenCalledTimes(1); + expect(mocks.runFfmpeg).toHaveBeenCalledWith( + [ + '-y', + '-i', + inputPath, + '-vf', + 'scale=-2:480,fps=10', + '-c:v', + 'libx264', + '-crf', + '28', + '-preset', + 'veryfast', + '-c:a', + 'copy', + outputPath, + ], + { signal, timeoutMs: DEFAULT_POLICY_TOOL_TIMEOUT_MS }, + ); + + expect(result.error).toBeUndefined(); + expect(result.artifacts).toEqual([ + { + kind: 'video', + storage: 'workspace', + title: 'Downscaled video', + workspacePath: 'downscaled.mp4', + mimeType: 'video/mp4', + sizeBytes: OUTPUT_SIZE, + metadata: { + omniDisclosure: + '原 1080p30/2MB → 480p10/300KB，分辨率与帧率下降，细节受损', + }, + }, + ]); + }); + + it('never upscales and rounds the target height down to even', async () => { + probe({ height: 359, frameRate: 24 }); + const { result } = await run(); + const args = mocks.runFfmpeg.mock.calls[0][0] as string[]; + expect(args).toContain('scale=-2:358,fps=10'); + expect(result.artifacts?.[0]?.metadata?.['omniDisclosure']).toBe( + '原 359p24/2MB → 358p10/300KB，分辨率与帧率下降，细节受损', + ); + }); + + it('falls back to AAC 64k when audio stream copy fails', async () => { + probe({ height: 720, frameRate: 25 }); + mocks.runFfmpeg + .mockResolvedValueOnce({ code: 1, stderr: 'pcm in mp4 unsupported' }) + .mockImplementationOnce(async (args: string[]) => { + await fs.writeFile(args[args.length - 1], Buffer.alloc(OUTPUT_SIZE)); + return { code: 0, stderr: '' }; + }); + + const { result } = await run(); + expect(result.error).toBeUndefined(); + expect(mocks.runFfmpeg).toHaveBeenCalledTimes(2); + const firstArgs = mocks.runFfmpeg.mock.calls[0][0] as string[]; + const secondArgs = mocks.runFfmpeg.mock.calls[1][0] as string[]; + expect(firstArgs).toContain('copy'); + expect(secondArgs).not.toContain('copy'); + expect(secondArgs.join(' ')).toContain('-c:a aac -b:a 64k'); + }); + + it('reports the ffmpeg error when both attempts fail', async () => { + probe({ height: 720, frameRate: 25 }); + mocks.runFfmpeg.mockResolvedValue({ + code: 187, + stderr: 'Conversion failed!', + }); + const { result } = await run(); + expect(mocks.runFfmpeg).toHaveBeenCalledTimes(2); + expect(result.error?.message).toMatch(/ffmpeg failed \(exit 187\)/); + expect(result.error?.message).toContain('Conversion failed!'); + expect(result.artifacts).toBeUndefined(); + }); + + it('threads policyTools..runtime.timeoutMs into runFfmpeg', async () => { + probe({ height: 720, frameRate: 25 }); + const configured = new OmniDownscaleVideoTool({ + getOmniPolicyToolsSettings: () => ({ + [OMNI_DOWNSCALE_VIDEO_TOOL_NAME]: { + runtime: { timeoutMs: 120_000 }, + }, + }), + }); + await run({}, configured); + expect(mocks.runFfmpeg).toHaveBeenCalledWith( + expect.any(Array), + expect.objectContaining({ timeoutMs: 120_000 }), + ); + }); + + it('threads tunable overrides into the ffmpeg args', async () => { + probe({ height: 2160, frameRate: 60 }); + await run({ maxHeight: 720, fps: 15, crf: 32, preset: 'fast' }); + const args = mocks.runFfmpeg.mock.calls[0][0] as string[]; + expect(args).toContain('scale=-2:720,fps=15'); + expect(args.join(' ')).toContain('-crf 32 -preset fast'); + }); + + it('reports an aborted run without attempting the audio fallback', async () => { + probe({ height: 720, frameRate: 25 }); + const controller = new AbortController(); + mocks.runFfmpeg.mockImplementation(async () => { + controller.abort(); + return { code: 1, stderr: '' }; + }); + const invocation = tool.build({ inputPath, outputDir }); + const result = await invocation.execute(controller.signal); + expect(result.error?.message).toBe('video downscaling aborted'); + expect(mocks.runFfmpeg).toHaveBeenCalledTimes(1); + }); + + it('errors when the probe cannot determine the video height', async () => { + probe({ frameRate: 25 }); + const { result } = await run(); + expect(result.error?.message).toMatch(/could not determine video height/); + expect(mocks.runFfmpeg).not.toHaveBeenCalled(); + }); + + it('renders an unknown original frame rate as "?"', async () => { + probe({ height: 480 }); + const { result } = await run(); + expect(result.artifacts?.[0]?.metadata?.['omniDisclosure']).toContain( + '原 480p?/', + ); + }); + + it.each([ + ['relative outputDir', { outputDir: 'staging' }], + ['unknown property', { extra: 1 }], + ['invalid preset', { preset: 'warp-speed' }], + ['crf out of range', { crf: 99 }], + ])('build rejects %s', (_label, overrides) => { + expect(() => + tool.build({ inputPath, outputDir, ...overrides } as never), + ).toThrow(); + }); +}); diff --git a/packages/core/src/omni/policy/tools/downscale-video.ts b/packages/core/src/omni/policy/tools/downscale-video.ts new file mode 100644 index 00000000000..0219c8dd49f --- /dev/null +++ b/packages/core/src/omni/policy/tools/downscale-video.ts @@ -0,0 +1,260 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import fs from 'node:fs/promises'; +import path from 'node:path'; +import type { + MediaPolicyToolDescriptor, + ToolInvocation, + ToolResult, +} from '../../../tools/tools.js'; +import { BaseToolInvocation, Kind } from '../../../tools/tools.js'; +import { probeMediaMetadata, runFfmpeg } from '../../ffmpeg.js'; +import { + assertMediaPolicyIo, + BaseMediaPolicyTool, + formatBytesShort, + MEDIA_POLICY_IO_SCHEMA_PROPERTIES, + mediaPolicyToolError, + mediaPolicyToolSuccess, + resolvePolicyToolTimeoutMs, + validateMediaPolicyIoParams, + type MediaPolicyIoParams, + type MediaPolicyToolConfigView, +} from './media-policy-tool.js'; + +export const OMNI_DOWNSCALE_VIDEO_TOOL_NAME = 'omni_downscale_video'; + +/** Fixed-call default parameters (mapping doc §6). */ +export const DOWNSCALE_VIDEO_DEFAULTS = { + maxHeight: 480, + fps: 10, + crf: 28, + preset: 'veryfast', +} as const; + +const OUTPUT_FILE_NAME = 'downscaled.mp4'; + +/** x264 presets accepted by the `preset` tunable. */ +const X264_PRESETS = [ + 'ultrafast', + 'superfast', + 'veryfast', + 'faster', + 'fast', + 'medium', + 'slow', + 'slower', + 'veryslow', +] as const; + +export interface DownscaleVideoParams extends MediaPolicyIoParams { + /** Output height ceiling in pixels (width follows aspect ratio). */ + maxHeight?: number; + /** Output frame rate. */ + fps?: number; + /** x264 constant rate factor (higher = smaller/lossier). */ + crf?: number; + /** x264 encoding preset. */ + preset?: string; +} + +const TUNABLE_SCHEMA_PROPERTIES = { + maxHeight: { + type: 'number', + description: + 'Output height ceiling in pixels (width follows aspect ratio). Default 480.', + minimum: 2, + }, + fps: { + type: 'number', + description: 'Output frame rate. Default 10.', + minimum: 1, + }, + crf: { + type: 'number', + description: + 'x264 constant rate factor, 0-51 (higher = smaller/lossier). Default 28.', + minimum: 0, + maximum: 51, + }, + preset: { + type: 'string', + description: 'x264 encoding preset. Default "veryfast".', + enum: [...X264_PRESETS], + }, +} as const; + +const DESCRIPTOR: MediaPolicyToolDescriptor = { + kind: 'media_policy', + inputMediaTypes: ['video'], + outputs: [ + { + kind: 'media', + mimeTypes: ['video/mp4'], + required: true, + lossy: true, + }, + ], + settingsSchema: { + type: 'object', + properties: TUNABLE_SCHEMA_PROPERTIES, + additionalProperties: false, + }, +}; + +class DownscaleVideoInvocation extends BaseToolInvocation< + DownscaleVideoParams, + ToolResult +> { + constructor( + params: DownscaleVideoParams, + private readonly timeoutMs: number, + ) { + super(params); + } + + getDescription(): string { + const maxHeight = + this.params.maxHeight ?? DOWNSCALE_VIDEO_DEFAULTS.maxHeight; + return `Downscale ${path.basename(this.params.inputPath)} to ${maxHeight}p`; + } + + async execute(signal: AbortSignal): Promise { + const maxHeight = + this.params.maxHeight ?? DOWNSCALE_VIDEO_DEFAULTS.maxHeight; + const fps = this.params.fps ?? DOWNSCALE_VIDEO_DEFAULTS.fps; + const crf = this.params.crf ?? DOWNSCALE_VIDEO_DEFAULTS.crf; + const preset = this.params.preset ?? DOWNSCALE_VIDEO_DEFAULTS.preset; + try { + const { inputSizeBytes } = await assertMediaPolicyIo(this.params); + const probe = await probeMediaMetadata( + this.params.inputPath, + 'video', + signal, + ); + if (probe.height === undefined) { + return mediaPolicyToolError( + `could not determine video height of ${path.basename(this.params.inputPath)}`, + ); + } + + // Target height computed in JS from the probe (not an ffmpeg scale + // expression — expression commas need filtergraph escaping and are + // easy to get subtly wrong): never upscale, and round down to even + // because libx264 requires even dimensions. `scale=-2:h` rounds the + // width to even automatically. + const targetHeight = Math.max( + 2, + Math.floor(Math.min(maxHeight, probe.height) / 2) * 2, + ); + const outputPath = path.join(this.params.outputDir, OUTPUT_FILE_NAME); + const argsFor = (audio: string[]): string[] => [ + '-y', + '-i', + this.params.inputPath, + '-vf', + `scale=-2:${targetHeight},fps=${fps}`, + '-c:v', + 'libx264', + '-crf', + String(crf), + '-preset', + preset, + ...audio, + outputPath, + ]; + + // Audio: try stream copy first (free); if the source codec cannot be + // muxed into mp4 (e.g. pcm, vorbis) ffmpeg fails fast, and the + // fallback re-encodes to AAC 64k (mapping doc §6: copy→aac 兜底). + let run = await runFfmpeg(argsFor(['-c:a', 'copy']), { + signal, + timeoutMs: this.timeoutMs, + }); + if (signal.aborted) { + return mediaPolicyToolError('video downscaling aborted'); + } + if (run.code !== 0) { + run = await runFfmpeg(argsFor(['-c:a', 'aac', '-b:a', '64k']), { + signal, + timeoutMs: this.timeoutMs, + }); + if (signal.aborted) { + return mediaPolicyToolError('video downscaling aborted'); + } + if (run.code !== 0) { + return mediaPolicyToolError( + `ffmpeg failed (exit ${run.code}) downscaling ${path.basename(this.params.inputPath)}: ${run.stderr.slice(-500)}`, + ); + } + } + + const outputSizeBytes = (await fs.stat(outputPath)).size; + const originalRate = + probe.frameRate !== undefined ? Math.round(probe.frameRate) : '?'; + const disclosure = `原 ${probe.height}p${originalRate}/${formatBytesShort(inputSizeBytes)} → ${targetHeight}p${fps}/${formatBytesShort(outputSizeBytes)}，分辨率与帧率下降，细节受损`; + + return mediaPolicyToolSuccess({ + outputDir: this.params.outputDir, + outputFileName: OUTPUT_FILE_NAME, + artifactKind: 'video', + title: 'Downscaled video', + mimeType: 'video/mp4', + sizeBytes: outputSizeBytes, + disclosure, + }); + } catch (error) { + return mediaPolicyToolError( + error instanceof Error ? error.message : String(error), + ); + } + } +} + +/** + * `omni_downscale_video` — lossy video degradation (ffmpeg): scale to a + * height ceiling, drop the frame rate, re-encode with x264 at a fixed CRF; + * audio is stream-copied with an AAC 64k fallback (mapping doc §6). + */ +export class OmniDownscaleVideoTool extends BaseMediaPolicyTool { + constructor(private readonly config: MediaPolicyToolConfigView) { + super( + OMNI_DOWNSCALE_VIDEO_TOOL_NAME, + 'DownscaleVideo', + 'Downscales a video to a maximum height and frame rate and re-encodes it, producing a smaller lossy derivative with a disclosure of the degradation.', + Kind.Other, + { + type: 'object', + properties: { + ...MEDIA_POLICY_IO_SCHEMA_PROPERTIES, + ...TUNABLE_SCHEMA_PROPERTIES, + }, + required: ['inputPath', 'outputDir'], + additionalProperties: false, + }, + ); + } + + override get mediaPolicyDescriptor(): MediaPolicyToolDescriptor { + return DESCRIPTOR; + } + + protected override validateToolParamValues( + params: DownscaleVideoParams, + ): string | null { + return validateMediaPolicyIoParams(params); + } + + protected createInvocation( + params: DownscaleVideoParams, + ): ToolInvocation { + return new DownscaleVideoInvocation( + params, + resolvePolicyToolTimeoutMs(this.config, this.name), + ); + } +} diff --git a/packages/core/src/omni/policy/tools/media-policy-tool.test.ts b/packages/core/src/omni/policy/tools/media-policy-tool.test.ts new file mode 100644 index 00000000000..17f2385486e --- /dev/null +++ b/packages/core/src/omni/policy/tools/media-policy-tool.test.ts @@ -0,0 +1,237 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import type { MediaPolicyToolDescriptor } from '../../../tools/tools.js'; +import { Kind, type ToolResult } from '../../../tools/tools.js'; +import { + assertMediaPolicyIo, + BaseMediaPolicyTool, + DEFAULT_POLICY_TOOL_TIMEOUT_MS, + formatBytesShort, + resolvePolicyToolTimeoutMs, + validateMediaPolicyIoParams, +} from './media-policy-tool.js'; +import { BaseToolInvocation } from '../../../tools/tools.js'; + +describe('formatBytesShort', () => { + it.each([ + [512, '512B'], + [8_200_000, '7.8MB'], + [943_718, '921.6KB'], + [2 * 1024 ** 3, '2GB'], + [180 * 1024 ** 2, '180MB'], + [1024, '1KB'], + ])('%d → %s', (bytes, expected) => { + expect(formatBytesShort(bytes)).toBe(expected); + }); +}); + +describe('resolvePolicyToolTimeoutMs', () => { + it('defaults to 600s when unset', () => { + expect(resolvePolicyToolTimeoutMs({}, 'omni_downscale_video')).toBe( + DEFAULT_POLICY_TOOL_TIMEOUT_MS, + ); + expect(DEFAULT_POLICY_TOOL_TIMEOUT_MS).toBe(600_000); + }); + + it('reads policyTools..runtime.timeoutMs', () => { + const config = { + getOmniPolicyToolsSettings: () => ({ + omni_downscale_video: { runtime: { timeoutMs: 120_000 } }, + }), + }; + expect(resolvePolicyToolTimeoutMs(config, 'omni_downscale_video')).toBe( + 120_000, + ); + }); + + it.each([ + ['tombstone entry', null], + ['malformed runtime', { runtime: 'fast' }], + ['non-numeric timeout', { runtime: { timeoutMs: 'soon' } }], + ['non-positive timeout', { runtime: { timeoutMs: 0 } }], + ['non-finite timeout', { runtime: { timeoutMs: Infinity } }], + ])('falls back to the default on %s', (_label, entry) => { + const config = { + getOmniPolicyToolsSettings: () => ({ + omni_downscale_video: entry as never, + }), + }; + expect(resolvePolicyToolTimeoutMs(config, 'omni_downscale_video')).toBe( + DEFAULT_POLICY_TOOL_TIMEOUT_MS, + ); + }); +}); + +describe('validateMediaPolicyIoParams', () => { + it('accepts absolute paths', () => { + expect( + validateMediaPolicyIoParams({ + inputPath: '/a/in.mp4', + outputDir: '/b/staging', + }), + ).toBeNull(); + }); + + it.each([ + ['relative inputPath', 'in.mp4', '/b', /inputPath must be an absolute/], + ['relative outputDir', '/a/in.mp4', 'out', /outputDir must be an absolute/], + ])('rejects %s', (_label, inputPath, outputDir, pattern) => { + expect(validateMediaPolicyIoParams({ inputPath, outputDir })).toMatch( + pattern, + ); + }); +}); + +describe('assertMediaPolicyIo', () => { + let root: string; + + beforeEach(async () => { + root = await fs.mkdtemp(path.join(os.tmpdir(), 'omni-mp-io-')); + }); + + afterEach(async () => { + await fs.rm(root, { recursive: true, force: true }); + }); + + it('returns the input size for a valid pair', async () => { + const inputPath = path.join(root, 'in.bin'); + await fs.writeFile(inputPath, Buffer.alloc(1234)); + const outputDir = path.join(root, 'staging'); + await fs.mkdir(outputDir); + await expect( + assertMediaPolicyIo({ inputPath, outputDir }), + ).resolves.toEqual({ inputSizeBytes: 1234 }); + }); + + it('rejects a missing input file', async () => { + const outputDir = path.join(root, 'staging'); + await fs.mkdir(outputDir); + await expect( + assertMediaPolicyIo({ inputPath: path.join(root, 'nope'), outputDir }), + ).rejects.toThrow(/input file not found/); + }); + + it('rejects a symlinked input (never reads through a link)', async () => { + const real = path.join(root, 'real.bin'); + await fs.writeFile(real, 'x'); + const link = path.join(root, 'link.bin'); + await fs.symlink(real, link); + const outputDir = path.join(root, 'staging'); + await fs.mkdir(outputDir); + await expect( + assertMediaPolicyIo({ inputPath: link, outputDir }), + ).rejects.toThrow(/not a regular file/); + }); + + it('rejects a missing output directory', async () => { + const inputPath = path.join(root, 'in.bin'); + await fs.writeFile(inputPath, 'x'); + await expect( + assertMediaPolicyIo({ inputPath, outputDir: path.join(root, 'nope') }), + ).rejects.toThrow(/output directory not found/); + }); + + it('rejects a symlinked output directory', async () => { + const inputPath = path.join(root, 'in.bin'); + await fs.writeFile(inputPath, 'x'); + const realDir = path.join(root, 'real-dir'); + await fs.mkdir(realDir); + const linkDir = path.join(root, 'link-dir'); + await fs.symlink(realDir, linkDir); + await expect( + assertMediaPolicyIo({ inputPath, outputDir: linkDir }), + ).rejects.toThrow(/not a real directory/); + }); +}); + +describe('BaseMediaPolicyTool validation', () => { + interface TestParams { + inputPath: string; + outputDir: string; + level?: number; + } + + class NoopInvocation extends BaseToolInvocation { + getDescription(): string { + return 'noop'; + } + async execute(): Promise { + return { llmContent: 'ok', returnDisplay: 'ok' }; + } + } + + class TestPolicyTool extends BaseMediaPolicyTool { + constructor() { + super('test_policy_tool', 'TestPolicyTool', 'test', Kind.Other, { + type: 'object', + properties: { + inputPath: { type: 'string' }, + outputDir: { type: 'string' }, + level: { type: 'number', minimum: 1 }, + }, + required: ['inputPath', 'outputDir'], + additionalProperties: false, + }); + } + override get mediaPolicyDescriptor(): MediaPolicyToolDescriptor { + return { + kind: 'media_policy', + inputMediaTypes: ['image'], + outputs: [{ kind: 'media', required: true, lossy: true }], + }; + } + protected override validateToolParamValues( + params: TestParams, + ): string | null { + return validateMediaPolicyIoParams(params); + } + protected createInvocation(params: TestParams): NoopInvocation { + return new NoopInvocation(params); + } + } + + const tool = new TestPolicyTool(); + + it('validates against the NATIVE parameter schema', () => { + expect( + tool.validateToolParams({ + inputPath: '/a/in.png', + outputDir: '/b/staging', + level: 3, + }), + ).toBeNull(); + }); + + it('rejects schema violations (unknown property, missing required)', () => { + expect( + tool.validateToolParams({ + inputPath: '/a/in.png', + outputDir: '/b/staging', + extra: true, + } as never), + ).not.toBeNull(); + expect( + tool.validateToolParams({ inputPath: '/a/in.png' } as never), + ).not.toBeNull(); + }); + + it('runs value validation after schema validation', () => { + expect( + tool.validateToolParams({ inputPath: 'rel.png', outputDir: '/b' }), + ).toMatch(/absolute/); + }); + + it('build throws on invalid params', () => { + expect(() => tool.build({ inputPath: 'rel.png', outputDir: '/b' })).toThrow( + /absolute/, + ); + }); +}); diff --git a/packages/core/src/omni/policy/tools/media-policy-tool.ts b/packages/core/src/omni/policy/tools/media-policy-tool.ts new file mode 100644 index 00000000000..c6d3b6e9cfe --- /dev/null +++ b/packages/core/src/omni/policy/tools/media-policy-tool.ts @@ -0,0 +1,201 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import fs from 'node:fs/promises'; +import path from 'node:path'; +import type { + MediaPolicyToolDescriptor, + ToolArtifact, + ToolArtifactKind, + ToolResult, +} from '../../../tools/tools.js'; +import { BaseDeclarativeTool } from '../../../tools/tools.js'; +import { ToolErrorType } from '../../../tools/tool-error.js'; +import { SchemaValidator } from '../../../utils/schemaValidator.js'; +import type { OmniPolicyToolsSettings } from '../types.js'; + +/** Default transcode timeout when `policyTools..runtime.timeoutMs` + * is not configured (mapping doc §6). */ +export const DEFAULT_POLICY_TOOL_TIMEOUT_MS = 600_000; + +/** Parameters every media-policy degradation tool shares: one input file, + * one harness-injected output directory (the invocation's staging dir — + * the tool's ONLY permitted output location). */ +export interface MediaPolicyIoParams { + /** Absolute path of the source media file. */ + inputPath: string; + /** Absolute path of the directory the tool must write into. */ + outputDir: string; +} + +/** JSON-schema fragments for the shared io parameters. */ +export const MEDIA_POLICY_IO_SCHEMA_PROPERTIES = { + inputPath: { + type: 'string', + description: 'Absolute path of the source media file.', + }, + outputDir: { + type: 'string', + description: + 'Absolute path of the directory the output file is written into.', + }, +} as const; + +/** Minimal structural view of Config used by policy tools. Optional so + * partial/stub configs (tests, embedders) fall back to defaults. */ +export interface MediaPolicyToolConfigView { + getOmniPolicyToolsSettings?: () => OmniPolicyToolsSettings | undefined; +} + +const isPlainRecord = (value: unknown): value is Record => + typeof value === 'object' && value !== null && !Array.isArray(value); + +/** Read `omni.processing.policyTools..runtime.timeoutMs` + * leniently; anything absent or malformed resolves to the default. */ +export function resolvePolicyToolTimeoutMs( + config: MediaPolicyToolConfigView, + toolName: string, +): number { + const entry = config.getOmniPolicyToolsSettings?.()?.[toolName]; + const runtime = + isPlainRecord(entry) && isPlainRecord(entry['runtime']) + ? entry['runtime'] + : undefined; + const timeoutMs = runtime?.['timeoutMs']; + return typeof timeoutMs === 'number' && + Number.isFinite(timeoutMs) && + timeoutMs > 0 + ? timeoutMs + : DEFAULT_POLICY_TOOL_TIMEOUT_MS; +} + +/** + * Base class for omni media-policy tools (real DeclarativeTools — the + * orchestrator executes them through the ordinary scheduler path, and + * Stage B's modelAccess can open them to the model). + * + * `mediaPolicyDescriptor` is abstract: every subclass MUST declare its + * descriptor — that code-level fact is what the modelAccess gate and the + * orchestrator key off. + */ +export abstract class BaseMediaPolicyTool< + TParams extends object, +> extends BaseDeclarativeTool { + abstract override get mediaPolicyDescriptor(): MediaPolicyToolDescriptor; + + /** + * Validate against the tool's NATIVE parameter schema, never the + * model-visible `schema` getter: Stage B's modelAccess projection makes + * `schema` a narrowed view (lockedArguments removed), while validation + * must keep accepting the harness-injected arguments the projection + * hides (policy design §9.4). + */ + override validateToolParams(params: TParams): string | null { + const errors = SchemaValidator.validate(this.parameterSchema, params); + if (errors) { + return errors; + } + return this.validateToolParamValues(params); + } +} + +/** Shared structural validation for the io params (schema has already + * checked types/required-ness). Returns an error message or null. */ +export function validateMediaPolicyIoParams( + params: MediaPolicyIoParams, +): string | null { + if (!path.isAbsolute(params.inputPath)) { + return `inputPath must be an absolute path (got ${JSON.stringify(params.inputPath)})`; + } + if (!path.isAbsolute(params.outputDir)) { + return `outputDir must be an absolute path (got ${JSON.stringify(params.outputDir)})`; + } + return null; +} + +/** + * Execution-time io checks (validateToolParams is synchronous, so + * filesystem state is asserted here): the input must be an existing + * REGULAR file (lstat — a symlink is refused, the tool must never read + * through a link planted in its input position) and the output directory + * an existing real directory. Returns the input size in bytes. + */ +export async function assertMediaPolicyIo( + params: MediaPolicyIoParams, +): Promise<{ inputSizeBytes: number }> { + let inputStat; + try { + inputStat = await fs.lstat(params.inputPath); + } catch { + throw new Error(`input file not found: ${params.inputPath}`); + } + if (!inputStat.isFile()) { + throw new Error(`input is not a regular file: ${params.inputPath}`); + } + let outStat; + try { + outStat = await fs.lstat(params.outputDir); + } catch { + throw new Error(`output directory not found: ${params.outputDir}`); + } + if (!outStat.isDirectory()) { + throw new Error(`output path is not a real directory: ${params.outputDir}`); + } + return { inputSizeBytes: inputStat.size }; +} + +/** Compact human-readable byte count for disclosure texts ("8.2MB", + * "0.9MB", "2GB", "180MB", "512KB"). */ +export function formatBytesShort(bytes: number): string { + const trim = (n: number): string => + (Math.round(n * 10) / 10).toString().replace(/\.0$/, ''); + if (bytes >= 1024 ** 3) return `${trim(bytes / 1024 ** 3)}GB`; + if (bytes >= 1024 ** 2) return `${trim(bytes / 1024 ** 2)}MB`; + if (bytes >= 1024) return `${trim(bytes / 1024)}KB`; + return `${bytes}B`; +} + +/** Uniform error ToolResult for a failed policy-tool execution. */ +export function mediaPolicyToolError(message: string): ToolResult { + return { + llmContent: `Error: ${message}`, + returnDisplay: message, + error: { message, type: ToolErrorType.EXECUTION_FAILED }, + }; +} + +/** + * Successful policy-tool result: a one-line summary for the model-facing + * channel and exactly one lossy media artifact whose + * `metadata.omniDisclosure` carries the disclosure text the orchestrator + * validates and delivers adjacent to the media (decision D8). + */ +export function mediaPolicyToolSuccess(args: { + outputDir: string; + outputFileName: string; + artifactKind: ToolArtifactKind; + title: string; + mimeType: string; + sizeBytes: number; + disclosure: string; +}): ToolResult { + const artifact: ToolArtifact = { + kind: args.artifactKind, + storage: 'workspace', + title: args.title, + // Relative to the invocation's staging directory — the orchestrator + // resolves and re-validates containment before promotion. + workspacePath: args.outputFileName, + mimeType: args.mimeType, + sizeBytes: args.sizeBytes, + metadata: { omniDisclosure: args.disclosure }, + }; + return { + llmContent: `${args.title}: ${args.disclosure}`, + returnDisplay: args.disclosure, + artifacts: [artifact], + }; +} diff --git a/packages/core/src/tools/tool-names.ts b/packages/core/src/tools/tool-names.ts index b6478ca4977..d9c56a25197 100644 --- a/packages/core/src/tools/tool-names.ts +++ b/packages/core/src/tools/tool-names.ts @@ -69,6 +69,11 @@ export const ToolNames = { RECORD_ARTIFACT: 'record_artifact', GET_GOAL: 'get_goal', UPDATE_GOAL: 'update_goal', + // Omni media-policy tools (fixed-policy-only by default; modelAccess + // config can open them to the model). + OMNI_DOWNSAMPLE_IMAGE: 'omni_downsample_image', + OMNI_DOWNSCALE_VIDEO: 'omni_downscale_video', + OMNI_DOWNSAMPLE_AUDIO: 'omni_downsample_audio', } as const; /** @@ -123,6 +128,9 @@ export const ToolDisplayNames = { RECORD_ARTIFACT: 'RecordArtifact', GET_GOAL: 'Goal', UPDATE_GOAL: 'UpdateGoal', + OMNI_DOWNSAMPLE_IMAGE: 'DownsampleImage', + OMNI_DOWNSCALE_VIDEO: 'DownscaleVideo', + OMNI_DOWNSAMPLE_AUDIO: 'DownsampleAudio', } as const; // Migration from old tool names to new tool names From e2cc1a436ac55d2eeb4144f16fd0ad6f8e2a674e Mon Sep 17 00:00:00 2001 From: LaZzyMan Date: Fri, 7 Aug 2026 10:18:09 +0800 Subject: [PATCH 07/62] feat(omni): add degradation result cache keyed by policy fingerprint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Commit 7 of the S4 policy pipeline (decision D2): reusing a previous transcode instead of re-paying it — a 424MB video downscale is minutes-long and must not run once per delivery round. - omni/json-cache-file.ts: the file mechanics extracted verbatim from upload-cache (per-file serialized load-modify-save, atomic tmp+rename 0600 writes, corrupt backup+rebuild with capped .corrupt-* backups, unreadable-but-existing file = operation no-op) as a shared OmniJsonCacheFile — the design doc mandates the degradation cache mirror these exact semantics, so they now exist once. - omni/upload-cache.ts: refactored onto the shared file; entry semantics (model/scope keys, TTL clamp, invalidation) unchanged — all 22 existing tests pass untouched. - omni/policy/degradation-cache.ts: (originalSha256, policyFingerprint) → { degradedSha256, extension, disclosure, mimeType } at .qwen/omni/policy-cache.json. policyFingerprint = sha256(toolName + key-sorted tunables + tool version); per-invocation io params (inputPath/outputDir) are excluded — they are plumbing, not policy identity. Entries carry no TTL (content-addressed identities never go stale); removeByOriginalSha256 / removeByDegradedSha256 serve the GC/corruption cascades. Object existence checks stay with the orchestrator. --- packages/core/src/omni/json-cache-file.ts | 179 +++++++++++++++ .../src/omni/policy/degradation-cache.test.ts | 200 +++++++++++++++++ .../core/src/omni/policy/degradation-cache.ts | 157 +++++++++++++ packages/core/src/omni/upload-cache.ts | 212 +++--------------- 4 files changed, 573 insertions(+), 175 deletions(-) create mode 100644 packages/core/src/omni/json-cache-file.ts create mode 100644 packages/core/src/omni/policy/degradation-cache.test.ts create mode 100644 packages/core/src/omni/policy/degradation-cache.ts diff --git a/packages/core/src/omni/json-cache-file.ts b/packages/core/src/omni/json-cache-file.ts new file mode 100644 index 00000000000..154da31f9ff --- /dev/null +++ b/packages/core/src/omni/json-cache-file.ts @@ -0,0 +1,179 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { randomBytes } from 'node:crypto'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { createDebugLogger, type DebugLogger } from '../utils/debugLogger.js'; + +/** Per-cache-file operation serializer: cache instances are constructed + * per use site, and safe-tool batches run deliveries concurrently in one + * process — unserialized load-modify-save would drop entries. Module + * scope is deliberate: two instances on the same file must share the + * chain. Cross-process writes remain last-writer-wins (documented). */ +const fileOps = new Map>(); + +function serialize(key: string, fn: () => Promise): Promise { + const prev = fileOps.get(key) ?? Promise.resolve(); + const run = prev.then(fn, fn); + const settled = run.then( + () => {}, + () => {}, + ); + fileOps.set(key, settled); + void settled.then(() => { + // Drop the tail once it settles — otherwise the map grows with every + // distinct cache file touched over the process lifetime. Only delete + // when OUR promise is still the tail: a later op may have chained on. + if (fileOps.get(key) === settled) fileOps.delete(key); + }); + return run; +} + +/** Keep at most this many `.corrupt-*` backups (newest wins): a crash + * loop over a corrupt file must not litter the directory without bound. */ +const MAX_CORRUPT_BACKUPS = 2; + +interface CacheFileShape { + version: 1; + entries: Record; +} + +/** + * Shared mechanics for omni's persistent JSON entry caches + * (`upload-cache.json`, `policy-cache.json`): one flat + * `{ version: 1, entries: {} }` file with + * + * - per-file serialized load-modify-save (in-process), + * - atomic writes (tmp + rename, 0600 file / 0700 dir), + * - corrupt files backed up as `.corrupt-` (newest + * {@link MAX_CORRUPT_BACKUPS} kept) and rebuilt empty — never fatal, + * - unreadable-but-existing files (EACCES, EMFILE, …) making the current + * operation a no-op instead of an empty rebuild: a transient read + * failure must never lead to a save that wipes every persisted entry. + * + * Entry semantics (keys, TTLs, invalidation) stay in the owning cache. + */ +export class OmniJsonCacheFile { + private readonly debugLogger: DebugLogger; + + constructor( + readonly filePath: string, + debugChannel: string, + ) { + this.debugLogger = createDebugLogger(debugChannel); + } + + /** + * Run one serialized operation against the entry map. `fn` returns the + * operation result plus whether it changed the map (triggering an + * atomic save). When the file exists but cannot be read, + * `unreadableResult` is returned and nothing is saved. + */ + async access( + unreadableResult: R, + fn: ( + entries: Record, + ) => + | { result: R; changed?: boolean } + | Promise<{ result: R; changed?: boolean }>, + ): Promise { + return serialize(this.filePath, async () => { + const data = await this.load(); + if (!data) return unreadableResult; + const { result, changed } = await fn(data.entries); + if (changed) await this.save(data); + return result; + }); + } + + /** + * Load the cache file. Returns null when the file exists but could not + * be read (EACCES, EMFILE, …): the caller must skip its operation for + * this call — proceeding with an empty snapshot and later saving it + * would overwrite N valid entries with one (self-inflicted cache wipe). + * Only a genuinely missing file means empty-and-writable. + */ + private async load(): Promise | null> { + let raw: string; + try { + raw = await fs.readFile(this.filePath, 'utf8'); + } catch (err) { + const code = (err as NodeJS.ErrnoException).code; + // ENOENT: no cache file yet (POSIX and Windows). ENOTDIR: a parent + // path component is a plain file — POSIX raises ENOTDIR where + // Windows reports ENOENT for the same condition. + if (code === 'ENOENT' || code === 'ENOTDIR') { + return { version: 1, entries: {} }; + } + this.debugLogger.debug( + `cache read failed, operation skipped: ${err instanceof Error ? err.message : err}`, + ); + return null; + } + try { + const parsed = JSON.parse(raw) as CacheFileShape; + // `entries` must be a plain non-null object: `typeof null` and + // `typeof []` are both 'object', and either shape would throw raw + // TypeErrors from every accessor (escaping the never-fatal contract + // and skipping backup+rebuild). + if ( + parsed?.version === 1 && + typeof parsed.entries === 'object' && + parsed.entries !== null && + !Array.isArray(parsed.entries) + ) { + return parsed; + } + throw new Error('unexpected shape'); + } catch { + // Corrupt cache: preserve for inspection, start fresh. Losing a + // cache only costs re-work — never fail the pipeline over it. + const backup = `${this.filePath}.corrupt-${Date.now()}`; + await fs.rename(this.filePath, backup).catch(() => {}); + await this.pruneCorruptBackups(); + this.debugLogger.debug(`corrupt cache backed up to ${backup}`); + return { version: 1, entries: {} }; + } + } + + /** Best-effort: keep only the newest {@link MAX_CORRUPT_BACKUPS}. */ + private async pruneCorruptBackups(): Promise { + const dir = path.dirname(this.filePath); + const prefix = `${path.basename(this.filePath)}.corrupt-`; + try { + const backups = (await fs.readdir(dir)) + .filter((n) => n.startsWith(prefix)) + // Millisecond timestamps are fixed-width for centuries, so the + // lexicographic sort is chronological; newest first. + .sort() + .reverse(); + for (const name of backups.slice(MAX_CORRUPT_BACKUPS)) { + await fs.rm(path.join(dir, name), { force: true }).catch(() => {}); + } + } catch { + // Pruning is hygiene; never let it affect the read path. + } + } + + private async save(data: CacheFileShape): Promise { + const tmp = `${this.filePath}.tmp-${randomBytes(4).toString('hex')}`; + try { + await fs.mkdir(path.dirname(this.filePath), { + recursive: true, + mode: 0o700, + }); + await fs.writeFile(tmp, JSON.stringify(data, null, 1), { mode: 0o600 }); + await fs.rename(tmp, this.filePath); + } catch (err) { + await fs.rm(tmp, { force: true }).catch(() => {}); + // Cache persistence is best-effort by design. + this.debugLogger.debug( + `cache write failed: ${err instanceof Error ? err.message : err}`, + ); + } + } +} diff --git a/packages/core/src/omni/policy/degradation-cache.test.ts b/packages/core/src/omni/policy/degradation-cache.test.ts new file mode 100644 index 00000000000..3021d8bd144 --- /dev/null +++ b/packages/core/src/omni/policy/degradation-cache.test.ts @@ -0,0 +1,200 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { + computePolicyFingerprint, + OmniDegradationCache, +} from './degradation-cache.js'; + +const ORIGINAL = 'a'.repeat(64); +const DEGRADED = 'b'.repeat(64); + +const ENTRY = { + degradedSha256: DEGRADED, + extension: '.jpg', + disclosure: + '原 4096×3072/8.2MB → 1568×1176/0.9MB，质量 75，细节与文字锐度受损', + mimeType: 'image/jpeg', +}; + +describe('computePolicyFingerprint', () => { + it('is stable across key order and identical inputs', () => { + const a = computePolicyFingerprint('omni_downsample_image', { + maxDimension: 1568, + quality: 75, + }); + const b = computePolicyFingerprint('omni_downsample_image', { + quality: 75, + maxDimension: 1568, + }); + expect(a).toBe(b); + expect(a).toMatch(/^[0-9a-f]{64}$/); + }); + + it('ignores the per-invocation io params (inputPath/outputDir)', () => { + const bare = computePolicyFingerprint('omni_downsample_image', { + quality: 75, + }); + const withIo = computePolicyFingerprint('omni_downsample_image', { + quality: 75, + inputPath: '/tmp/a/in.png', + outputDir: '/tmp/staging/deadbeef', + }); + expect(withIo).toBe(bare); + }); + + it('ignores undefined values (absent tunable == undefined tunable)', () => { + expect( + computePolicyFingerprint('t', { quality: 75, maxDimension: undefined }), + ).toBe(computePolicyFingerprint('t', { quality: 75 })); + }); + + it.each([ + ['tool name', ['other_tool', { quality: 75 }, undefined]], + ['argument value', ['t', { quality: 80 }, undefined]], + ['argument set', ['t', { quality: 75, maxDimension: 800 }, undefined]], + ['tool version', ['t', { quality: 75 }, '2']], + ] as Array<[string, [string, Record, string | undefined]]>)( + 'changes when the %s changes', + (_label, [tool, args, version]) => { + const base = computePolicyFingerprint('t', { quality: 75 }); + expect(computePolicyFingerprint(tool, args, version)).not.toBe(base); + }, + ); + + it('sorts keys recursively in nested arguments', () => { + expect( + computePolicyFingerprint('t', { opts: { b: 2, a: [1, { d: 4, c: 3 }] } }), + ).toBe( + computePolicyFingerprint('t', { opts: { a: [1, { c: 3, d: 4 }], b: 2 } }), + ); + }); +}); + +describe('OmniDegradationCache', () => { + let root: string; + let cache: OmniDegradationCache; + const fp = computePolicyFingerprint('omni_downsample_image', { + maxDimension: 1568, + quality: 75, + }); + + beforeEach(async () => { + root = await fs.mkdtemp(path.join(os.tmpdir(), 'omni-degcache-')); + cache = new OmniDegradationCache(root); + }); + + afterEach(async () => { + await fs.rm(root, { recursive: true, force: true }); + }); + + it('round-trips an entry and persists across instances', async () => { + await expect(cache.get(ORIGINAL, fp)).resolves.toBeNull(); + await cache.put(ORIGINAL, fp, ENTRY); + const hit = await cache.get(ORIGINAL, fp); + expect(hit).toMatchObject(ENTRY); + expect(Date.parse(hit!.createdAt)).not.toBeNaN(); + + const second = new OmniDegradationCache(root); + await expect(second.get(ORIGINAL, fp)).resolves.toMatchObject(ENTRY); + }); + + it('writes to policy-cache.json under the omni root', async () => { + await cache.put(ORIGINAL, fp, ENTRY); + const raw = JSON.parse( + await fs.readFile(path.join(root, 'policy-cache.json'), 'utf8'), + ); + expect(raw.version).toBe(1); + expect(Object.keys(raw.entries)).toEqual([`${ORIGINAL}|${fp}`]); + }); + + it('misses on a different fingerprint or original hash', async () => { + await cache.put(ORIGINAL, fp, ENTRY); + const otherFp = computePolicyFingerprint('omni_downsample_image', { + maxDimension: 800, + }); + await expect(cache.get(ORIGINAL, otherFp)).resolves.toBeNull(); + await expect(cache.get('c'.repeat(64), fp)).resolves.toBeNull(); + }); + + it('re-put for the same key replaces the entry', async () => { + await cache.put(ORIGINAL, fp, ENTRY); + await cache.put(ORIGINAL, fp, { + ...ENTRY, + degradedSha256: 'd'.repeat(64), + }); + await expect(cache.get(ORIGINAL, fp)).resolves.toMatchObject({ + degradedSha256: 'd'.repeat(64), + }); + }); + + it('removeByOriginalSha256 drops every policy result for the source', async () => { + const fp2 = computePolicyFingerprint('omni_downsample_image', { + quality: 50, + }); + await cache.put(ORIGINAL, fp, ENTRY); + await cache.put(ORIGINAL, fp2, ENTRY); + await cache.put('c'.repeat(64), fp, ENTRY); + + await cache.removeByOriginalSha256(ORIGINAL); + await expect(cache.get(ORIGINAL, fp)).resolves.toBeNull(); + await expect(cache.get(ORIGINAL, fp2)).resolves.toBeNull(); + await expect(cache.get('c'.repeat(64), fp)).resolves.not.toBeNull(); + }); + + it('removeByDegradedSha256 drops every entry pointing at the derivative', async () => { + await cache.put(ORIGINAL, fp, ENTRY); + await cache.put('c'.repeat(64), fp, ENTRY); + await cache.put('e'.repeat(64), fp, { + ...ENTRY, + degradedSha256: 'f'.repeat(64), + }); + + await cache.removeByDegradedSha256(DEGRADED); + await expect(cache.get(ORIGINAL, fp)).resolves.toBeNull(); + await expect(cache.get('c'.repeat(64), fp)).resolves.toBeNull(); + await expect(cache.get('e'.repeat(64), fp)).resolves.not.toBeNull(); + }); + + it('backs up a corrupt cache file and starts fresh (never fatal)', async () => { + const filePath = path.join(root, 'policy-cache.json'); + await fs.writeFile(filePath, '{corrupt'); + await expect(cache.get(ORIGINAL, fp)).resolves.toBeNull(); + const names = await fs.readdir(root); + expect(names.some((n) => n.startsWith('policy-cache.json.corrupt-'))).toBe( + true, + ); + // And the cache is usable again. + await cache.put(ORIGINAL, fp, ENTRY); + await expect(cache.get(ORIGINAL, fp)).resolves.toMatchObject(ENTRY); + }); + + it('writes atomically: no .tmp litter, 0600 file mode', async () => { + await cache.put(ORIGINAL, fp, ENTRY); + const names = await fs.readdir(root); + expect(names.filter((n) => n.includes('.tmp-'))).toEqual([]); + if (process.platform !== 'win32') { + const stat = await fs.stat(path.join(root, 'policy-cache.json')); + expect(stat.mode & 0o777).toBe(0o600); + } + }); + + it('serializes concurrent puts without losing entries', async () => { + await Promise.all( + Array.from({ length: 8 }, (_, i) => + cache.put(ORIGINAL, computePolicyFingerprint('t', { i }), ENTRY), + ), + ); + const raw = JSON.parse( + await fs.readFile(path.join(root, 'policy-cache.json'), 'utf8'), + ); + expect(Object.keys(raw.entries)).toHaveLength(8); + }); +}); diff --git a/packages/core/src/omni/policy/degradation-cache.ts b/packages/core/src/omni/policy/degradation-cache.ts new file mode 100644 index 00000000000..bc96556347a --- /dev/null +++ b/packages/core/src/omni/policy/degradation-cache.ts @@ -0,0 +1,157 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { createHash } from 'node:crypto'; +import path from 'node:path'; +import { OmniJsonCacheFile } from '../json-cache-file.js'; + +/** + * Identity of one degradation result (decision D2): everything the + * orchestrator needs to reuse a previously transcoded derivative without + * re-running the tool — the derived object's content hash (locating it in + * `objects/`), the extension it was stored under (storage.ts convention: + * leading dot), and the disclosure text that must accompany the lossy + * derivative on every delivery. + */ +export interface DegradationCacheEntry { + degradedSha256: string; + /** Object-store extension INCLUDING the leading dot (".jpg"). */ + extension: string; + /** Disclosure the tool emitted (D8) — redelivered verbatim on reuse. */ + disclosure: string; + mimeType: string; + createdAt: string; +} + +/** Io params are per-invocation plumbing, never policy identity: the same + * policy applied to the same object must hit regardless of where the + * source file sat or which staging dir the run used. */ +const FINGERPRINT_EXCLUDED_KEYS = new Set(['inputPath', 'outputDir']); + +/** Deterministic JSON: objects serialized with sorted keys at every + * depth, so `{a,b}` and `{b,a}` fingerprint identically. */ +function stableStringify(value: unknown): string { + if (Array.isArray(value)) { + return `[${value.map(stableStringify).join(',')}]`; + } + if (typeof value === 'object' && value !== null) { + const record = value as Record; + const body = Object.keys(record) + .sort() + .map((k) => `${JSON.stringify(k)}:${stableStringify(record[k])}`) + .join(','); + return `{${body}}`; + } + return JSON.stringify(value) ?? 'null'; +} + +/** + * `policyFingerprint = sha256(toolName + normalized arguments + tool + * version)` (decision D2). Arguments are normalized by dropping the + * per-invocation io params and key-sorting the rest, so semantically + * identical calls fingerprint identically. `toolVersion` exists to + * invalidate cached results when a tool's transcode behavior changes + * without any argument changing. + */ +export function computePolicyFingerprint( + toolName: string, + args: Record, + toolVersion = '1', +): string { + const tunables: Record = {}; + for (const [k, v] of Object.entries(args)) { + if (!FINGERPRINT_EXCLUDED_KEYS.has(k) && v !== undefined) { + tunables[k] = v; + } + } + return createHash('sha256') + .update(`${toolName}\n${stableStringify(tunables)}\n${toolVersion}`) + .digest('hex'); +} + +/** + * Persistent map from `(originalSha256, policyFingerprint)` to the + * degraded derivative's identity (decision D2). Lives at + * `.qwen/omni/policy-cache.json`; a hit whose object still exists in + * `objects/` lets the orchestrator skip a minutes-long transcode. The + * existence check is the orchestrator's job — this cache only answers + * "what did this policy produce last time". + * + * File mechanics (serialized ops, atomic writes, corrupt backup+rebuild, + * unreadable-file no-op) are shared with the upload cache via + * {@link OmniJsonCacheFile}. Entries carry no TTL: identities are + * content-addressed and never go stale — they are invalidated + * explicitly when the underlying object disappears (GC/corruption). + */ +export class OmniDegradationCache { + private readonly file: OmniJsonCacheFile; + + constructor(omniRootDir: string) { + this.file = new OmniJsonCacheFile( + path.join(omniRootDir, 'policy-cache.json'), + 'omni:policy-cache', + ); + } + + private key(originalSha256: string, policyFingerprint: string): string { + return `${originalSha256}|${policyFingerprint}`; + } + + async get( + originalSha256: string, + policyFingerprint: string, + ): Promise { + return this.file.access(null, (entries) => ({ + result: entries[this.key(originalSha256, policyFingerprint)] ?? null, + })); + } + + async put( + originalSha256: string, + policyFingerprint: string, + entry: Omit, + ): Promise { + return this.file.access(undefined, (entries) => { + entries[this.key(originalSha256, policyFingerprint)] = { + ...entry, + createdAt: new Date().toISOString(), + }; + return { result: undefined, changed: true }; + }); + } + + /** Drop every policy result derived FROM the object (source object + * gone/corrupt — GC cascade). */ + async removeByOriginalSha256(originalSha256: string): Promise { + return this.file.access(undefined, (entries) => { + const prefix = `${originalSha256}|`; + let changed = false; + for (const k of Object.keys(entries)) { + if (k.startsWith(prefix)) { + delete entries[k]; + changed = true; + } + } + return { result: undefined, changed }; + }); + } + + /** Drop every entry POINTING AT the derived object (derivative + * gone/corrupt — the next run must re-transcode, not chase a missing + * object). */ + async removeByDegradedSha256(degradedSha256: string): Promise { + return this.file.access(undefined, (entries) => { + let changed = false; + for (const [k, v] of Object.entries(entries)) { + if (v.degradedSha256 === degradedSha256) { + delete entries[k]; + changed = true; + } + } + return { result: undefined, changed }; + }); + } +} diff --git a/packages/core/src/omni/upload-cache.ts b/packages/core/src/omni/upload-cache.ts index f76660a2cf1..d38663b8368 100644 --- a/packages/core/src/omni/upload-cache.ts +++ b/packages/core/src/omni/upload-cache.ts @@ -4,38 +4,12 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { randomBytes } from 'node:crypto'; -import fs from 'node:fs/promises'; import path from 'node:path'; import { createDebugLogger } from '../utils/debugLogger.js'; +import { OmniJsonCacheFile } from './json-cache-file.js'; const debugLogger = createDebugLogger('omni:upload-cache'); -/** Per-cache-file operation serializer: cache instances are constructed - * per delivery, and safe-tool batches run deliveries concurrently in one - * process — unserialized load-modify-save would drop entries (worst case - * one extra re-upload, but cheap to prevent). Module scope is deliberate: - * two instances on the same root must share the chain. Cross-process - * writes remain last-writer-wins (documented). */ -const fileOps = new Map>(); - -function serialize(key: string, fn: () => Promise): Promise { - const prev = fileOps.get(key) ?? Promise.resolve(); - const run = prev.then(fn, fn); - const settled = run.then( - () => {}, - () => {}, - ); - fileOps.set(key, settled); - void settled.then(() => { - // Drop the tail once it settles — otherwise the map grows with every - // distinct cache file touched over the process lifetime. Only delete - // when OUR promise is still the tail: a later op may have chained on. - if (fileOps.get(key) === settled) fileOps.delete(key); - }); - return run; -} - /** Default oss:// URL validity horizon: 47h (official 48h minus margin). */ export const DEFAULT_UPLOAD_CACHE_TTL_HOURS = 47; @@ -43,28 +17,21 @@ export const DEFAULT_UPLOAD_CACHE_TTL_HOURS = 47; * so a longer local TTL would confidently serve dead URLs. */ const MAX_UPLOAD_CACHE_TTL_HOURS = 48; -/** Keep at most this many `.corrupt-*` backups (newest wins): a crash - * loop over a corrupt file must not litter the directory without bound. */ -const MAX_CORRUPT_BACKUPS = 2; - interface UploadCacheEntry { ossUrl: string; uploadedAt: string; expiresAt: string; } -interface UploadCacheFile { - version: 1; - /** Key: `||`. */ - entries: Record; -} - /** * Persistent map from object identity to a still-valid DashScope temporary * URL: `(sha256, model, scope) → { ossUrl, uploadedAt, expiresAt }` * (storage design §8). Lives at `.qwen/omni/upload-cache.json`. * - * Invariants: + * File mechanics (serialized ops, atomic writes, corrupt backup+rebuild, + * unreadable-file no-op) live in {@link OmniJsonCacheFile}. Entry + * invariants: + * * - the cache file is the ONLY place an oss:// URL is persisted by omni — * the URL is a delivery cache, never an identity; * - keys include the model: the docs declare uploads model-bound (looser @@ -78,18 +45,11 @@ interface UploadCacheFile { * - expired entries are misses; they are pruned on read and swept * wholesale on every {@link put} (so never-read-again entries cannot * accumulate forever); - * - a corrupt cache file is backed up and rebuilt empty (never fatal), - * keeping at most the newest {@link MAX_CORRUPT_BACKUPS} backups; - * - a cache file that exists but cannot be READ (EACCES, EMFILE, …) makes - * the current operation a no-op instead of an empty-file rebuild — a - * transient read failure must never lead to a save that wipes every - * previously persisted entry; - * - writes are atomic (tmp + rename, 0600) and last-writer-wins across - * processes — acceptable for the experiment (worst case: a lost entry - * causes one extra re-upload). + * - writes are last-writer-wins across processes — acceptable for the + * experiment (worst case: a lost entry causes one extra re-upload). */ export class OmniUploadCache { - private readonly filePath: string; + private readonly file: OmniJsonCacheFile; private readonly ttlMs: number; private readonly scope: string; @@ -98,7 +58,10 @@ export class OmniUploadCache { ttlHours = DEFAULT_UPLOAD_CACHE_TTL_HOURS, scope = '', ) { - this.filePath = path.join(omniRootDir, 'upload-cache.json'); + this.file = new OmniJsonCacheFile( + path.join(omniRootDir, 'upload-cache.json'), + 'omni:upload-cache', + ); // Positive TTLs are clamped to the 48h server URL lifetime — a // configured 168 must not outlive the URL. 0/negative still disables. this.ttlMs = Math.min(ttlHours, MAX_UPLOAD_CACHE_TTL_HOURS) * 3600_000; @@ -110,93 +73,6 @@ export class OmniUploadCache { return this.ttlMs > 0; } - /** - * Load the cache file. Returns null when the file exists but could not - * be read (EACCES, EMFILE, …): the caller must skip its operation for - * this call — proceeding with an empty snapshot and later saving it - * would overwrite N valid entries with one (self-inflicted cache wipe). - * Only a genuinely missing file means empty-and-writable. - */ - private async load(): Promise { - let raw: string; - try { - raw = await fs.readFile(this.filePath, 'utf8'); - } catch (err) { - const code = (err as NodeJS.ErrnoException).code; - // ENOENT: no cache file yet (POSIX and Windows). ENOTDIR: a parent - // path component is a plain file — POSIX raises ENOTDIR where - // Windows reports ENOENT for the same condition. - if (code === 'ENOENT' || code === 'ENOTDIR') { - return { version: 1, entries: {} }; - } - debugLogger.debug( - `upload cache read failed, operation skipped: ${err instanceof Error ? err.message : err}`, - ); - return null; - } - try { - const parsed = JSON.parse(raw) as UploadCacheFile; - // `entries` must be a plain non-null object: `typeof null` and - // `typeof []` are both 'object', and either shape would throw raw - // TypeErrors from every accessor below (escaping the never-fatal - // contract and skipping backup+rebuild). - if ( - parsed?.version === 1 && - typeof parsed.entries === 'object' && - parsed.entries !== null && - !Array.isArray(parsed.entries) - ) { - return parsed; - } - throw new Error('unexpected shape'); - } catch { - // Corrupt cache: preserve for inspection, start fresh. Losing the - // cache only costs re-uploads — never fail the pipeline over it. - const backup = `${this.filePath}.corrupt-${Date.now()}`; - await fs.rename(this.filePath, backup).catch(() => {}); - await this.pruneCorruptBackups(); - debugLogger.debug(`corrupt upload cache backed up to ${backup}`); - return { version: 1, entries: {} }; - } - } - - /** Best-effort: keep only the newest {@link MAX_CORRUPT_BACKUPS}. */ - private async pruneCorruptBackups(): Promise { - const dir = path.dirname(this.filePath); - const prefix = `${path.basename(this.filePath)}.corrupt-`; - try { - const backups = (await fs.readdir(dir)) - .filter((n) => n.startsWith(prefix)) - // Millisecond timestamps are fixed-width for centuries, so the - // lexicographic sort is chronological; newest first. - .sort() - .reverse(); - for (const name of backups.slice(MAX_CORRUPT_BACKUPS)) { - await fs.rm(path.join(dir, name), { force: true }).catch(() => {}); - } - } catch { - // Pruning is hygiene; never let it affect the read path. - } - } - - private async save(data: UploadCacheFile): Promise { - const tmp = `${this.filePath}.tmp-${randomBytes(4).toString('hex')}`; - try { - await fs.mkdir(path.dirname(this.filePath), { - recursive: true, - mode: 0o700, - }); - await fs.writeFile(tmp, JSON.stringify(data, null, 1), { mode: 0o600 }); - await fs.rename(tmp, this.filePath); - } catch (err) { - await fs.rm(tmp, { force: true }).catch(() => {}); - // Cache persistence is best-effort by design. - debugLogger.debug( - `upload cache write failed: ${err instanceof Error ? err.message : err}`, - ); - } - } - private key(sha256: string, model: string): string { return `${sha256}|${model}|${this.scope}`; } @@ -204,35 +80,25 @@ export class OmniUploadCache { /** Valid cached URL or null. Expired entries are pruned on read. */ async get(sha256: string, model: string): Promise { if (!this.enabled) return null; - return serialize(this.filePath, () => this.getInner(sha256, model)); - } - - private async getInner( - sha256: string, - model: string, - ): Promise { - const data = await this.load(); - if (!data) return null; - const k = this.key(sha256, model); - const entry = data.entries[k]; - if (!entry) return null; - const expiresAtMs = Date.parse(entry.expiresAt); - // Malformed timestamps (NaN) must expire, not live forever. - if (!Number.isFinite(expiresAtMs) || expiresAtMs <= Date.now()) { - delete data.entries[k]; - await this.save(data); - return null; - } - return entry.ossUrl; + return this.file.access(null, (entries) => { + const k = this.key(sha256, model); + const entry = entries[k]; + if (!entry) return { result: null }; + const expiresAtMs = Date.parse(entry.expiresAt); + // Malformed timestamps (NaN) must expire, not live forever. + if (!Number.isFinite(expiresAtMs) || expiresAtMs <= Date.now()) { + delete entries[k]; + return { result: null, changed: true }; + } + return { result: entry.ossUrl }; + }); } async put(sha256: string, model: string, ossUrl: string): Promise { if (!this.enabled) return; - return serialize(this.filePath, async () => { - const data = await this.load(); - if (!data) return; + return this.file.access(undefined, (entries) => { const now = Date.now(); - data.entries[this.key(sha256, model)] = { + entries[this.key(sha256, model)] = { ossUrl, uploadedAt: new Date(now).toISOString(), expiresAt: new Date(now + this.ttlMs).toISOString(), @@ -242,11 +108,11 @@ export class OmniUploadCache { // otherwise accumulate forever — every load/save re-parses and // rewrites the whole table, monotonically slowing with dead history. // put() is the natural hook: it already holds the serialized write. - for (const [k, v] of Object.entries(data.entries)) { + for (const [k, v] of Object.entries(entries)) { const t = Date.parse(v.expiresAt); - if (!Number.isFinite(t) || t <= now) delete data.entries[k]; + if (!Number.isFinite(t) || t <= now) delete entries[k]; } - await this.save(data); + return { result: undefined, changed: true }; }); } @@ -260,20 +126,18 @@ export class OmniUploadCache { * not which endpoint scope minted it. */ async invalidateByUrl(ossUrl: string): Promise { - return serialize(this.filePath, async () => { - const data = await this.load(); - if (!data) return; + return this.file.access(undefined, (entries) => { let changed = false; - for (const [k, v] of Object.entries(data.entries)) { + for (const [k, v] of Object.entries(entries)) { if (v.ossUrl === ossUrl) { - delete data.entries[k]; + delete entries[k]; changed = true; } } if (changed) { debugLogger.debug(`invalidated upload cache entries for ${ossUrl}`); - await this.save(data); } + return { result: undefined, changed }; }); } @@ -284,18 +148,16 @@ export class OmniUploadCache { * corrupt for every endpoint. */ async removeBySha256(sha256: string): Promise { - return serialize(this.filePath, async () => { - const data = await this.load(); - if (!data) return; + return this.file.access(undefined, (entries) => { const prefix = `${sha256}|`; let changed = false; - for (const k of Object.keys(data.entries)) { + for (const k of Object.keys(entries)) { if (k.startsWith(prefix)) { - delete data.entries[k]; + delete entries[k]; changed = true; } } - if (changed) await this.save(data); + return { result: undefined, changed }; }); } } From 0bb99b704af294b166078c2a778ff5bd74bdf752 Mon Sep 17 00:00:00 2001 From: LaZzyMan Date: Fri, 7 Aug 2026 11:10:44 +0800 Subject: [PATCH 08/62] feat(omni): add fixed-policy orchestrator and reorder the delivery pipeline Introduce runFixedPolicies: for each recognized media resource it matches normalized fixed policies (modality, origins, when-conditions), executes the policy's media tool through the real executeToolCall protocol (fixed_policy execution origin, recordToolResult:false, isolated staging outputDir), validates the returned policy artifacts against the tool's declared descriptor (workspace containment, recognized kind/mime match, mandatory disclosure for lossy outputs), promotes derivatives into objects/ and records them in the degradation cache keyed by original sha + policy fingerprint. Reorder processMediaForOmniDelivery so transport guards judge the FINAL delivery set instead of the source: recognize -> recovery -> fixed policies -> guards -> hash -> upload. Degraded deliveries carry a disclosure that is emitted as a text Part immediately before the media Part everywhere media surfaces (read tool results, tool-result inline media conversion), and the OpenAI converter moves the disclosure together with its media part when splitting tool media into a follow-up user message. --- .../openaiContentGenerator/converter.test.ts | 120 ++++ .../core/openaiContentGenerator/converter.ts | 13 + packages/core/src/omni/disclosure.ts | 33 + packages/core/src/omni/index.test.ts | 279 ++++++++ packages/core/src/omni/index.ts | 203 ++++-- .../core/src/omni/policy/orchestrator.test.ts | 619 ++++++++++++++++++ packages/core/src/omni/policy/orchestrator.ts | 562 ++++++++++++++++ packages/core/src/omni/policy/types.ts | 61 ++ .../core/src/omni/tool-result-media.test.ts | 80 +++ packages/core/src/omni/tool-result-media.ts | 48 +- 10 files changed, 1949 insertions(+), 69 deletions(-) create mode 100644 packages/core/src/omni/disclosure.ts create mode 100644 packages/core/src/omni/policy/orchestrator.test.ts create mode 100644 packages/core/src/omni/policy/orchestrator.ts diff --git a/packages/core/src/core/openaiContentGenerator/converter.test.ts b/packages/core/src/core/openaiContentGenerator/converter.test.ts index f2168cf8c07..f4611937993 100644 --- a/packages/core/src/core/openaiContentGenerator/converter.test.ts +++ b/packages/core/src/core/openaiContentGenerator/converter.test.ts @@ -1502,6 +1502,126 @@ describe('OpenAIContentConverter', () => { ); }); + it('moves an omni degradation disclosure together with its media part when splitting tool media', () => { + // The omni pipeline emits a disclosure text Part IMMEDIATELY before + // each lossy derivative's media Part. When splitToolMedia relocates + // the media into the follow-up user message, the disclosure must move + // WITH it — stranded in the text-only tool message, the model could + // not attribute it to the media. Ordinary text parts stay behind. + const request: GenerateContentParameters = { + model: 'models/test', + contents: [ + { + role: 'model', + parts: [{ functionCall: { id: 'call_1', name: 'Read', args: {} } }], + }, + { + role: 'user', + parts: [ + { + functionResponse: { + id: 'call_1', + name: 'Read', + response: { output: 'Image content' }, + parts: [ + { text: 'ordinary tool text' }, + { text: '【媒体降质】photo.png：downsampled to 1568px' }, + { + inlineData: { + mimeType: 'image/png', + data: 'base64encodedimagedata', + }, + }, + ] as unknown as Part[], + }, + }, + ], + }, + ], + }; + + const messages = converter.convertGeminiRequestToOpenAI(request, { + ...requestContext, + splitToolMedia: true, + }); + + const toolMessage = messages.find((m) => m.role === 'tool'); + expect(toolMessage?.content).toBe('Image content\nordinary tool text'); + + const userMessage = messages.find((m) => m.role === 'user'); + const userContent = userMessage?.content as Array<{ + type: string; + text?: string; + image_url?: { url: string }; + }>; + expect(userContent.map((p) => p.type)).toEqual([ + 'text', + 'text', + 'image_url', + ]); + expect(userContent[0].text).toBe( + '(attached media from previous tool call)', + ); + // Disclosure sits immediately before its media part. + expect(userContent[1].text).toBe( + '【媒体降质】photo.png：downsampled to 1568px', + ); + expect(userContent[2].image_url?.url).toBe( + 'data:image/png;base64,base64encodedimagedata', + ); + }); + + it('gives a disclosure only to the media part directly following it', () => { + // Two media parts after one disclosure: only the adjacent one owns + // it — the second media part must not pull the disclosure past the + // first (prev-tracking, not "last disclosure seen"). + const request: GenerateContentParameters = { + model: 'models/test', + contents: [ + { + role: 'model', + parts: [{ functionCall: { id: 'call_1', name: 'Read', args: {} } }], + }, + { + role: 'user', + parts: [ + { + functionResponse: { + id: 'call_1', + name: 'Read', + response: { output: 'two images' }, + parts: [ + { text: '【媒体降质】a.png：lossy' }, + { inlineData: { mimeType: 'image/png', data: 'first' } }, + { inlineData: { mimeType: 'image/png', data: 'second' } }, + ] as unknown as Part[], + }, + }, + ], + }, + ], + }; + + const messages = converter.convertGeminiRequestToOpenAI(request, { + ...requestContext, + splitToolMedia: true, + }); + const userMessage = messages.find((m) => m.role === 'user'); + const userContent = userMessage?.content as Array<{ + type: string; + text?: string; + image_url?: { url: string }; + }>; + expect(userContent.map((p) => p.type)).toEqual([ + 'text', + 'text', + 'image_url', + 'image_url', + ]); + expect(userContent[1].text).toBe('【媒体降质】a.png：lossy'); + expect(userContent[2].image_url?.url).toBe('data:image/png;base64,first'); + }); + it('should keep all tool messages contiguous and merge split media into a single follow-up user message for parallel tool calls (issue #3616)', () => { // Two assistant tool calls in parallel. Both responses come back in the // same `user` content as separate functionResponse parts. The first diff --git a/packages/core/src/core/openaiContentGenerator/converter.ts b/packages/core/src/core/openaiContentGenerator/converter.ts index d8dd2dcd038..7a92100c44e 100644 --- a/packages/core/src/core/openaiContentGenerator/converter.ts +++ b/packages/core/src/core/openaiContentGenerator/converter.ts @@ -40,6 +40,7 @@ import { } from '../tool-call-preparation.js'; import { InvalidStreamError } from '../invalid-stream-error.js'; import { normalizeMcpToolName } from '../../utils/tool-name-utils.js'; +import { isDisclosureText } from '../../omni/disclosure.js'; import { setGenAiUsageProvenance } from '../../telemetry/gen-ai-usage.js'; const debugLogger = createDebugLogger('CONVERTER'); @@ -668,6 +669,11 @@ function processContent( ) { const mediaParts: OpenAIContentPart[] = []; const textParts: OpenAI.Chat.ChatCompletionContentPartText[] = []; + // Track the previous part so an omni media-degradation disclosure + // (emitted immediately before its media part) moves WITH the media + // into the follow-up user message instead of being stranded in the + // text-only tool message, where the model could not attribute it. + let prev: OpenAIContentPart | undefined; for (const cp of toolMessage.content as OpenAIContentPart[]) { if ( cp && @@ -676,10 +682,17 @@ function processContent( cp.type === 'video_url' || cp.type === 'file') ) { + if (prev?.type === 'text' && isDisclosureText(prev.text)) { + textParts.pop(); + mediaParts.push(prev); + } mediaParts.push(cp); } else if (cp && cp.type === 'text') { textParts.push(cp); } + // Consecutive media parts after one disclosure must not each + // claim it: only the part directly following the text does. + prev = cp; } if (mediaParts.length > 0) { const textOnly = textParts.map((p) => p.text).join('\n'); diff --git a/packages/core/src/omni/disclosure.ts b/packages/core/src/omni/disclosure.ts new file mode 100644 index 00000000000..ba4d5a7a9fc --- /dev/null +++ b/packages/core/src/omni/disclosure.ts @@ -0,0 +1,33 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Disclosure text delivery (decision D8): a lossy policy derivative must + * reach the model with its disclosure IMMEDIATELY adjacent to the media + * Part, so provider converters that relocate media (splitToolMedia) can + * move the pair together and the model can attribute the disclosure to + * the right resource. + * + * Deliberately a leaf module — imported by both the omni pipeline and the + * OpenAI converter, so it must not pull in either side. + */ + +/** Marks a text Part as a media-degradation disclosure. Converters key on + * this prefix to keep the disclosure adjacent to its media part. */ +export const OMNI_DISCLOSURE_TEXT_PREFIX = '【媒体降质】'; + +/** Model-facing disclosure text for one degraded resource. */ +export function formatDisclosureText( + displayName: string, + disclosure: string, +): string { + return `${OMNI_DISCLOSURE_TEXT_PREFIX}${displayName}：${disclosure}`; +} + +/** Whether a text is a disclosure emitted by {@link formatDisclosureText}. */ +export function isDisclosureText(text: string): boolean { + return text.startsWith(OMNI_DISCLOSURE_TEXT_PREFIX); +} diff --git a/packages/core/src/omni/index.test.ts b/packages/core/src/omni/index.test.ts index 8688a712250..ee96ad3eb8c 100644 --- a/packages/core/src/omni/index.test.ts +++ b/packages/core/src/omni/index.test.ts @@ -688,3 +688,282 @@ describe('processMediaForOmniDelivery upload cache integration', () => { await expect(fs.stat(expired)).rejects.toMatchObject({ code: 'ENOENT' }); }); }); + +describe('processMediaForOmniDelivery fixed-policy integration', () => { + // The orchestrator itself is unit-tested in policy/orchestrator.test.ts; + // these tests pin the pipeline wiring around it: when it runs, what it + // receives, how its output replaces the source, that the transport guard + // judges the FINAL delivery (decision D1), and how failures surface. + let tmpDir: string; + + beforeEach(async () => { + vi.resetModules(); + tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'omni-policy-int-')); + }); + + afterEach(async () => { + vi.resetAllMocks(); + vi.doUnmock('./ffmpeg.js'); + vi.doUnmock('./recognition.js'); + vi.doUnmock('./storage.js'); + vi.doUnmock('./upload.js'); + vi.doUnmock('./policy/orchestrator.js'); + await fs.rm(tmpDir, { recursive: true, force: true }); + }); + + const SOURCE_RECOGNIZED = { + modality: 'image', + detectedMimeType: 'image/png', + sizeBytes: 5000, + metadata: { width: 4000, height: 3000 }, + }; + const DEGRADED_RECOGNIZED = { + modality: 'image', + detectedMimeType: 'image/jpeg', + sizeBytes: 100, + metadata: { width: 1568, height: 1176 }, + }; + // Only `.length > 0` matters to the pipeline; the mocked orchestrator + // never reads the entries. + const POLICY_STUB = [{ id: 'img-downsample' }]; + + function policyConfig(overrides?: { + maxUploadFileBytes?: number; + policies?: unknown[]; + }): Config { + return { + isOmniEnabled: vi.fn().mockReturnValue(true), + isTrustedFolder: vi.fn().mockReturnValue(true), + getContentGeneratorConfig: vi.fn().mockReturnValue(DASHSCOPE_CGC), + getModel: vi.fn().mockReturnValue('qwen3.5-omni-plus'), + getOmniMaxUploadFileBytes: vi + .fn() + .mockReturnValue(overrides?.maxUploadFileBytes ?? 0), + getOmniMaxEstimatedTokens: vi.fn().mockReturnValue(0), + getOmniProcessingConfig: vi.fn().mockReturnValue({ + fixedPolicies: overrides?.policies ?? POLICY_STUB, + transportGuardPolicies: [], + }), + storage: { getQwenDir: () => tmpDir }, + } as unknown as Config; + } + + async function armPipeline(runFixedPoliciesMock: ReturnType) { + const putFileMock = vi + .fn() + .mockResolvedValue({ objectPath: '/tmp/obj.jpg', deduped: false }); + const uploadFileMock = vi.fn().mockResolvedValue('oss://bucket/degraded'); + const hashFileMock = vi.fn().mockResolvedValue('a'.repeat(64)); + vi.doMock('./ffmpeg.js', () => ({ + isFfmpegAvailable: vi.fn().mockResolvedValue(true), + isFfprobeAvailable: vi.fn().mockResolvedValue(true), + })); + vi.doMock('./recognition.js', () => ({ + recognizeMediaFile: vi.fn().mockResolvedValue(SOURCE_RECOGNIZED), + hashFileSha256: hashFileMock, + extensionForMime: vi.fn().mockReturnValue('.jpg'), + })); + const objectsDir = path.join(tmpDir, 'objects'); + vi.doMock('./storage.js', () => ({ + OmniObjectStore: class { + putFile = putFileMock; + getOmniRootDir() { + return tmpDir; + } + getObjectsDir() { + return objectsDir; + } + }, + })); + vi.doMock('./upload.js', () => ({ + DashScopeUploader: class { + uploadFile = uploadFileMock; + }, + OSS_URL_PREFIX: 'oss://', + })); + vi.doMock('./policy/orchestrator.js', () => ({ + runFixedPolicies: runFixedPoliciesMock, + OmniPolicyExecutionError: class extends Error {}, + })); + const mod = await import('./index.js'); + return { putFileMock, uploadFileMock, hashFileMock, mod }; + } + + async function realFile(name: string): Promise { + const filePath = path.join(tmpDir, name); + await fs.writeFile(filePath, 'not really media'); + return filePath; + } + + it('replaces the source with the policy derivative and carries its disclosure', async () => { + const degradedPath = path.join(tmpDir, 'objects', 'deadbeef.jpg'); + const runMock = vi.fn().mockResolvedValue({ + deliveries: [ + { + filePath: degradedPath, + recognized: DEGRADED_RECOGNIZED, + sha256: 'b'.repeat(64), + disclosure: 'downsampled to 1568px', + degraded: true, + }, + ], + records: [], + }); + const { putFileMock, hashFileMock, mod } = await armPipeline(runMock); + const filePath = await realFile('pic.png'); + const config = policyConfig(); + + const result = await mod.processMediaForOmniDelivery(filePath, config); + + // The orchestrator received the source resource with user provenance. + expect(runMock).toHaveBeenCalledTimes(1); + expect(runMock).toHaveBeenCalledWith( + config, + { + filePath, + recognized: SOURCE_RECOGNIZED, + displayName: 'pic.png', + origin: 'user', + }, + expect.objectContaining({ policies: POLICY_STUB }), + ); + // Storage/upload operate on the DERIVATIVE under its promotion hash; + // the source is never re-hashed (the derivative arrived with one). + expect(putFileMock).toHaveBeenCalledWith( + degradedPath, + 'b'.repeat(64), + '.jpg', + undefined, + ); + expect(hashFileMock).not.toHaveBeenCalled(); + expect(result.fileUri).toBe('oss://bucket/degraded'); + expect(result.mimeType).toBe('image/jpeg'); + expect(result.sha256).toBe('b'.repeat(64)); + expect(result.recognized).toBe(DEGRADED_RECOGNIZED); + expect(result.disclosure).toBe('downsampled to 1568px'); + expect(result.degraded).toBe(true); + }); + + it('skips the orchestrator entirely when no fixed policies are configured', async () => { + const runMock = vi.fn(); + const { mod } = await armPipeline(runMock); + const result = await mod.processMediaForOmniDelivery( + await realFile('pic.png'), + policyConfig({ policies: [] }), + ); + expect(runMock).not.toHaveBeenCalled(); + expect(result.disclosure).toBeUndefined(); + expect(result.degraded).toBeUndefined(); + }); + + it('judges the transport byte guard on the FINAL delivery, not the source', async () => { + // Source 5000 bytes, cap 500: without policies this delivery would be + // rejected. The derivative is 100 bytes — the reordered pipeline (D1) + // must accept it. + const runMock = vi.fn().mockResolvedValue({ + deliveries: [ + { + filePath: path.join(tmpDir, 'objects', 'deadbeef.jpg'), + recognized: DEGRADED_RECOGNIZED, + sha256: 'b'.repeat(64), + disclosure: 'downsampled to 1568px', + degraded: true, + }, + ], + records: [], + }); + const { mod } = await armPipeline(runMock); + const result = await mod.processMediaForOmniDelivery( + await realFile('pic.png'), + policyConfig({ maxUploadFileBytes: 500 }), + ); + expect(result.degraded).toBe(true); + }); + + it('still rejects when the FINAL delivery exceeds the byte cap', async () => { + const runMock = vi.fn().mockResolvedValue({ + deliveries: [ + { + filePath: path.join(tmpDir, 'objects', 'deadbeef.jpg'), + recognized: { ...DEGRADED_RECOGNIZED, sizeBytes: 900 }, + sha256: 'b'.repeat(64), + degraded: true, + }, + ], + records: [], + }); + const { mod } = await armPipeline(runMock); + await expect( + mod.processMediaForOmniDelivery( + await realFile('pic.png'), + policyConfig({ maxUploadFileBytes: 500 }), + ), + ).rejects.toMatchObject({ name: 'OmniTransportGuardError' }); + }); + + it('wraps orchestrator failures into a sanitized OmniDeliveryError', async () => { + const runMock = vi.fn().mockRejectedValue(new Error('policy blew up')); + const { mod } = await armPipeline(runMock); + await expect( + mod.processMediaForOmniDelivery( + await realFile('pic.png'), + policyConfig(), + ), + ).rejects.toMatchObject({ + name: 'OmniDeliveryError', + message: 'Fixed-policy processing failed for pic.png: policy blew up', + }); + }); + + it('rejects a delivery set that is not exactly one resource', async () => { + const runMock = vi.fn().mockResolvedValue({ deliveries: [], records: [] }); + const { mod } = await armPipeline(runMock); + await expect( + mod.processMediaForOmniDelivery( + await realFile('pic.png'), + policyConfig(), + ), + ).rejects.toMatchObject({ + name: 'OmniDeliveryError', + message: + 'Fixed policies produced 0 deliverables for pic.png; exactly one is supported.', + }); + }); + + it('readMediaViaOmniDelivery places the disclosure immediately before the fileData part', async () => { + const runMock = vi.fn().mockResolvedValue({ + deliveries: [ + { + filePath: path.join(tmpDir, 'objects', 'deadbeef.jpg'), + recognized: DEGRADED_RECOGNIZED, + sha256: 'b'.repeat(64), + disclosure: 'downsampled to 1568px', + degraded: true, + }, + ], + records: [], + }); + const { mod } = await armPipeline(runMock); + const result = await mod.readMediaViaOmniDelivery({ + filePath: await realFile('pic.png'), + config: policyConfig(), + displayName: 'pic.png', + relativePathForDisplay: 'pic.png', + expectedModality: 'image', + }); + const parts = result.llmContent as Array>; + expect(parts).toHaveLength(3); + // Zoom hint shows the DELIVERED image's resolution (the derivative). + expect(parts[0]!['text']).toContain('1568x1176'); + expect(parts[1]!['text']).toBe( + '【媒体降质】pic.png：downsampled to 1568px', + ); + expect(parts[2]).toEqual({ + fileData: { + fileUri: 'oss://bucket/degraded', + mimeType: 'image/jpeg', + displayName: 'pic.png', + }, + }); + }); +}); diff --git a/packages/core/src/omni/index.ts b/packages/core/src/omni/index.ts index 89174de96bf..dab019d37c6 100644 --- a/packages/core/src/omni/index.ts +++ b/packages/core/src/omni/index.ts @@ -34,6 +34,12 @@ import { DEFAULT_UPLOAD_CACHE_TTL_HOURS, } from './upload-cache.js'; import { runStartupRecoveryOnce } from './recovery.js'; +import { formatDisclosureText } from './disclosure.js'; +import { + runFixedPolicies, + type PolicyDeliveryResource, +} from './policy/orchestrator.js'; +import type { OmniProcessingConfigView } from './policy/types.js'; export { assertOmniRuntimeDependencies, @@ -78,6 +84,22 @@ export { resetRecoveryLatchForTests, } from './recovery.js'; export { resetCredentialCacheForTests } from './upload.js'; +export { + OMNI_DISCLOSURE_TEXT_PREFIX, + formatDisclosureText, + isDisclosureText, +} from './disclosure.js'; +export { + runFixedPolicies, + OmniPolicyExecutionError, + type PolicyDeliveryResource, + type PolicyRunRecord, +} from './policy/orchestrator.js'; +export type { + FixedPolicyOrigin, + NormalizedFixedPolicy, + NormalizedOmniProcessingConfig, +} from './policy/types.js'; const debugLogger = createDebugLogger('omni'); @@ -140,6 +162,12 @@ export interface OmniMediaDelivery { /** True when the oss URL came from the persistent upload cache (no * network transfer happened for this delivery). */ uploadCacheHit: boolean; + /** Disclosure text that must accompany the media Part (present iff the + * delivered content is a lossy policy derivative). */ + disclosure?: string; + /** True when a fixed policy replaced the source with a lossy + * derivative. */ + degraded?: boolean; } /** Thrown for omni pipeline failures. The pipeline fails closed: callers @@ -200,18 +228,24 @@ export function isOmniDeliveryActive(config: Config): boolean { } /** - * Omni pipeline: recognize → transport guard → hash → promote into the - * content-addressed store → upload via the DashScope temporary channel → - * return the oss:// URL plus the token estimate. + * Omni pipeline: recognize → fixed policies (degradation) → transport + * guard → hash → promote into the content-addressed store → upload via the + * DashScope temporary channel → return the oss:// URL plus the token + * estimate. * - * All modalities are uploaded AS-IS — no resizing, no transcoding. - * Degradation is the job of S4 policies (which must disclose); the default - * path never silently alters content. Successful uploads are remembered in - * the persistent upload cache (`.qwen/omni/upload-cache.json`, keyed by - * sha256 + model + endpoint scope) for the oss URL validity window, so a - * re-read of unchanged content skips both the store copy and the network - * transfer. Throws OmniDeliveryError / OmniTransportGuardError on failure; - * user aborts propagate untouched. + * The default pipeline never SILENTLY alters content: any degradation is + * performed by configured fixed policies through real media-policy tools, + * and every lossy derivative carries a mandatory disclosure (decision D8) + * that reaches the model next to the media Part. The transport guard runs + * AFTER the policies (decision D1) so it judges what is actually delivered + * — an oversized original that a policy shrank must pass, and a policy + * failure leaves the guard as the backstop. Successful uploads are + * remembered in the persistent upload cache + * (`.qwen/omni/upload-cache.json`, keyed by sha256 + model + endpoint + * scope) for the oss URL validity window, so a re-read of unchanged + * content skips both the store copy and the network transfer. Throws + * OmniDeliveryError / OmniTransportGuardError on failure; user aborts + * propagate untouched. */ export async function processMediaForOmniDelivery( filePath: string, @@ -226,6 +260,9 @@ export async function processMediaForOmniDelivery( * user-recognizable name instead. */ displayName?: string; + /** Provenance for fixed-policy origin matching. Defaults to 'user'; + * the tool-result funnel passes 'tool'. */ + origin?: 'user' | 'tool'; }, ): Promise { const { expectedModality, signal } = options ?? {}; @@ -243,15 +280,15 @@ export async function processMediaForOmniDelivery( ); } - // Byte guard from a cheap stat BEFORE hashing/probing — a 60GB capture - // must not stream through SHA-256 only to be rejected. - const stat = await fs.stat(filePath).catch((err) => { + // Existence pre-check with a clean caller-facing error (recognition + // failures on a missing file read worse). The byte guard is NOT applied + // here anymore — it judges the post-policy delivery set below. + await fs.stat(filePath).catch((err) => { throw new OmniDeliveryError( `Cannot stat media file ${displayName}: ${sanitizeErrorMessage(err, [filePath])}`, { cause: err }, ); }); - assertWithinByteLimit(config, stat.size, displayName); let recognized: RecognizedMedia; try { @@ -267,23 +304,6 @@ export async function processMediaForOmniDelivery( ); } - // Token guard AFTER probe (needs metadata), BEFORE hash/copy/upload — a - // token-oversized input must not pay a full-file SHA-256 to be rejected. - const tokenEstimate = assertWithinTokenLimit(config, recognized, displayName); - - // Content hash: identity of the stored object. Computed only once all - // guards have passed, immediately before promotion into the store. - let sha256: string; - try { - sha256 = await hashFileSha256(filePath, signal); - } catch (err) { - if (signal?.aborted) throw err; - throw new OmniDeliveryError( - `Failed to hash media file ${displayName}: ${sanitizeErrorMessage(err, [filePath])}`, - { cause: err }, - ); - } - const store = new OmniObjectStore(config.storage.getQwenDir()); const cgc = config.getContentGeneratorConfig(); // Scope the cache to the endpoint credential: an oss:// URL minted for one @@ -303,9 +323,72 @@ export async function processMediaForOmniDelivery( cacheScope, ); // Lazy one-time hygiene scan (expired .part files, promotion orphans, - // sampled object verification). Never throws. + // sampled object verification). MUST run before the orchestrator: the + // scan deletes staging/ wholesale, which would race live invocations. await runStartupRecoveryOnce(store, uploadCache); + // Fixed-policy preprocessing (decision D5: this single site covers + // @-commands, tool results, the URL funnel and ACP). Structural view — + // the real accessor arrives with config normalization; a config without + // it (or with no policies) changes nothing. + let final: PolicyDeliveryResource = { filePath, recognized }; + const policies = + (config as OmniProcessingConfigView).getOmniProcessingConfig?.() + ?.fixedPolicies ?? []; + if (policies.length > 0) { + let deliveries: PolicyDeliveryResource[]; + try { + ({ deliveries } = await runFixedPolicies( + config, + { + filePath, + recognized, + displayName, + origin: options?.origin ?? 'user', + }, + { store, policies, signal }, + )); + } catch (err) { + if (signal?.aborted) throw err; + throw new OmniDeliveryError( + `Fixed-policy processing failed for ${displayName}: ` + + `${sanitizeErrorMessage(err, [filePath, store.getOmniRootDir()])}`, + { cause: err }, + ); + } + // The S4 delivery contract is one Part per source: every degradation + // tool is 1:1 with `source: omit`, so a differently-shaped set means + // a configuration this stage does not support yet. + if (deliveries.length !== 1) { + throw new OmniDeliveryError( + `Fixed policies produced ${deliveries.length} deliverables for ${displayName}; exactly one is supported.`, + ); + } + final = deliveries[0]; + } + + // Transport guard on the FINAL delivery set (decision D1): the bytes + // and token estimate judged are the ones actually delivered. + assertWithinByteLimit(config, final.recognized.sizeBytes, displayName); + const tokenEstimate = assertWithinTokenLimit( + config, + final.recognized, + displayName, + ); + + // Content hash: identity of the stored object. Derivatives arrive with + // their hash from promotion; sources are hashed here, after all guards. + let sha256: string; + try { + sha256 = final.sha256 ?? (await hashFileSha256(final.filePath, signal)); + } catch (err) { + if (signal?.aborted) throw err; + throw new OmniDeliveryError( + `Failed to hash media file ${displayName}: ${sanitizeErrorMessage(err, [final.filePath])}`, + { cause: err }, + ); + } + // Cache lookup BEFORE store promotion: a hit means the server already // holds these bytes for this model+endpoint, so neither the local copy // nor the upload is needed (zoom_image reads the original path, not the @@ -318,29 +401,31 @@ export async function processMediaForOmniDelivery( ); return { fileUri: cachedUrl, - mimeType: recognized.detectedMimeType, + mimeType: final.recognized.detectedMimeType, sha256, - recognized, + recognized: final.recognized, tokenEstimate, // No new copy was made: the content is already known to the system // (a prior delivery both stored and uploaded it). deduped: true, uploadCacheHit: true, + disclosure: final.disclosure, + degraded: final.degraded, }; } - const extension = extensionForMime(recognized.detectedMimeType); + const extension = extensionForMime(final.recognized.detectedMimeType); let objectPath: string; let deduped: boolean; try { - const put = await store.putFile(filePath, sha256, extension, signal); + const put = await store.putFile(final.filePath, sha256, extension, signal); objectPath = put.objectPath; deduped = put.deduped; } catch (err) { if (signal?.aborted) throw err; throw new OmniDeliveryError( `Failed to store media in the omni object store: ` + - `${sanitizeErrorMessage(err, [filePath, store.getOmniRootDir()])}`, + `${sanitizeErrorMessage(err, [final.filePath, store.getOmniRootDir()])}`, { cause: err }, ); } @@ -354,7 +439,7 @@ export async function processMediaForOmniDelivery( fileUri = await uploader.uploadFile({ filePath: objectPath, model, - mimeType: recognized.detectedMimeType, + mimeType: final.recognized.detectedMimeType, signal, }); } catch (err) { @@ -370,19 +455,21 @@ export async function processMediaForOmniDelivery( } debugLogger.debug( - `omni ${recognized.modality} delivered: sha256=${sha256.slice(0, 12)}… ` + - `size=${recognized.sizeBytes} est=${tokenEstimate.estimatedTokenCount}(${tokenEstimate.status}) ` + - `deduped=${deduped} uri=${fileUri}`, + `omni ${final.recognized.modality} delivered: sha256=${sha256.slice(0, 12)}… ` + + `size=${final.recognized.sizeBytes} est=${tokenEstimate.estimatedTokenCount}(${tokenEstimate.status}) ` + + `deduped=${deduped} degraded=${final.degraded === true} uri=${fileUri}`, ); await uploadCache.put(sha256, model, fileUri); return { fileUri, - mimeType: recognized.detectedMimeType, + mimeType: final.recognized.detectedMimeType, sha256, - recognized, + recognized: final.recognized, tokenEstimate, deduped, uploadCacheHit: false, + disclosure: final.disclosure, + degraded: final.degraded, }; } @@ -445,20 +532,28 @@ export async function readMediaViaOmniDelivery(params: { displayName, }, }; + const parts: Array<{ text: string } | typeof fileDataPart> = []; const { width, height } = delivery.recognized.metadata; - const llmContent = + if ( delivery.recognized.modality === 'image' && width !== undefined && height !== undefined - ? [ - { - text: - `Image ${displayName}: full resolution ${width}x${height} px. ` + - `Use zoom_image for a closer look at details.`, - }, - fileDataPart, - ] - : fileDataPart; + ) { + parts.push({ + text: + `Image ${displayName}: full resolution ${width}x${height} px. ` + + `Use zoom_image for a closer look at details.`, + }); + } + // Disclosure IMMEDIATELY before its media part (decision D8): provider + // converters that relocate media move the adjacent pair together. + if (delivery.disclosure) { + parts.push({ + text: formatDisclosureText(displayName, delivery.disclosure), + }); + } + parts.push(fileDataPart); + const llmContent = parts.length === 1 ? fileDataPart : parts; return { llmContent, returnDisplay: `Read ${delivery.recognized.modality} file (omni upload): ${relativePathForDisplay}`, diff --git a/packages/core/src/omni/policy/orchestrator.test.ts b/packages/core/src/omni/policy/orchestrator.test.ts new file mode 100644 index 00000000000..be8a263086c --- /dev/null +++ b/packages/core/src/omni/policy/orchestrator.test.ts @@ -0,0 +1,619 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { createHash } from 'node:crypto'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import type { Config } from '../../config/config.js'; +import type { ToolCallRequestInfo } from '../../core/turn.js'; +import type { MediaPolicyToolDescriptor } from '../../tools/tools.js'; +import type { RecognizedMedia } from '../recognition.js'; +import { OmniObjectStore } from '../storage.js'; +import { + computePolicyFingerprint, + OmniDegradationCache, +} from './degradation-cache.js'; +import { + OmniPolicyExecutionError, + runFixedPolicies, + type PolicySourceResource, +} from './orchestrator.js'; +import type { NormalizedFixedPolicy } from './types.js'; + +// The orchestrator resolves the executor with a dynamic import; vitest +// intercepts it the same as a static one. +const executeToolCallMock = vi.hoisted(() => vi.fn()); +vi.mock('../../core/nonInteractiveToolExecutor.js', () => ({ + executeToolCall: executeToolCallMock, +})); + +// Partial mock: recognizeMediaFile would need ffprobe; everything else +// (hashFileSha256 in particular — putFile re-hashes for real) stays real. +const recognizeMediaFileMock = vi.hoisted(() => vi.fn()); +vi.mock('../recognition.js', async (importOriginal) => ({ + ...(await importOriginal()), + recognizeMediaFile: recognizeMediaFileMock, +})); + +const SOURCE_BYTES = 'original-image-bytes'; +const DEGRADED_BYTES = 'degraded-image-bytes'; + +function sha256Of(text: string): string { + return createHash('sha256').update(text).digest('hex'); +} + +function recognizedImage( + overrides: Partial = {}, +): RecognizedMedia { + return { + modality: 'image', + detectedMimeType: 'image/png', + sizeBytes: SOURCE_BYTES.length, + metadata: { width: 4000, height: 3000 }, + ...overrides, + }; +} + +const DEGRADED_RECOGNIZED: RecognizedMedia = { + modality: 'image', + detectedMimeType: 'image/jpeg', + sizeBytes: DEGRADED_BYTES.length, + metadata: { width: 1568, height: 1176 }, +}; + +const DESCRIPTOR: MediaPolicyToolDescriptor = { + kind: 'media_policy', + inputMediaTypes: ['image'], + outputs: [ + { kind: 'media', mimeTypes: ['image/jpeg'], required: true, lossy: true }, + { kind: 'text', role: 'disclosure', required: true }, + ], +}; + +function makePolicy( + overrides: Partial = {}, +): NormalizedFixedPolicy { + return { + id: 'img-downsample', + priority: 0, + mediaTypes: ['image'], + origins: ['user', 'tool'], + onConditionUnavailable: 'skip', + toolName: 'omni_downsample_image', + arguments: { maxDimension: 1568 }, + maxRunsPerLineage: 1, + onFailure: 'continue', + output: { reprocessMedia: false, source: 'omit' }, + stage: 'preprocessing', + ...overrides, + }; +} + +function makeConfig( + descriptorByTool: Record, +) { + return { + getToolRegistry: () => ({ + getTool: (name: string) => + descriptorByTool[name] + ? { mediaPolicyDescriptor: descriptorByTool[name] } + : undefined, + }), + } as unknown as Config; +} + +describe('runFixedPolicies', () => { + let tmpDir: string; + let store: OmniObjectStore; + let sourcePath: string; + let source: PolicySourceResource; + let config: Config; + + /** Default success behavior: write a degraded artifact into the staging + * dir and return it as a workspace policy artifact with a disclosure. */ + function mockToolSuccess( + options: { + bytes?: string; + disclosure?: string | undefined; + fileName?: string; + } = {}, + ): void { + const bytes = options.bytes ?? DEGRADED_BYTES; + const fileName = options.fileName ?? 'out.jpg'; + const disclosure = + 'disclosure' in options + ? options.disclosure + : 'Downsampled from 4000x3000 to 1568x1176.'; + executeToolCallMock.mockImplementation( + async (_config: Config, request: ToolCallRequestInfo) => { + const outputDir = request.args['outputDir'] as string; + await fs.writeFile(path.join(outputDir, fileName), bytes); + return { + callId: request.callId, + responseParts: [], + resultDisplay: undefined, + error: undefined, + errorType: undefined, + policyArtifacts: { + toolName: request.name, + invocationId: request.callId, + executionOrigin: request.executionOrigin, + artifacts: [ + { + kind: 'image', + storage: 'workspace', + title: fileName, + workspacePath: fileName, + mimeType: 'image/jpeg', + ...(disclosure !== undefined + ? { metadata: { omniDisclosure: disclosure } } + : {}), + }, + ], + }, + }; + }, + ); + } + + beforeEach(async () => { + vi.clearAllMocks(); + tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'omni-orchestrator-')); + store = new OmniObjectStore(path.join(tmpDir, '.qwen')); + sourcePath = path.join(tmpDir, 'photo.png'); + await fs.writeFile(sourcePath, SOURCE_BYTES); + source = { + filePath: sourcePath, + recognized: recognizedImage(), + displayName: 'photo.png', + origin: 'user', + }; + config = makeConfig({ omni_downsample_image: DESCRIPTOR }); + recognizeMediaFileMock.mockImplementation(async (filePath: string) => { + if (filePath.endsWith('.jpg')) return DEGRADED_RECOGNIZED; + return recognizedImage(); + }); + }); + + afterEach(async () => { + await fs.rm(tmpDir, { recursive: true, force: true }); + }); + + it('delivers the source untouched when no policy matches its modality', async () => { + const { deliveries, records } = await runFixedPolicies(config, source, { + store, + policies: [makePolicy({ mediaTypes: ['video'] })], + }); + expect(deliveries).toEqual([ + { + filePath: sourcePath, + recognized: recognizedImage(), + sha256: undefined, + disclosure: undefined, + degraded: undefined, + }, + ]); + expect(records).toEqual([]); + expect(executeToolCallMock).not.toHaveBeenCalled(); + }); + + it('skips policies whose origins exclude the resource provenance', async () => { + const { deliveries, records } = await runFixedPolicies(config, source, { + store, + policies: [makePolicy({ origins: ['tool'] })], + }); + expect(deliveries).toHaveLength(1); + expect(deliveries[0].filePath).toBe(sourcePath); + expect(records).toEqual([]); + expect(executeToolCallMock).not.toHaveBeenCalled(); + }); + + it('executes a matching policy via the executor protocol and promotes the artifact', async () => { + mockToolSuccess(); + const { deliveries, records } = await runFixedPolicies(config, source, { + store, + policies: [makePolicy()], + }); + + // Exact executor protocol (the "complete minimal protocol" contract). + expect(executeToolCallMock).toHaveBeenCalledTimes(1); + const [calledConfig, request, signal, opts] = + executeToolCallMock.mock.calls[0]; + expect(calledConfig).toBe(config); + expect(signal).toBeInstanceOf(AbortSignal); + expect(opts).toEqual({ recordToolResult: false }); + const req = request as ToolCallRequestInfo; + expect(req.name).toBe('omni_downsample_image'); + expect(req.isClientInitiated).toBe(true); + expect(req.callId).toMatch(/^[0-9a-f]{16}$/); + expect(req.prompt_id).toBe(`omni-fixed-policy-${req.callId}`); + expect(req.executionOrigin).toEqual({ + kind: 'fixed_policy', + policyId: 'img-downsample', + stage: 'preprocessing', + }); + const stagingDir = path.join(store.getStagingDir(), req.callId); + expect(req.args).toEqual({ + maxDimension: 1568, + inputPath: sourcePath, + outputDir: stagingDir, + }); + + // Derivative promoted into objects/, source omitted. + const degradedSha = sha256Of(DEGRADED_BYTES); + const objectPath = store.objectPathFor(degradedSha, '.jpg'); + expect(deliveries).toEqual([ + { + filePath: objectPath, + recognized: DEGRADED_RECOGNIZED, + sha256: degradedSha, + disclosure: 'Downsampled from 4000x3000 to 1568x1176.', + degraded: true, + }, + ]); + await expect(fs.readFile(objectPath, 'utf8')).resolves.toBe(DEGRADED_BYTES); + + // Staging cleaned up; degradation cache written. + await expect(fs.readdir(store.getStagingDir())).resolves.toEqual([]); + const cache = new OmniDegradationCache(store.getOmniRootDir()); + const entry = await cache.get( + sha256Of(SOURCE_BYTES), + computePolicyFingerprint('omni_downsample_image', { maxDimension: 1568 }), + ); + expect(entry).toMatchObject({ + degradedSha256: degradedSha, + extension: '.jpg', + disclosure: 'Downsampled from 4000x3000 to 1568x1176.', + mimeType: 'image/jpeg', + }); + + expect(records).toEqual([ + { + policyId: 'img-downsample', + toolName: 'omni_downsample_image', + outcome: 'succeeded', + resource: 'photo.png', + }, + ]); + }); + + it('keeps the source alongside the derivative when output.source is keep', async () => { + mockToolSuccess(); + const { deliveries } = await runFixedPolicies(config, source, { + store, + policies: [ + makePolicy({ output: { reprocessMedia: false, source: 'keep' } }), + ], + }); + expect(deliveries.map((d) => d.filePath)).toEqual([ + sourcePath, + store.objectPathFor(sha256Of(DEGRADED_BYTES), '.jpg'), + ]); + }); + + it('reuses a degradation-cache hit without invoking the tool', async () => { + const degradedSha = sha256Of(DEGRADED_BYTES); + const objectPath = store.objectPathFor(degradedSha, '.jpg'); + await fs.mkdir(path.dirname(objectPath), { recursive: true }); + await fs.writeFile(objectPath, DEGRADED_BYTES); + const cache = new OmniDegradationCache(store.getOmniRootDir()); + await cache.put( + sha256Of(SOURCE_BYTES), + computePolicyFingerprint('omni_downsample_image', { maxDimension: 1568 }), + { + degradedSha256: degradedSha, + extension: '.jpg', + disclosure: 'cached disclosure', + mimeType: 'image/jpeg', + }, + ); + + const { deliveries, records } = await runFixedPolicies(config, source, { + store, + policies: [makePolicy()], + }); + expect(executeToolCallMock).not.toHaveBeenCalled(); + expect(deliveries).toEqual([ + { + filePath: objectPath, + recognized: DEGRADED_RECOGNIZED, + sha256: degradedSha, + disclosure: 'cached disclosure', + degraded: true, + }, + ]); + expect(records[0]).toMatchObject({ outcome: 'cache_hit' }); + }); + + it('drops a stale cache entry (object missing) and re-executes', async () => { + const staleSha = sha256Of('stale-derivative'); + const cache = new OmniDegradationCache(store.getOmniRootDir()); + const fingerprint = computePolicyFingerprint('omni_downsample_image', { + maxDimension: 1568, + }); + await cache.put(sha256Of(SOURCE_BYTES), fingerprint, { + degradedSha256: staleSha, + extension: '.jpg', + disclosure: 'stale disclosure', + mimeType: 'image/jpeg', + }); + mockToolSuccess(); + + const { deliveries, records } = await runFixedPolicies(config, source, { + store, + policies: [makePolicy()], + }); + expect(executeToolCallMock).toHaveBeenCalledTimes(1); + expect(records[0]).toMatchObject({ outcome: 'succeeded' }); + expect(deliveries[0].sha256).toBe(sha256Of(DEGRADED_BYTES)); + // The stale entry was replaced by the fresh derivative's identity. + const entry = await cache.get(sha256Of(SOURCE_BYTES), fingerprint); + expect(entry?.degradedSha256).toBe(sha256Of(DEGRADED_BYTES)); + }); + + it('treats a hash-identical output as a no-op: source delivered, nothing cached', async () => { + mockToolSuccess({ bytes: SOURCE_BYTES }); + // Identical bytes hash identically even though the mock labels the + // artifact image/jpeg — the fixed-point check runs on content hashes. + const { deliveries, records } = await runFixedPolicies(config, source, { + store, + policies: [makePolicy()], // source: 'omit' must NOT apply on no-op + }); + expect(records[0]).toMatchObject({ outcome: 'no_op' }); + expect(deliveries).toHaveLength(1); + expect(deliveries[0].filePath).toBe(sourcePath); + const cache = new OmniDegradationCache(store.getOmniRootDir()); + await expect( + cache.get( + sha256Of(SOURCE_BYTES), + computePolicyFingerprint('omni_downsample_image', { + maxDimension: 1568, + }), + ), + ).resolves.toBeNull(); + }); + + it('silently skips a policy whose `when` does not match', async () => { + const { deliveries, records } = await runFixedPolicies(config, source, { + store, + policies: [ + makePolicy({ + when: { + left: { field: 'resource.width' }, + operator: 'gt', + right: { value: 5000 }, + }, + }), + ], + }); + expect(records).toEqual([]); + expect(deliveries[0].filePath).toBe(sourcePath); + expect(executeToolCallMock).not.toHaveBeenCalled(); + }); + + it('records condition_unavailable with the missing fields when skipping', async () => { + const { records } = await runFixedPolicies(config, source, { + store, + policies: [ + makePolicy({ + when: { + left: { field: 'resource.durationMs' }, + operator: 'gt', + right: { value: 1000 }, + }, + }), + ], + }); + expect(records).toEqual([ + { + policyId: 'img-downsample', + toolName: 'omni_downsample_image', + outcome: 'condition_unavailable', + resource: 'photo.png', + missingFields: ['resource.durationMs'], + }, + ]); + expect(executeToolCallMock).not.toHaveBeenCalled(); + }); + + it('runs anyway on an undecidable condition when onConditionUnavailable is run', async () => { + mockToolSuccess(); + const { records } = await runFixedPolicies(config, source, { + store, + policies: [ + makePolicy({ + onConditionUnavailable: 'run', + when: { + left: { field: 'resource.durationMs' }, + operator: 'gt', + right: { value: 1000 }, + }, + }), + ], + }); + expect(executeToolCallMock).toHaveBeenCalledTimes(1); + expect(records[0]).toMatchObject({ outcome: 'succeeded' }); + }); + + it('caps re-derivation per lineage via maxRunsPerLineage', async () => { + mockToolSuccess(); + const { deliveries } = await runFixedPolicies(config, source, { + store, + policies: [ + makePolicy({ + origins: ['user', 'tool', 'policy'], + maxRunsPerLineage: 1, + output: { reprocessMedia: true, source: 'omit' }, + }), + ], + }); + // The derivative re-enters matching but the lineage already spent the + // policy's single run — exactly one execution, derivative delivered. + expect(executeToolCallMock).toHaveBeenCalledTimes(1); + expect(deliveries).toHaveLength(1); + expect(deliveries[0].degraded).toBe(true); + }); + + it('keeps the source and continues on failure when onFailure is continue', async () => { + executeToolCallMock.mockResolvedValue({ + callId: 'x', + responseParts: [], + resultDisplay: undefined, + error: new Error('ffmpeg exploded'), + errorType: undefined, + }); + const { deliveries, records } = await runFixedPolicies(config, source, { + store, + policies: [makePolicy()], + }); + expect(deliveries).toHaveLength(1); + expect(deliveries[0].filePath).toBe(sourcePath); + expect(records).toEqual([ + { + policyId: 'img-downsample', + toolName: 'omni_downsample_image', + outcome: 'failed', + resource: 'photo.png', + error: 'ffmpeg exploded', + }, + ]); + // Failure never leaves partial staging state (D10 Stage A). + await expect(fs.readdir(store.getStagingDir())).resolves.toEqual([]); + }); + + it('throws OmniPolicyExecutionError when onFailure is abort', async () => { + executeToolCallMock.mockResolvedValue({ + callId: 'x', + responseParts: [], + resultDisplay: undefined, + error: new Error('ffmpeg exploded'), + errorType: undefined, + }); + await expect( + runFixedPolicies(config, source, { + store, + policies: [makePolicy({ onFailure: 'abort' })], + }), + ).rejects.toMatchObject({ + name: 'OmniPolicyExecutionError', + policyId: 'img-downsample', + message: 'Fixed policy img-downsample failed: ffmpeg exploded', + }); + }); + + it('fails closed on transport_guard-stage failures regardless of onFailure', async () => { + executeToolCallMock.mockResolvedValue({ + callId: 'x', + responseParts: [], + resultDisplay: undefined, + error: new Error('guard tool crashed'), + errorType: undefined, + }); + await expect( + runFixedPolicies(config, source, { + store, + policies: [ + makePolicy({ onFailure: 'continue', stage: 'transport_guard' }), + ], + }), + ).rejects.toBeInstanceOf(OmniPolicyExecutionError); + }); + + it('rejects an artifact whose workspacePath escapes the staging directory', async () => { + const evilPath = path.join(tmpDir, 'evil.jpg'); + executeToolCallMock.mockImplementation( + async (_config: Config, request: ToolCallRequestInfo) => { + await fs.writeFile(evilPath, DEGRADED_BYTES); + return { + callId: request.callId, + responseParts: [], + resultDisplay: undefined, + error: undefined, + errorType: undefined, + policyArtifacts: { + toolName: request.name, + invocationId: request.callId, + executionOrigin: request.executionOrigin, + artifacts: [ + { + kind: 'image', + storage: 'workspace', + title: 'evil.jpg', + workspacePath: path.relative( + path.join(store.getStagingDir(), request.callId), + evilPath, + ), + metadata: { omniDisclosure: 'x' }, + }, + ], + }, + }; + }, + ); + const { deliveries, records } = await runFixedPolicies(config, source, { + store, + policies: [makePolicy()], + }); + expect(records[0]).toMatchObject({ outcome: 'failed' }); + expect(records[0].error).toContain('escapes the staging directory'); + expect(deliveries[0].filePath).toBe(sourcePath); + }); + + it('rejects a lossy artifact that carries no omniDisclosure', async () => { + mockToolSuccess({ disclosure: undefined }); + const { records } = await runFixedPolicies(config, source, { + store, + policies: [makePolicy()], + }); + expect(records[0]).toMatchObject({ outcome: 'failed' }); + expect(records[0].error).toContain('lossy but carries no omniDisclosure'); + }); + + it('rejects a tool without a media-policy descriptor', async () => { + const { records } = await runFixedPolicies(makeConfig({}), source, { + store, + policies: [makePolicy()], + }); + expect(records[0]).toMatchObject({ outcome: 'failed' }); + expect(records[0].error).toContain('not a registered media-policy tool'); + }); + + it('executes policies in priority order, ties broken by id', async () => { + mockToolSuccess(); + // Distinct arguments per policy: identical arguments would fingerprint + // identically and turn the later runs into degradation-cache hits. + await runFixedPolicies(config, source, { + store, + policies: [ + makePolicy({ + id: 'b-low', + priority: 1, + arguments: { maxDimension: 100 }, + output: { reprocessMedia: false, source: 'keep' }, + }), + makePolicy({ + id: 'a-high', + priority: 10, + arguments: { maxDimension: 300 }, + output: { reprocessMedia: false, source: 'keep' }, + }), + makePolicy({ + id: 'a-low', + priority: 1, + arguments: { maxDimension: 200 }, + output: { reprocessMedia: false, source: 'keep' }, + }), + ], + }); + const order = executeToolCallMock.mock.calls.map((call) => { + const origin = (call[1] as ToolCallRequestInfo).executionOrigin; + return origin?.kind === 'fixed_policy' ? origin.policyId : undefined; + }); + expect(order).toEqual(['a-high', 'a-low', 'b-low']); + }); +}); diff --git a/packages/core/src/omni/policy/orchestrator.ts b/packages/core/src/omni/policy/orchestrator.ts new file mode 100644 index 00000000000..ccfc328c42c --- /dev/null +++ b/packages/core/src/omni/policy/orchestrator.ts @@ -0,0 +1,562 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { randomBytes } from 'node:crypto'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import type { Config } from '../../config/config.js'; +import type { ToolCallRequestInfo } from '../../core/turn.js'; +import type { + MediaPolicyToolDescriptor, + ToolArtifact, +} from '../../tools/tools.js'; +import { createDebugLogger } from '../../utils/debugLogger.js'; +import { estimateRawResourceTokens } from '../estimation.js'; +import { + extensionForMime, + hashFileSha256, + recognizeMediaFile, + type RecognizedMedia, +} from '../recognition.js'; +import type { OmniObjectStore } from '../storage.js'; +import { + evaluateFixedPolicyCondition, + type FixedPolicyConditionContext, + type ResourceConditionField, +} from './conditions.js'; +import { + computePolicyFingerprint, + OmniDegradationCache, +} from './degradation-cache.js'; +import type { FixedPolicyOrigin, NormalizedFixedPolicy } from './types.js'; + +const debugLogger = createDebugLogger('omni:policy'); + +/** One resource in the final delivery set produced by the orchestrator. */ +export interface PolicyDeliveryResource { + /** Absolute path of the deliverable file (the original input, or a + * promoted derivative inside `objects/`). */ + filePath: string; + recognized: RecognizedMedia; + /** Content hash when already known (always set for derivatives; set on + * the source only if a policy run had to hash it). */ + sha256?: string; + /** Disclosure that must accompany the resource (lossy derivatives). */ + disclosure?: string; + /** True when the resource is a lossy derivative of the user's input. */ + degraded?: boolean; +} + +/** Debug/telemetry record of one policy decision that did real work (or + * failed to). Pure matching misses are deliberately unrecorded — with the + * system defaults active on every delivery, zero-policy runs must stay + * zero-noise. */ +export interface PolicyRunRecord { + policyId: string; + toolName: string; + outcome: + | 'succeeded' + | 'cache_hit' + | 'no_op' + | 'failed' + | 'condition_unavailable'; + /** Display label of the resource the policy ran against. */ + resource: string; + /** Fields that made a `when` condition undecidable. */ + missingFields?: string[]; + error?: string; +} + +export interface RunFixedPoliciesOptions { + store: OmniObjectStore; + policies: NormalizedFixedPolicy[]; + signal?: AbortSignal; + /** Request/session condition namespaces, when the caller has them. The + * resource namespace is always derived from each item's recognition. */ + conditionContext?: Pick; + /** Injectable for tests; defaults to the store-rooted cache. */ + degradationCache?: OmniDegradationCache; +} + +/** Root resource entering the orchestrator. */ +export interface PolicySourceResource { + filePath: string; + recognized: RecognizedMedia; + /** User-recognizable name for records and error messages. */ + displayName: string; + origin: Extract; +} + +/** Thrown when a policy invocation fails and the failure must abort the + * delivery (`onFailure: 'abort'`, or any transport-guard policy). */ +export class OmniPolicyExecutionError extends Error { + constructor( + message: string, + readonly policyId: string, + options?: { cause?: unknown }, + ) { + super(message, options); + this.name = 'OmniPolicyExecutionError'; + } +} + +/** Work-queue item: a resource that may still be matched by policies. */ +interface WorkItem { + filePath: string; + recognized: RecognizedMedia; + label: string; + origin: FixedPolicyOrigin; + sha256?: string; + disclosure?: string; + degraded?: boolean; + /** Per-derivation-chain run counts (policy id → runs). Copied — never + * shared — on derivation, so sibling branches cap independently. */ + lineageRuns: Map; + deliver: boolean; + /** Whether the item enters policy matching (`output.reprocessMedia`). */ + process: boolean; +} + +/** Result of one actual policy execution. */ +interface PolicyExecution { + outcome: 'succeeded' | 'cache_hit' | 'no_op'; + derived: Array<{ + filePath: string; + recognized: RecognizedMedia; + sha256: string; + disclosure?: string; + degraded: boolean; + }>; +} + +function resourceConditionContext( + recognized: RecognizedMedia, + shared?: Pick, +): FixedPolicyConditionContext { + const resource: Partial> = {}; + const set = ( + field: ResourceConditionField, + value: number | undefined, + ): void => { + if (typeof value === 'number' && !Number.isNaN(value)) { + resource[field] = value; + } + }; + const m = recognized.metadata; + set('sizeBytes', recognized.sizeBytes); + set('durationMs', m.durationMs); + set('width', m.width); + set('height', m.height); + // The probe reports the (single) primary stream's dimensions, so the + // max* aliases resolve to the same values. + set('maxWidth', m.width); + set('maxHeight', m.height); + set('frameRate', m.frameRate); + set('frameCount', m.frameCount); + set('bitRate', m.bitRate); + set('sampleRateHz', m.sampleRateHz); + set('channels', m.channels); + const estimate = estimateRawResourceTokens(recognized); + if (estimate.status === 'ok') { + set('estimatedTokenCount', estimate.estimatedTokenCount); + } + return { resource, ...shared }; +} + +/** Deterministic execution order: priority descending, id ascending. */ +function sortPolicies( + policies: NormalizedFixedPolicy[], +): NormalizedFixedPolicy[] { + return [...policies].sort( + (a, b) => b.priority - a.priority || a.id.localeCompare(b.id), + ); +} + +/** + * Run the fixed-policy pipeline over one recognized media resource + * (decisions D1/D3/D5): match each policy in priority order, execute the + * matched media-policy tool through the ordinary scheduler path inside an + * exclusive staging directory, validate the artifacts against the tool's + * descriptor, promote them into the content-addressed store, and return + * the final delivery set plus records of the work performed. + * + * Termination is structural: each policy runs at most `maxRunsPerLineage` + * times per derivation chain and the policy set is finite, so the derived + * tree is finite (global budgets are the next commit's backstop). + * + * Failure semantics (decision D10): a failed invocation never leaves + * partial state (its staging dir is removed); `onFailure: 'continue'` + * keeps the source in the delivery set (the transport guard remains the + * backstop), while `'abort'` — and any transport-guard-stage failure — + * throws {@link OmniPolicyExecutionError}. + */ +export async function runFixedPolicies( + config: Config, + source: PolicySourceResource, + options: RunFixedPoliciesOptions, +): Promise<{ + deliveries: PolicyDeliveryResource[]; + records: PolicyRunRecord[]; +}> { + const policies = sortPolicies(options.policies); + const cache = + options.degradationCache ?? + new OmniDegradationCache(options.store.getOmniRootDir()); + const records: PolicyRunRecord[] = []; + const items: WorkItem[] = [ + { + filePath: source.filePath, + recognized: source.recognized, + label: source.displayName, + origin: source.origin, + lineageRuns: new Map(), + deliver: true, + process: true, + }, + ]; + + // Index-based: executions append derived items behind the cursor. + for (let i = 0; i < items.length; i++) { + const item = items[i]; + if (!item.process) continue; + for (const policy of policies) { + if (!policy.mediaTypes.includes(item.recognized.modality)) continue; + if (!policy.origins.includes(item.origin)) continue; + const runs = item.lineageRuns.get(policy.id) ?? 0; + if (runs >= policy.maxRunsPerLineage) continue; + if (policy.when) { + const evaluation = evaluateFixedPolicyCondition( + policy.when, + resourceConditionContext(item.recognized, options.conditionContext), + ); + if (evaluation.outcome === 'no_match') continue; + if ( + evaluation.outcome === 'unavailable' && + policy.onConditionUnavailable === 'skip' + ) { + records.push({ + policyId: policy.id, + toolName: policy.toolName, + outcome: 'condition_unavailable', + resource: item.label, + missingFields: evaluation.missingFields, + }); + continue; + } + // 'unavailable' + onConditionUnavailable 'run' falls through. + } + item.lineageRuns.set(policy.id, runs + 1); + try { + const execution = await executePolicy( + config, + item, + policy, + options.store, + cache, + options.signal, + ); + records.push({ + policyId: policy.id, + toolName: policy.toolName, + outcome: execution.outcome, + resource: item.label, + }); + if (execution.outcome === 'no_op') continue; + if (policy.output.source === 'omit') item.deliver = false; + for (const derived of execution.derived) { + items.push({ + ...derived, + label: `${item.label} → ${policy.id}`, + origin: 'policy', + lineageRuns: new Map(item.lineageRuns), + deliver: true, + process: policy.output.reprocessMedia, + }); + } + } catch (err) { + if (options.signal?.aborted) throw err; + const message = err instanceof Error ? err.message : String(err); + records.push({ + policyId: policy.id, + toolName: policy.toolName, + outcome: 'failed', + resource: item.label, + error: message, + }); + debugLogger.debug( + `fixed policy ${policy.id} (${policy.toolName}) failed on ${item.label}: ${message}`, + ); + if ( + policy.onFailure === 'abort' || + policy.stage === 'transport_guard' + ) { + throw new OmniPolicyExecutionError( + `Fixed policy ${policy.id} failed: ${message}`, + policy.id, + { cause: err }, + ); + } + // 'continue': the source stays in the delivery set; the transport + // guard remains the backstop for oversized content. + } + } + } + + return { + deliveries: items + .filter((item) => item.deliver) + .map((item) => ({ + filePath: item.filePath, + recognized: item.recognized, + sha256: item.sha256, + disclosure: item.disclosure, + degraded: item.degraded, + })), + records, + }; +} + +/** Validated view of one artifact after descriptor/staging checks. */ +interface ValidatedArtifact { + absolutePath: string; + recognized: RecognizedMedia; + sha256: string; + disclosure?: string; + lossy: boolean; +} + +/** + * Execute one policy against one work item: degradation-cache lookup, + * otherwise a real tool invocation in a fresh staging directory followed + * by artifact validation and promotion (staging lifecycle §5, order D12: + * promote first, then substitute, then delete staging). + */ +async function executePolicy( + config: Config, + item: WorkItem, + policy: NormalizedFixedPolicy, + store: OmniObjectStore, + cache: OmniDegradationCache, + signal: AbortSignal | undefined, +): Promise { + const tool = config.getToolRegistry().getTool(policy.toolName); + const descriptor = tool?.mediaPolicyDescriptor; + if (!descriptor) { + throw new Error( + `tool ${policy.toolName} is not a registered media-policy tool`, + ); + } + + // The source hash keys the degradation cache; computed lazily so runs + // without matching policies never pay it. + item.sha256 ??= await hashFileSha256(item.filePath, signal); + const fingerprint = computePolicyFingerprint( + policy.toolName, + policy.arguments, + ); + const hit = await cache.get(item.sha256, fingerprint); + if (hit) { + const objectPath = store.objectPathFor(hit.degradedSha256, hit.extension); + const stat = await fs.lstat(objectPath).catch(() => undefined); + if (stat?.isFile() && !stat.isSymbolicLink()) { + const recognized = await recognizeMediaFile(objectPath, { signal }); + debugLogger.debug( + `degradation cache hit: policy=${policy.id} sha256=${item.sha256.slice(0, 12)}…`, + ); + return { + outcome: 'cache_hit', + derived: [ + { + filePath: objectPath, + recognized, + sha256: hit.degradedSha256, + disclosure: hit.disclosure, + degraded: true, + }, + ], + }; + } + // Stale: the derivative left the store (GC, manual deletion). Drop + // every entry pointing at it and re-transcode. + await cache.removeByDegradedSha256(hit.degradedSha256); + } + + const invocationId = randomBytes(8).toString('hex'); + const stagingDir = await store.createStagingDir(invocationId); + try { + const request: ToolCallRequestInfo = { + callId: invocationId, + name: policy.toolName, + args: { + ...policy.arguments, + inputPath: item.filePath, + outputDir: stagingDir, + }, + isClientInitiated: true, + prompt_id: `omni-fixed-policy-${invocationId}`, + executionOrigin: { + kind: 'fixed_policy', + policyId: policy.id, + stage: policy.stage, + }, + }; + // Dynamic import: the executor pulls in the scheduler, whose module + // graph reaches back into omni surfaces — the runtime dependency is + // resolved at call time (same pattern as the scheduler's tool-result + // funnel import) to keep module evaluation cycle-free. + const { executeToolCall } = await import( + '../../core/nonInteractiveToolExecutor.js' + ); + const response = await executeToolCall( + config, + request, + signal ?? new AbortController().signal, + { recordToolResult: false }, + ); + if (response.error) { + throw new Error(response.error.message, { cause: response.error }); + } + const batch = response.policyArtifacts; + if (!batch || batch.artifacts.length === 0) { + throw new Error( + `tool ${policy.toolName} succeeded but produced no policy artifacts`, + ); + } + + const validated: ValidatedArtifact[] = []; + for (const artifact of batch.artifacts) { + validated.push( + await validateArtifact(artifact, descriptor, stagingDir, signal), + ); + } + assertRequiredOutputsPresent(descriptor, validated, policy.toolName); + + // Fixed-point: identical output means this iteration changed nothing — + // deliver the source and stop deriving (no cache entry either; a no-op + // is a property of this input, re-derivable cheaply). + if (validated.every((a) => a.sha256 === item.sha256)) { + return { outcome: 'no_op', derived: [] }; + } + + // Promotion first (D12): once an artifact is in objects/ it is + // content-addressed and immutable; only then substitute + cache. + const derived: PolicyExecution['derived'] = []; + for (const artifact of validated) { + const extension = extensionForMime(artifact.recognized.detectedMimeType); + const put = await store.putFile( + artifact.absolutePath, + artifact.sha256, + extension, + signal, + ); + derived.push({ + filePath: put.objectPath, + recognized: artifact.recognized, + sha256: artifact.sha256, + disclosure: artifact.disclosure, + degraded: artifact.lossy, + }); + } + // The cache maps one input to ONE derivative; multi-output tools are + // simply not cached (re-run instead of guessing which output to key). + if (validated.length === 1 && validated[0].disclosure) { + await cache.put(item.sha256, fingerprint, { + degradedSha256: validated[0].sha256, + extension: extensionForMime(validated[0].recognized.detectedMimeType), + disclosure: validated[0].disclosure, + mimeType: validated[0].recognized.detectedMimeType, + }); + } + return { outcome: 'succeeded', derived }; + } finally { + // Success and failure both end without a staging dir (this commit's + // Stage A behavior; quarantine-on-failure is the Stage B follow-up). + await store.removeStagingDir(invocationId).catch(() => {}); + } +} + +/** + * Validate one artifact against the staging contract (§5) and the tool's + * descriptor (D8): workspace-storage with a path strictly inside the + * staging dir, a regular non-symlink file, recognized content matching a + * declared media output, and — for lossy outputs — a non-empty + * `metadata.omniDisclosure`. + */ +async function validateArtifact( + artifact: ToolArtifact, + descriptor: MediaPolicyToolDescriptor, + stagingDir: string, + signal: AbortSignal | undefined, +): Promise { + if (artifact.storage !== 'workspace' || !artifact.workspacePath) { + throw new Error( + `policy artifact "${artifact.title}" is not a workspace file`, + ); + } + const absolutePath = path.resolve(stagingDir, artifact.workspacePath); + if (!absolutePath.startsWith(stagingDir + path.sep)) { + throw new Error( + `policy artifact "${artifact.title}" escapes the staging directory`, + ); + } + const stat = await fs.lstat(absolutePath).catch(() => undefined); + if (!stat?.isFile() || stat.isSymbolicLink()) { + throw new Error( + `policy artifact "${artifact.title}" is missing or not a regular file`, + ); + } + // Authoritative recognition of the actual bytes — the tool's declared + // mimeType/kind are cross-checked, never trusted. + const recognized = await recognizeMediaFile(absolutePath, { signal }); + const spec = descriptor.outputs.find( + (o) => + o.kind === 'media' && o.mimeTypes?.includes(recognized.detectedMimeType), + ); + if (!spec) { + throw new Error( + `policy artifact "${artifact.title}" has undeclared media type ${recognized.detectedMimeType}`, + ); + } + if (artifact.kind !== recognized.modality) { + throw new Error( + `policy artifact "${artifact.title}" declares kind ${String(artifact.kind)} but contains ${recognized.modality} content`, + ); + } + const disclosure = artifact.metadata?.['omniDisclosure']; + if (spec.lossy && (typeof disclosure !== 'string' || disclosure === '')) { + throw new Error( + `policy artifact "${artifact.title}" is lossy but carries no omniDisclosure`, + ); + } + return { + absolutePath, + recognized, + sha256: await hashFileSha256(absolutePath, signal), + disclosure: + typeof disclosure === 'string' && disclosure ? disclosure : undefined, + lossy: spec.lossy === true, + }; +} + +/** Every required media output declared by the descriptor must have been + * produced (§5 completeness check). */ +function assertRequiredOutputsPresent( + descriptor: MediaPolicyToolDescriptor, + validated: ValidatedArtifact[], + toolName: string, +): void { + for (const spec of descriptor.outputs) { + if (spec.kind !== 'media' || !spec.required) continue; + const produced = validated.some((a) => + spec.mimeTypes?.includes(a.recognized.detectedMimeType), + ); + if (!produced) { + throw new Error( + `tool ${toolName} did not produce its required ${spec.mimeTypes?.join('/') ?? 'media'} output`, + ); + } + } +} diff --git a/packages/core/src/omni/policy/types.ts b/packages/core/src/omni/policy/types.ts index 8652c476353..0be21fb900c 100644 --- a/packages/core/src/omni/policy/types.ts +++ b/packages/core/src/omni/policy/types.ts @@ -32,6 +32,67 @@ export type { FixedPolicyField, } from './conditions.js'; +import type { FixedPolicyCondition } from './conditions.js'; +import type { OmniModality } from '../recognition.js'; + +/** Provenance labels a fixed policy can match on: `user` = user-attached + * input, `tool` = tool-result media, `policy` = a derivative produced by + * another fixed policy. */ +export type FixedPolicyOrigin = 'user' | 'tool' | 'policy'; + +/** + * One fixed policy AFTER config normalization (policy design §8): every + * field present, defaults applied, structure validated. The orchestrator + * consumes only this shape — raw settings never reach it. + */ +export interface NormalizedFixedPolicy { + /** Unique id (settings key). Ties run records, staging dirs and the + * `fixed_policy` execution origin back to their configuration. */ + id: string; + /** Bigger runs first; ties broken by id (ascending) for determinism. */ + priority: number; + /** Modalities the policy applies to. */ + mediaTypes: OmniModality[]; + /** Resource provenances the policy applies to. */ + origins: FixedPolicyOrigin[]; + /** Optional condition; absent means "always applies". */ + when?: FixedPolicyCondition; + /** What to do when `when` cannot be decided (default: skip). */ + onConditionUnavailable: 'skip' | 'run'; + /** Media-policy tool the policy invokes. */ + toolName: string; + /** Fixed tool arguments (io params are injected per invocation). */ + arguments: Record; + /** Max executions of THIS policy along one derivation chain. */ + maxRunsPerLineage: number; + /** Failure behavior: keep the source in the delivery set and move on, + * or abort the whole media delivery. */ + onFailure: 'continue' | 'abort'; + output: { + /** Whether derivatives re-enter policy matching. */ + reprocessMedia: boolean; + /** Whether the source stays in the delivery set alongside the + * derivatives (`keep`) or is replaced by them (`omit`). */ + source: 'keep' | 'omit'; + }; + /** Pipeline stage the policy runs in. Transport-guard policies fail + * closed regardless of `onFailure`. */ + stage: 'preprocessing' | 'transport_guard'; +} + +/** Normalized `omni.processing` view the pipeline consumes. */ +export interface NormalizedOmniProcessingConfig { + fixedPolicies: NormalizedFixedPolicy[]; + transportGuardPolicies: NormalizedFixedPolicy[]; +} + +/** Structural Config view for the processing config accessor (optional so + * stub configs and embedders without omni settings keep working; the real + * accessor lands with config normalization). */ +export interface OmniProcessingConfigView { + getOmniProcessingConfig?: () => NormalizedOmniProcessingConfig | undefined; +} + /** * Raw (pre-normalization) shape of one * `omni.processing.policyTools.` settings entry. Full semantic diff --git a/packages/core/src/omni/tool-result-media.test.ts b/packages/core/src/omni/tool-result-media.test.ts index 98c3559b7ed..251d6624a38 100644 --- a/packages/core/src/omni/tool-result-media.test.ts +++ b/packages/core/src/omni/tool-result-media.test.ts @@ -233,6 +233,86 @@ describe('processToolResultOmniMedia', () => { expect(deliverMock).not.toHaveBeenCalled(); }); + it('emits the degradation disclosure text immediately before the fileData part', async () => { + deliverMock.mockResolvedValue({ + fileUri: 'oss://bucket/degraded', + mimeType: 'image/jpeg', + sha256: 'b'.repeat(64), + recognized: { modality: 'image' }, + tokenEstimate: { + estimatedTokenCount: 1, + method: 'raw-resource-v1', + status: 'ok', + }, + deduped: false, + disclosure: 'downsampled to 1568px', + degraded: true, + }); + const parts = [inlinePart('image/png', PNG_BYTES)]; + const result = await processToolResultOmniMedia( + parts, + cfg({ image: true }), + signal, + ); + expect(result).toHaveLength(2); + expect(result[0]!.text).toBe( + '【媒体降质】tool-media.image：downsampled to 1568px', + ); + expect(result[1]!.fileData?.fileUri).toBe('oss://bucket/degraded'); + // The pipeline was told this media came from a tool (policy origins). + expect(deliverMock).toHaveBeenCalledWith( + expect.any(String), + expect.anything(), + expect.objectContaining({ + origin: 'tool', + displayName: 'tool-media.image', + expectedModality: 'image', + }), + ); + }); + + it('expands a disclosed delivery inside functionResponse.parts', async () => { + deliverMock.mockResolvedValue({ + fileUri: 'oss://bucket/degraded', + mimeType: 'image/jpeg', + sha256: 'b'.repeat(64), + recognized: { modality: 'image' }, + tokenEstimate: { + estimatedTokenCount: 1, + method: 'raw-resource-v1', + status: 'ok', + }, + deduped: false, + disclosure: 'downsampled to 1568px', + degraded: true, + }); + const parts: Part[] = [ + { + functionResponse: { + id: 'call_1', + name: 'Read', + response: { output: 'ok' }, + parts: [ + { text: 'caption' }, + inlinePart('image/png', PNG_BYTES), + ] as Part[], + }, + } as Part, + ]; + const result = await processToolResultOmniMedia( + parts, + cfg({ image: true }), + signal, + ); + const nested = result[0]!.functionResponse?.parts as Part[]; + expect(nested).toHaveLength(3); + expect(nested[0]!.text).toBe('caption'); + expect(nested[1]!.text).toBe( + '【媒体降质】tool-media.image：downsampled to 1568px', + ); + expect(nested[2]!.fileData?.fileUri).toBe('oss://bucket/degraded'); + }); + it('converts media nested inside functionResponse.parts (the production funnel shape)', async () => { // Both physical funnels deliver tool-result media wrapped by // convertToFunctionResponse as {functionResponse: {…, parts: diff --git a/packages/core/src/omni/tool-result-media.ts b/packages/core/src/omni/tool-result-media.ts index 0b15915a3a0..185e778c981 100644 --- a/packages/core/src/omni/tool-result-media.ts +++ b/packages/core/src/omni/tool-result-media.ts @@ -11,6 +11,7 @@ import type { Part } from '@google/genai'; import type { Config } from '../config/config.js'; import { createDebugLogger } from '../utils/debugLogger.js'; import { isOmniDeliveryActive, processMediaForOmniDelivery } from './index.js'; +import { formatDisclosureText } from './disclosure.js'; import { OmniTransportGuardError } from './guard.js'; import { OmniObjectStore } from './storage.js'; import { sniffMediaType } from './recognition.js'; @@ -61,11 +62,15 @@ export async function processToolResultOmniMedia( let uploadsRemaining = MAX_UPLOADS_PER_TOOL_RESULT; let uploadBytesRemaining = MAX_UPLOAD_BYTES_PER_TOOL_RESULT; - const convertPart = async (part: Part): Promise => { + /** Returns the replacement Parts for one Part: `[part]` (unchanged), + * `[fileData]`, or `[disclosureText, fileData]` when a fixed policy + * degraded the media — the disclosure must sit IMMEDIATELY before its + * media part (decision D8) so converters can move the pair together. */ + const convertPart = async (part: Part): Promise => { const inline = part.inlineData; - if (!inline?.data || !inline.mimeType) return part; + if (!inline?.data || !inline.mimeType) return [part]; const top = inline.mimeType.split('/')[0]; - if (top !== 'image' && top !== 'audio' && top !== 'video') return part; + if (top !== 'image' && top !== 'audio' && top !== 'video') return [part]; // Sniff the decoded bytes before touching disk — non-media or // unsupported containers stay inline untouched. The SNIFFED modality @@ -74,13 +79,13 @@ export async function processToolResultOmniMedia( // config on the strength of its declared MIME type. const bytes = Buffer.from(inline.data, 'base64'); const sniffed = sniffMediaType(bytes.subarray(0, 4096)); - if (!sniffed) return part; - if (!modalities[sniffed.modality]) return part; + if (!sniffed) return [part]; + if (!modalities[sniffed.modality]) return [part]; if (uploadsRemaining <= 0 || bytes.length > uploadBytesRemaining) { debugLogger.debug( `tool-result media budget exhausted; keeping part inline (${bytes.length} bytes)`, ); - return part; + return [part]; } // Everything from staging-dir setup onward sits inside the try: mkdir @@ -97,20 +102,29 @@ export async function processToolResultOmniMedia( try { await fs.mkdir(stagingDir, { recursive: true, mode: 0o700 }); await fs.writeFile(tempPath, bytes, { mode: 0o600 }); + const displayName = inline.displayName ?? `tool-media.${top}`; const delivery = await processMediaForOmniDelivery(tempPath, config, { expectedModality: sniffed.modality, signal, + displayName, + origin: 'tool', }); changed = true; uploadsRemaining--; uploadBytesRemaining -= bytes.length; - return { + const fileDataPart: Part = { fileData: { fileUri: delivery.fileUri, mimeType: delivery.mimeType, - displayName: inline.displayName ?? `tool-media.${top}`, + displayName, }, }; + return delivery.disclosure + ? [ + { text: formatDisclosureText(displayName, delivery.disclosure) }, + fileDataPart, + ] + : [fileDataPart]; } catch (err) { if (signal.aborted) throw err; if (err instanceof OmniTransportGuardError) { @@ -121,16 +135,18 @@ export async function processToolResultOmniMedia( // rationale ("produced locally, already in memory") covers only // failures of the *transfer*. changed = true; - return { - text: `[Tool media part withheld by the omni transport guard: ${err.message}]`, - }; + return [ + { + text: `[Tool media part withheld by the omni transport guard: ${err.message}]`, + }, + ]; } debugLogger.debug( `tool-result media upload failed, keeping inline: ${ err instanceof Error ? err.message : String(err) }`, ); - return part; + return [part]; } finally { await fs.rm(tempPath, { force: true }).catch(() => {}); } @@ -144,8 +160,10 @@ export async function processToolResultOmniMedia( let nestedChanged = false; for (const nestedPart of nested as Part[]) { const converted = await convertPart(nestedPart); - if (converted !== nestedPart) nestedChanged = true; - convertedNested.push(converted); + if (converted.length !== 1 || converted[0] !== nestedPart) { + nestedChanged = true; + } + convertedNested.push(...converted); } if (nestedChanged) { result.push({ @@ -161,7 +179,7 @@ export async function processToolResultOmniMedia( } continue; } - result.push(await convertPart(part)); + result.push(...(await convertPart(part))); } return changed ? result : responseParts; From dc434818a01a90a4fd86b76ad5131131dce3cfb2 Mon Sep 17 00:00:00 2001 From: LaZzyMan Date: Fri, 7 Aug 2026 11:25:57 +0800 Subject: [PATCH 09/62] feat(omni): normalize omni.processing config with system default policies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Startup normalization of the fixed-policy pipeline configuration (policy design §13 applicable subset): - omni/policy/config.ts: normalizeOmniProcessingConfig merges user settings over system defaults (id-merge, whole-entry replacement, null tombstones for fixedPolicies only) and validates structure, enums, when-conditions, tool references (registered + media_policy descriptor + required/lossy-disclosure outputs), fixed arguments against the io-stripped settingsSchema, reserved io keys, guard rules (no when, source=omit, mandatory three-modality coverage), limits (§12.2 defaults), the 1 GiB upload cap and the 48h URL TTL. Any violation throws OmniPolicyConfigError and aborts startup — a mis-configured guard must never degrade into sending over-limit media. - System defaults (D7 dual registration): the three degradation tools registered as preprocessing fixedPolicies WITH when-thresholds and as transportGuard.policies WITHOUT when. - core Config: thread fixedPolicies / transportGuard.policies / limits / quarantine settings; normalize in initialize() after tool warmup; expose getOmniProcessingConfig and quarantine getters. - cli: thread the new omni.processing / omni.storage.quarantine keys. --- packages/cli/src/config/config.ts | 11 + packages/core/src/config/config.ts | 64 +- packages/core/src/omni/policy/config.test.ts | 787 +++++++++++++++++++ packages/core/src/omni/policy/config.ts | 723 +++++++++++++++++ packages/core/src/omni/policy/types.ts | 22 + 5 files changed, 1606 insertions(+), 1 deletion(-) create mode 100644 packages/core/src/omni/policy/config.test.ts create mode 100644 packages/core/src/omni/policy/config.ts diff --git a/packages/cli/src/config/config.ts b/packages/cli/src/config/config.ts index 7863c3c9203..90a1f280190 100755 --- a/packages/cli/src/config/config.ts +++ b/packages/cli/src/config/config.ts @@ -2203,6 +2203,17 @@ export async function loadCliConfig( omniPolicyTools: settings.omni?.processing?.policyTools as | OmniPolicyToolsSettings | undefined, + omniFixedPolicies: settings.omni?.processing?.fixedPolicies as + | Record + | undefined, + omniTransportGuardPolicies: settings.omni?.processing?.transportGuard + ?.policies as Record | undefined, + omniProcessingLimits: settings.omni?.processing?.limits as + | Record + | undefined, + omniQuarantineRetentionDays: + settings.omni?.storage?.quarantine?.retentionDays, + omniQuarantineMaxBytes: settings.omni?.storage?.quarantine?.maxBytes, // CDP tunnel (Plan C, #5626): with the tunnel on, browser automation goes // through the CDP tunnel (far lighter than the OS-level computer-use // driver), so disable computer-use to keep the agent off that heavy path. diff --git a/packages/core/src/config/config.ts b/packages/core/src/config/config.ts index 903a55beea7..7f2eeec585b 100644 --- a/packages/core/src/config/config.ts +++ b/packages/core/src/config/config.ts @@ -28,7 +28,10 @@ import { selectVisionBridgeModel, } from '../services/visionBridge/vision-bridge-service.js'; import type { AnyToolInvocation } from '../tools/tools.js'; -import type { OmniPolicyToolsSettings } from '../omni/policy/types.js'; +import type { + NormalizedOmniProcessingConfig, + OmniPolicyToolsSettings, +} from '../omni/policy/types.js'; import type { ArenaManager } from '../agents/arena/ArenaManager.js'; import { ArenaAgentClient } from '../agents/arena/ArenaAgentClient.js'; import type { TeamManager } from '../agents/team/TeamManager.js'; @@ -1114,6 +1117,17 @@ export interface ConfigParameters { /** Raw `omni.processing.policyTools` map (per-tool settings/runtime/ * modelAccess). Normalized lazily by the omni policy modules. */ omniPolicyTools?: OmniPolicyToolsSettings; + /** Raw `omni.processing.fixedPolicies` map (id → policy | null + * tombstone). Normalized at startup against system defaults. */ + omniFixedPolicies?: Record; + /** Raw `omni.processing.transportGuard.policies` map. */ + omniTransportGuardPolicies?: Record; + /** Raw `omni.processing.limits` per-root derivation budgets. */ + omniProcessingLimits?: Record; + /** `omni.storage.quarantine.retentionDays` (default 7). */ + omniQuarantineRetentionDays?: number; + /** `omni.storage.quarantine.maxBytes` (default 5 GiB). */ + omniQuarantineMaxBytes?: number; /** Image generation model selected through `/model --image`. */ imageModel?: string; /** @@ -1945,6 +1959,14 @@ export class Config { private readonly omniUrlDownloadMaxFileBytes?: number; private readonly omniUploadUrlTtlHours?: number; private readonly omniPolicyTools?: OmniPolicyToolsSettings; + private readonly omniFixedPolicies?: Record; + private readonly omniTransportGuardPolicies?: Record; + private readonly omniProcessingLimits?: Record; + private readonly omniQuarantineRetentionDays?: number; + private readonly omniQuarantineMaxBytes?: number; + /** Normalized `omni.processing` view; set once during initialize() + * (after the tool registry exists) when omni is enabled. */ + private omniProcessingConfig?: NormalizedOmniProcessingConfig; private workflowsEnabled = false; private readonly skipWorkflowUsageWarning: boolean = false; private readonly computerUseEnabled: boolean = true; @@ -2226,6 +2248,11 @@ export class Config { this.omniUrlDownloadMaxFileBytes = params.omniUrlDownloadMaxFileBytes; this.omniUploadUrlTtlHours = params.omniUploadUrlTtlHours; this.omniPolicyTools = params.omniPolicyTools; + this.omniFixedPolicies = params.omniFixedPolicies; + this.omniTransportGuardPolicies = params.omniTransportGuardPolicies; + this.omniProcessingLimits = params.omniProcessingLimits; + this.omniQuarantineRetentionDays = params.omniQuarantineRetentionDays; + this.omniQuarantineMaxBytes = params.omniQuarantineMaxBytes; this.workflowsEnabled = params.workflowsEnabled ?? false; this.skipWorkflowUsageWarning = params.skipWorkflowUsageWarning ?? false; this.computerUseEnabled = params.computerUseEnabled ?? true; @@ -2959,6 +2986,27 @@ export class Config { }); recordStartupEvent('config_initialize_tool_warmup_end'); + // Normalize the omni fixed-policy configuration now that the tool + // registry can resolve policy-tool references. A violation throws + // OmniPolicyConfigError and aborts startup — a mis-configured + // transport guard must never degrade into sending over-limit media. + if (this.isOmniEnabled()) { + const { normalizeOmniProcessingConfig } = await import( + '../omni/policy/config.js' + ); + this.omniProcessingConfig = normalizeOmniProcessingConfig( + { + fixedPolicies: this.omniFixedPolicies, + transportGuardPolicies: this.omniTransportGuardPolicies, + limits: this.omniProcessingLimits, + policyTools: this.omniPolicyTools, + maxUploadFileBytes: this.omniMaxUploadFileBytes, + urlTtlHours: this.omniUploadUrlTtlHours, + }, + this.toolRegistry, + ); + } + // Fire-and-forget MCP discovery. Each server's tools land in the // registry as it becomes ready; the cli's AppContainer debounces // `setTools()` (~16ms / one frame) so the model sees the new tools @@ -6406,6 +6454,20 @@ export class Config { return this.omniPolicyTools; } + /** Normalized `omni.processing` view. Undefined until initialize() + * completes (or when omni is disabled). */ + getOmniProcessingConfig(): NormalizedOmniProcessingConfig | undefined { + return this.omniProcessingConfig; + } + + getOmniQuarantineRetentionDays(): number { + return this.omniQuarantineRetentionDays ?? 7; + } + + getOmniQuarantineMaxBytes(): number { + return this.omniQuarantineMaxBytes ?? 5 * 1024 * 1024 * 1024; + } + resolveImageGenerationModel( setting: string | undefined, ): ImageGenerationConfig | undefined { diff --git a/packages/core/src/omni/policy/config.test.ts b/packages/core/src/omni/policy/config.test.ts new file mode 100644 index 00000000000..446c080d17d --- /dev/null +++ b/packages/core/src/omni/policy/config.test.ts @@ -0,0 +1,787 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { describe, expect, it } from 'vitest'; +import { + DEFAULT_OMNI_PROCESSING_LIMITS, + OmniPolicyConfigError, + normalizeOmniProcessingConfig, +} from './config.js'; +import type { + OmniPolicyToolLookup, + RawOmniProcessingSettings, +} from './config.js'; +import type { MediaPolicyToolDescriptor } from '../../tools/tools.js'; +import type { OmniModality } from '../recognition.js'; + +const TUNABLE_SCHEMA = { + type: 'object', + properties: { + maxDimension: { type: 'number', minimum: 1 }, + quality: { type: 'number', minimum: 1, maximum: 100 }, + }, + additionalProperties: false, +}; + +interface ToolStub { + mediaPolicyDescriptor?: MediaPolicyToolDescriptor; + schema?: { parametersJsonSchema?: unknown }; +} + +function makeTool( + inputMediaTypes: OmniModality[], + overrides: Partial = {}, +): ToolStub { + return { + mediaPolicyDescriptor: { + kind: 'media_policy', + inputMediaTypes, + outputs: [ + { kind: 'media', required: true, lossy: true }, + { kind: 'text', role: 'disclosure', required: true }, + ], + settingsSchema: TUNABLE_SCHEMA, + ...overrides, + }, + schema: { + parametersJsonSchema: { + type: 'object', + properties: { + inputPath: { type: 'string' }, + outputDir: { type: 'string' }, + maxDimension: { type: 'number' }, + quality: { type: 'number' }, + }, + }, + }, + }; +} + +function defaultTools(): Record { + return { + omni_downsample_image: makeTool(['image']), + omni_downscale_video: makeTool(['video']), + omni_downsample_audio: makeTool(['audio']), + }; +} + +function lookup(tools: Record): OmniPolicyToolLookup { + return { getTool: (name) => tools[name] }; +} + +function normalize( + raw: RawOmniProcessingSettings = {}, + tools: Record = defaultTools(), +) { + return normalizeOmniProcessingConfig(raw, lookup(tools)); +} + +describe('normalizeOmniProcessingConfig', () => { + describe('system defaults', () => { + it('produces the three default fixed policies with when-thresholds', () => { + const config = normalize(); + expect(config.fixedPolicies.map((p) => p.id).sort()).toEqual([ + 'audio-downsample', + 'image-downsample', + 'video-downscale', + ]); + const image = config.fixedPolicies.find( + (p) => p.id === 'image-downsample', + ); + expect(image).toEqual({ + id: 'image-downsample', + priority: 0, + mediaTypes: ['image'], + origins: ['user', 'tool'], + when: { + any: [ + { + left: { field: 'resource.width' }, + operator: 'gt', + right: { value: 1568 }, + }, + { + left: { field: 'resource.height' }, + operator: 'gt', + right: { value: 1568 }, + }, + ], + }, + onConditionUnavailable: 'skip', + toolName: 'omni_downsample_image', + arguments: {}, + maxRunsPerLineage: 1, + onFailure: 'continue', + output: { reprocessMedia: false, source: 'omit' }, + stage: 'preprocessing', + }); + const video = config.fixedPolicies.find( + (p) => p.id === 'video-downscale', + ); + expect(video?.toolName).toBe('omni_downscale_video'); + expect(video?.when).toEqual({ + any: [ + { + left: { field: 'resource.height' }, + operator: 'gt', + right: { value: 480 }, + }, + { + left: { field: 'resource.sizeBytes' }, + operator: 'gt', + right: { value: 209715200 }, + }, + ], + }); + const audio = config.fixedPolicies.find( + (p) => p.id === 'audio-downsample', + ); + expect(audio?.toolName).toBe('omni_downsample_audio'); + expect(audio?.when).toEqual({ + any: [ + { + left: { field: 'resource.bitRate' }, + operator: 'gt', + right: { value: 96000 }, + }, + { + left: { field: 'resource.sampleRateHz' }, + operator: 'gt', + right: { value: 24000 }, + }, + ], + }); + }); + + it('produces the three default guard policies without when, stage transport_guard', () => { + const config = normalize(); + expect(config.transportGuardPolicies.map((p) => p.id).sort()).toEqual([ + 'audio-downsample', + 'image-downsample', + 'video-downscale', + ]); + for (const policy of config.transportGuardPolicies) { + expect(policy.when).toBeUndefined(); + expect(policy.stage).toBe('transport_guard'); + expect(policy.output.source).toBe('omit'); + } + }); + + it('defaults limits per policy design §12.2', () => { + expect(normalize().limits).toEqual({ + maxConcurrentResources: 1, + reservedOutputTokens: 8192, + maxLineageDepth: 8, + maxPolicyRunsPerRoot: 64, + maxArtifactsPerRoot: 256, + maxDerivedBytesPerRoot: 1073741824, + maxTransportPasses: 3, + }); + expect(normalize().limits).toEqual(DEFAULT_OMNI_PROCESSING_LIMITS); + }); + }); + + describe('id-merge semantics', () => { + it('removes a default fixed policy on null tombstone', () => { + const config = normalize({ + fixedPolicies: { 'image-downsample': null }, + }); + expect(config.fixedPolicies.map((p) => p.id).sort()).toEqual([ + 'audio-downsample', + 'video-downscale', + ]); + }); + + it('replaces a default entry wholesale (no field-level merge)', () => { + const config = normalize({ + fixedPolicies: { + 'image-downsample': { + mediaTypes: ['image'], + toolName: 'omni_downsample_image', + arguments: { maxDimension: 1024 }, + }, + }, + }); + const image = config.fixedPolicies.find( + (p) => p.id === 'image-downsample', + ); + // The default's `when` does NOT survive: whole-entry replacement. + expect(image?.when).toBeUndefined(); + expect(image?.arguments).toEqual({ maxDimension: 1024 }); + }); + + it('accepts additional user policies alongside defaults', () => { + const config = normalize({ + fixedPolicies: { + 'my-policy': { + priority: 5, + mediaTypes: ['image'], + toolName: 'omni_downsample_image', + }, + }, + }); + expect(config.fixedPolicies).toHaveLength(4); + const mine = config.fixedPolicies.find((p) => p.id === 'my-policy'); + expect(mine?.priority).toBe(5); + expect(mine?.stage).toBe('preprocessing'); + }); + + it('rejects transport-guard tombstones (the guard is mandatory)', () => { + expect(() => + normalize({ transportGuardPolicies: { 'image-downsample': null } }), + ).toThrow( + 'omni.processing.transportGuard.policies.image-downsample: ' + + 'transport guard policies cannot be removed (the guard is ' + + 'mandatory); override the entry instead', + ); + }); + + it('rejects non-object policy maps', () => { + expect(() => normalize({ fixedPolicies: ['nope'] })).toThrow( + 'omni.processing.fixedPolicies: must be an object map of policy id → policy', + ); + expect(() => + normalize({ fixedPolicies: { bad: 'string' as never } }), + ).toThrow( + 'omni.processing.fixedPolicies.bad: must be an object (or null to remove a default)', + ); + }); + }); + + describe('policy entry validation', () => { + it('rejects unknown keys (§13 #1)', () => { + expect(() => + normalize({ + fixedPolicies: { + p: { + mediaTypes: ['image'], + toolName: 'omni_downsample_image', + retries: 3, + }, + }, + }), + ).toThrow('omni.processing.fixedPolicies.p: unknown key "retries"'); + }); + + it('rejects malformed policy ids', () => { + expect(() => + normalize({ + fixedPolicies: { + 'has space': { + mediaTypes: ['image'], + toolName: 'omni_downsample_image', + }, + }, + }), + ).toThrow(OmniPolicyConfigError); + }); + + it('rejects empty or unknown mediaTypes (§13 #3)', () => { + expect(() => + normalize({ + fixedPolicies: { + p: { mediaTypes: [], toolName: 'omni_downsample_image' }, + }, + }), + ).toThrow( + 'omni.processing.fixedPolicies.p.mediaTypes: must be a non-empty array', + ); + expect(() => + normalize({ + fixedPolicies: { + p: { mediaTypes: ['text'], toolName: 'omni_downsample_image' }, + }, + }), + ).toThrow( + 'omni.processing.fixedPolicies.p.mediaTypes: unknown modality "text" ' + + '(expected image, video, audio)', + ); + }); + + it('rejects unknown origins (§13 #4)', () => { + expect(() => + normalize({ + fixedPolicies: { + p: { + mediaTypes: ['image'], + origins: ['model'], + toolName: 'omni_downsample_image', + }, + }, + }), + ).toThrow( + 'omni.processing.fixedPolicies.p.origins: unknown origin "model" ' + + '(expected user, tool, policy)', + ); + }); + + it('rejects onConditionUnavailable "abortTurn" with an explicit not-yet-supported error', () => { + expect(() => + normalize({ + fixedPolicies: { + p: { + mediaTypes: ['image'], + toolName: 'omni_downsample_image', + onConditionUnavailable: 'abortTurn', + }, + }, + }), + ).toThrow(/"abortTurn" is not yet supported/); + }); + + it('rejects invalid onFailure', () => { + expect(() => + normalize({ + fixedPolicies: { + p: { + mediaTypes: ['image'], + toolName: 'omni_downsample_image', + onFailure: 'retry', + }, + }, + }), + ).toThrow( + 'omni.processing.fixedPolicies.p.onFailure: must be "continue" or "abort" (got "retry")', + ); + }); + + it('rejects non-positive maxRunsPerLineage', () => { + expect(() => + normalize({ + fixedPolicies: { + p: { + mediaTypes: ['image'], + toolName: 'omni_downsample_image', + maxRunsPerLineage: 0, + }, + }, + }), + ).toThrow( + 'omni.processing.fixedPolicies.p.maxRunsPerLineage: must be a positive integer (got 0)', + ); + }); + + it('rejects unknown output keys and illegal output.source (§13 #23)', () => { + expect(() => + normalize({ + fixedPolicies: { + p: { + mediaTypes: ['image'], + toolName: 'omni_downsample_image', + output: { keepBoth: true }, + }, + }, + }), + ).toThrow( + 'omni.processing.fixedPolicies.p.output: unknown key "keepBoth"', + ); + expect(() => + normalize({ + fixedPolicies: { + p: { + mediaTypes: ['image'], + toolName: 'omni_downsample_image', + output: { source: 'drop' }, + }, + }, + }), + ).toThrow( + 'omni.processing.fixedPolicies.p.output.source: must be "keep" or "omit" (got "drop")', + ); + }); + + it('allows output.source "keep" for preprocessing policies', () => { + const config = normalize({ + fixedPolicies: { + p: { + mediaTypes: ['image'], + toolName: 'omni_downsample_image', + output: { source: 'keep', reprocessMedia: true }, + }, + }, + }); + const p = config.fixedPolicies.find((x) => x.id === 'p'); + expect(p?.output).toEqual({ reprocessMedia: true, source: 'keep' }); + }); + + it('rejects invalid when-conditions via the shared validator (§13 #5)', () => { + expect(() => + normalize({ + fixedPolicies: { + p: { + mediaTypes: ['image'], + toolName: 'omni_downsample_image', + when: { + left: { field: 'resource.nonexistent' }, + operator: 'gt', + right: { value: 1 }, + }, + }, + }, + }), + ).toThrow(/omni\.processing\.fixedPolicies\.p\.when/); + }); + }); + + describe('tool reference validation (§13 #6/#8/#14)', () => { + it('rejects a missing toolName', () => { + expect(() => + normalize({ fixedPolicies: { p: { mediaTypes: ['image'] } } }), + ).toThrow( + 'omni.processing.fixedPolicies.p.toolName: must be a non-empty string', + ); + }); + + it('rejects an unregistered tool (covers excluded tools too)', () => { + expect(() => + normalize({ + fixedPolicies: { + p: { mediaTypes: ['image'], toolName: 'no_such_tool' }, + }, + }), + ).toThrow( + 'omni.processing.fixedPolicies.p.toolName: tool "no_such_tool" is ' + + 'not registered (unknown name, or excluded by tool filtering)', + ); + }); + + it('rejects a registered tool without a media_policy descriptor', () => { + const tools = defaultTools(); + tools['read_file'] = { schema: { parametersJsonSchema: {} } }; + expect(() => + normalize( + { + fixedPolicies: { + p: { mediaTypes: ['image'], toolName: 'read_file' }, + }, + }, + tools, + ), + ).toThrow( + 'omni.processing.fixedPolicies.p.toolName: tool "read_file" is not ' + + 'a media policy tool (no media_policy descriptor)', + ); + }); + + it('rejects a tool declaring no required output', () => { + const tools = defaultTools(); + tools['weak_tool'] = makeTool(['image'], { + outputs: [{ kind: 'media', required: false, lossy: false }], + }); + expect(() => + normalize( + { + fixedPolicies: { + p: { mediaTypes: ['image'], toolName: 'weak_tool' }, + }, + }, + tools, + ), + ).toThrow(/declares no required output/); + }); + + it('rejects a lossy tool without a disclosure output (§13 #8)', () => { + const tools = defaultTools(); + tools['sneaky_tool'] = makeTool(['image'], { + outputs: [{ kind: 'media', required: true, lossy: true }], + }); + expect(() => + normalize( + { + fixedPolicies: { + p: { mediaTypes: ['image'], toolName: 'sneaky_tool' }, + }, + }, + tools, + ), + ).toThrow( + 'omni.processing.fixedPolicies.p.toolName: tool "sneaky_tool" ' + + 'declares a lossy media output but no disclosure text output', + ); + }); + + it('rejects mediaTypes the tool does not accept', () => { + expect(() => + normalize({ + fixedPolicies: { + p: { + mediaTypes: ['image', 'video'], + toolName: 'omni_downsample_image', + }, + }, + }), + ).toThrow( + 'omni.processing.fixedPolicies.p.mediaTypes: tool ' + + '"omni_downsample_image" does not accept "video" input (accepts image)', + ); + }); + }); + + describe('fixed arguments validation (§13 #11)', () => { + it('rejects reserved io keys in arguments', () => { + expect(() => + normalize({ + fixedPolicies: { + p: { + mediaTypes: ['image'], + toolName: 'omni_downsample_image', + arguments: { inputPath: '/tmp/x.png' }, + }, + }, + }), + ).toThrow( + 'omni.processing.fixedPolicies.p.arguments: "inputPath" is injected ' + + 'by the orchestrator per invocation and must not be configured', + ); + }); + + it('validates arguments against the settingsSchema (io-stripped)', () => { + expect(() => + normalize({ + fixedPolicies: { + p: { + mediaTypes: ['image'], + toolName: 'omni_downsample_image', + arguments: { bogus: true }, + }, + }, + }), + ).toThrow(/omni\.processing\.fixedPolicies\.p\.arguments/); + // Valid tunables pass through untouched. + const config = normalize({ + fixedPolicies: { + p: { + mediaTypes: ['image'], + toolName: 'omni_downsample_image', + arguments: { maxDimension: 800, quality: 70 }, + }, + }, + }); + expect(config.fixedPolicies.find((x) => x.id === 'p')?.arguments).toEqual( + { maxDimension: 800, quality: 70 }, + ); + }); + }); + + describe('transport guard rules (§13 #15-#17)', () => { + it('rejects guard policies declaring when', () => { + expect(() => + normalize({ + transportGuardPolicies: { + 'image-downsample': { + mediaTypes: ['image'], + toolName: 'omni_downsample_image', + when: { + left: { field: 'resource.width' }, + operator: 'gt', + right: { value: 1 }, + }, + }, + }, + }), + ).toThrow( + 'omni.processing.transportGuard.policies.image-downsample.when: ' + + 'transport guard policies must not declare "when" (they run ' + + 'exactly when transport limits are exceeded)', + ); + }); + + it('rejects guard policies with output.source "keep"', () => { + expect(() => + normalize({ + transportGuardPolicies: { + 'image-downsample': { + mediaTypes: ['image'], + toolName: 'omni_downsample_image', + output: { source: 'keep' }, + }, + }, + }), + ).toThrow( + 'omni.processing.transportGuard.policies.image-downsample.output.source: ' + + 'transport guard policies must use "omit" (the over-limit source ' + + 'cannot stay in the delivery set)', + ); + }); + + it('rejects a merged guard set that does not cover all three modalities', () => { + const tools = defaultTools(); + // Point every guard entry at image only → video+audio uncovered. + expect(() => + normalize( + { + transportGuardPolicies: { + 'video-downscale': { + mediaTypes: ['image'], + toolName: 'omni_downsample_image', + }, + 'audio-downsample': { + mediaTypes: ['image'], + toolName: 'omni_downsample_image', + }, + }, + }, + tools, + ), + ).toThrow( + 'omni.processing.transportGuard.policies: no guard policy covers ' + + 'video, audio — the merged set must cover image, video, and audio', + ); + }); + }); + + describe('limits (§12.2)', () => { + it('merges overrides over defaults', () => { + const config = normalize({ limits: { maxLineageDepth: 3 } }); + expect(config.limits).toEqual({ + ...DEFAULT_OMNI_PROCESSING_LIMITS, + maxLineageDepth: 3, + }); + }); + + it('rejects unknown limit keys', () => { + expect(() => normalize({ limits: { maxFoo: 1 } })).toThrow( + 'omni.processing.limits: unknown key "maxFoo"', + ); + }); + + it('rejects non-positive-integer values', () => { + expect(() => normalize({ limits: { maxLineageDepth: 0 } })).toThrow( + 'omni.processing.limits.maxLineageDepth: must be a positive integer (got 0)', + ); + expect(() => normalize({ limits: { maxLineageDepth: 2.5 } })).toThrow( + 'omni.processing.limits.maxLineageDepth: must be a positive integer (got 2.5)', + ); + }); + + it('allows reservedOutputTokens of zero', () => { + const config = normalize({ limits: { reservedOutputTokens: 0 } }); + expect(config.limits.reservedOutputTokens).toBe(0); + }); + }); + + describe('channel caps (§13 #18/#19)', () => { + it('rejects maxUploadFileBytes above the 1 GiB channel cap', () => { + expect(() => normalize({ maxUploadFileBytes: 1073741824 + 1 })).toThrow( + 'omni.processing.transportGuard.maxUploadFileBytes: 1073741825 ' + + 'exceeds the DashScope per-file upload cap (1073741824)', + ); + expect(() => normalize({ maxUploadFileBytes: 1073741824 })).not.toThrow(); + }); + + it('rejects urlTtlHours outside 0..48', () => { + expect(() => normalize({ urlTtlHours: 49 })).toThrow( + 'omni.delivery.upload.urlTtlHours: must be a number between 0 and 48 (got 49)', + ); + expect(() => normalize({ urlTtlHours: -1 })).toThrow( + OmniPolicyConfigError, + ); + expect(() => normalize({ urlTtlHours: 48 })).not.toThrow(); + expect(() => normalize({ urlTtlHours: 0 })).not.toThrow(); + }); + }); + + describe('policyTools validation (§13 #7/#20/#21)', () => { + it('accepts null tombstones and valid entries', () => { + expect(() => + normalize({ + policyTools: { + omni_downsample_image: null, + omni_downscale_video: { + settings: { maxDimension: 640 }, + runtime: { timeoutMs: 30000 }, + }, + }, + }), + ).not.toThrow(); + }); + + it('rejects entries naming a non-media-policy tool', () => { + expect(() => + normalize({ policyTools: { no_such_tool: { settings: {} } } }), + ).toThrow( + 'omni.processing.policyTools.no_such_tool: "no_such_tool" is not a ' + + 'registered media policy tool', + ); + }); + + it('validates settings against the settingsSchema (§13 #7)', () => { + expect(() => + normalize({ + policyTools: { + omni_downsample_image: { settings: { bogus: 1 } }, + }, + }), + ).toThrow( + /omni\.processing\.policyTools\.omni_downsample_image\.settings/, + ); + }); + + it('rejects non-positive runtime.timeoutMs', () => { + expect(() => + normalize({ + policyTools: { + omni_downsample_image: { runtime: { timeoutMs: -5 } }, + }, + }), + ).toThrow( + 'omni.processing.policyTools.omni_downsample_image.runtime.timeoutMs: ' + + 'must be a positive integer (got -5)', + ); + }); + + it('rejects overlapping defaultArguments and lockedArguments (§13 #21)', () => { + expect(() => + normalize({ + policyTools: { + omni_downsample_image: { + modelAccess: { + defaultArguments: { quality: 80 }, + lockedArguments: { quality: 60 }, + }, + }, + }, + }), + ).toThrow( + 'omni.processing.policyTools.omni_downsample_image.modelAccess: ' + + '"quality" present in both defaultArguments and lockedArguments', + ); + }); + + it('rejects parameterSchema properties absent from the native schema (§13 #20)', () => { + expect(() => + normalize({ + policyTools: { + omni_downsample_image: { + modelAccess: { + parameterSchema: { + properties: { quality: {}, sharpen: {} }, + }, + }, + }, + }, + }), + ).toThrow( + 'omni.processing.policyTools.omni_downsample_image.modelAccess.parameterSchema: ' + + '"sharpen" not present in the tool\'s native schema (projection may only narrow)', + ); + }); + + it('accepts a narrowing-only parameterSchema', () => { + expect(() => + normalize({ + policyTools: { + omni_downsample_image: { + modelAccess: { + parameterSchema: { properties: { quality: {} } }, + }, + }, + }, + }), + ).not.toThrow(); + }); + }); +}); diff --git a/packages/core/src/omni/policy/config.ts b/packages/core/src/omni/policy/config.ts new file mode 100644 index 00000000000..4ec806aba7d --- /dev/null +++ b/packages/core/src/omni/policy/config.ts @@ -0,0 +1,723 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import type { MediaPolicyToolDescriptor } from '../../tools/tools.js'; +import { SchemaValidator } from '../../utils/schemaValidator.js'; +import type { OmniModality } from '../recognition.js'; +import { validateFixedPolicyCondition } from './conditions.js'; +import type { FixedPolicyCondition } from './conditions.js'; +import type { + FixedPolicyOrigin, + NormalizedFixedPolicy, + NormalizedOmniProcessingConfig, + NormalizedOmniProcessingLimits, + OmniPolicyToolsSettings, +} from './types.js'; + +/** + * Startup normalization of `omni.processing` (policy design §13 applicable + * subset — see the S4 mapping doc §7). Raw settings enter, a fully + * defaulted and validated {@link NormalizedOmniProcessingConfig} leaves; + * any violation throws {@link OmniPolicyConfigError} and MUST abort + * startup — a mis-configured guard must never degrade into delivering + * over-limit media. + */ + +/** A configuration error in `omni.processing.*`. Startup-fatal. */ +export class OmniPolicyConfigError extends Error { + constructor(message: string) { + super(message); + this.name = 'OmniPolicyConfigError'; + } +} + +/** Per-root derivation budget defaults (policy design §12.2). */ +export const DEFAULT_OMNI_PROCESSING_LIMITS: NormalizedOmniProcessingLimits = { + maxConcurrentResources: 1, + reservedOutputTokens: 8192, + maxLineageDepth: 8, + maxPolicyRunsPerRoot: 64, + maxArtifactsPerRoot: 256, + maxDerivedBytesPerRoot: 1024 * 1024 * 1024, + maxTransportPasses: 3, +}; + +const GIB = 1024 * 1024 * 1024; +/** DashScope temporary-upload per-file cap (§13 #18). */ +const MAX_UPLOAD_FILE_BYTES_CEILING = GIB; +/** DashScope temporary uploads live 48h (§13 #19). */ +const MAX_URL_TTL_HOURS = 48; + +/** Raw fixed-policy entry shape accepted from settings. Everything + * optional except `mediaTypes` and `toolName`; unknown keys rejected. */ +const POLICY_ENTRY_KEYS = new Set([ + 'priority', + 'mediaTypes', + 'origins', + 'when', + 'onConditionUnavailable', + 'toolName', + 'arguments', + 'maxRunsPerLineage', + 'onFailure', + 'output', +]); +const OUTPUT_KEYS = new Set(['reprocessMedia', 'source']); +const MODALITIES: readonly OmniModality[] = ['image', 'video', 'audio']; +const ORIGINS: readonly FixedPolicyOrigin[] = ['user', 'tool', 'policy']; +/** io params are harness-injected per invocation — fixed `arguments` + * naming them would be overwritten silently, so they are rejected. */ +const RESERVED_ARGUMENT_KEYS = ['inputPath', 'outputDir', 'resourceId']; + +/** Policy ids feed run records and execution origins; keep them to a + * conservative token charset. */ +const POLICY_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]*$/; + +const DEFAULT_WHEN_IMAGE: FixedPolicyCondition = { + any: [ + { + left: { field: 'resource.width' }, + operator: 'gt', + right: { value: 1568 }, + }, + { + left: { field: 'resource.height' }, + operator: 'gt', + right: { value: 1568 }, + }, + ], +}; +const DEFAULT_WHEN_VIDEO: FixedPolicyCondition = { + any: [ + { + left: { field: 'resource.height' }, + operator: 'gt', + right: { value: 480 }, + }, + { + left: { field: 'resource.sizeBytes' }, + operator: 'gt', + right: { value: 200 * 1024 * 1024 }, + }, + ], +}; +const DEFAULT_WHEN_AUDIO: FixedPolicyCondition = { + any: [ + { + left: { field: 'resource.bitRate' }, + operator: 'gt', + right: { value: 96_000 }, + }, + { + left: { field: 'resource.sampleRateHz' }, + operator: 'gt', + right: { value: 24_000 }, + }, + ], +}; + +interface SystemDefaultPolicy { + mediaTypes: OmniModality[]; + toolName: string; + when: FixedPolicyCondition; +} + +const SYSTEM_DEFAULT_POLICY_BASES: Record = { + 'image-downsample': { + mediaTypes: ['image'], + toolName: 'omni_downsample_image', + when: DEFAULT_WHEN_IMAGE, + }, + 'video-downscale': { + mediaTypes: ['video'], + toolName: 'omni_downscale_video', + when: DEFAULT_WHEN_VIDEO, + }, + 'audio-downsample': { + mediaTypes: ['audio'], + toolName: 'omni_downsample_audio', + when: DEFAULT_WHEN_AUDIO, + }, +}; + +/** + * System default policies, registered twice (decision D7): as + * `fixedPolicies` WITH when-thresholds (preprocessing — degrade before the + * guard ever sees the media) and as `transportGuard.policies` WITHOUT + * `when` (guard stage runs them only when the final delivery set still + * exceeds transport limits). One tool set covers both duties and satisfies + * the guard's three-modality coverage requirement out of the box. + */ +export function systemDefaultFixedPolicies(): Record< + string, + Record +> { + const entries: Record> = {}; + for (const [id, base] of Object.entries(SYSTEM_DEFAULT_POLICY_BASES)) { + entries[id] = { + mediaTypes: base.mediaTypes, + toolName: base.toolName, + when: base.when, + }; + } + return entries; +} + +export function systemDefaultTransportGuardPolicies(): Record< + string, + Record +> { + const entries: Record> = {}; + for (const [id, base] of Object.entries(SYSTEM_DEFAULT_POLICY_BASES)) { + entries[id] = { + mediaTypes: base.mediaTypes, + toolName: base.toolName, + }; + } + return entries; +} + +/** Raw inputs to normalization, as threaded from settings. */ +export interface RawOmniProcessingSettings { + /** `omni.processing.fixedPolicies` (id → entry | null tombstone). */ + fixedPolicies?: unknown; + /** `omni.processing.transportGuard.policies` (id → entry; tombstones + * are a configuration error — the guard cannot be disabled). */ + transportGuardPolicies?: unknown; + /** `omni.processing.limits`. */ + limits?: unknown; + /** `omni.processing.policyTools`. */ + policyTools?: OmniPolicyToolsSettings; + /** `omni.processing.transportGuard.maxUploadFileBytes`. */ + maxUploadFileBytes?: number; + /** `omni.delivery.upload.urlTtlHours`. */ + urlTtlHours?: number; +} + +/** Tool lookup surface normalization validates against (§13 #6/#14: a + * tool that is unregistered — including excluded via tool filtering — or + * not a media-policy tool fails normalization). */ +export interface OmniPolicyToolLookup { + getTool(name: string): + | { + mediaPolicyDescriptor?: MediaPolicyToolDescriptor; + schema?: { parametersJsonSchema?: unknown }; + } + | undefined; +} + +const isPlainRecord = (value: unknown): value is Record => + typeof value === 'object' && value !== null && !Array.isArray(value); + +function fail(message: string): never { + throw new OmniPolicyConfigError(message); +} + +function requirePositiveInteger( + value: unknown, + where: string, + { allowZero = false }: { allowZero?: boolean } = {}, +): number { + if ( + typeof value !== 'number' || + !Number.isInteger(value) || + (allowZero ? value < 0 : value <= 0) + ) { + fail( + `${where}: must be ${allowZero ? 'a non-negative' : 'a positive'} integer (got ${JSON.stringify(value)})`, + ); + } + return value; +} + +/** ID-merge two policy maps: user entry replaces the whole default entry; + * `null` tombstones (where allowed) remove it. */ +function mergePolicyMaps( + defaults: Record>, + raw: unknown, + where: string, + { allowTombstones }: { allowTombstones: boolean }, +): Record> { + const merged: Record | null> = { + ...defaults, + }; + if (raw === undefined) { + return merged as Record>; + } + if (!isPlainRecord(raw)) { + fail(`${where}: must be an object map of policy id → policy`); + } + for (const [id, entry] of Object.entries(raw)) { + if (entry === null) { + if (!allowTombstones) { + fail( + `${where}.${id}: transport guard policies cannot be removed ` + + `(the guard is mandatory); override the entry instead`, + ); + } + delete merged[id]; + continue; + } + if (!isPlainRecord(entry)) { + fail(`${where}.${id}: must be an object (or null to remove a default)`); + } + merged[id] = entry; + } + return merged as Record>; +} + +function normalizePolicy( + id: string, + entry: Record, + stage: 'preprocessing' | 'transport_guard', + where: string, + tools: OmniPolicyToolLookup, +): NormalizedFixedPolicy { + // §13 #2: unique ids come free with the map shape; the charset check + // keeps ids safe for run records and log lines. + if (!POLICY_ID_PATTERN.test(id)) { + fail( + `${where}.${id}: policy id must match ${POLICY_ID_PATTERN} ` + + `(letters, digits, ".", "_", "-")`, + ); + } + // §13 #1: strict structure — unknown keys are errors, not warnings. + for (const key of Object.keys(entry)) { + if (!POLICY_ENTRY_KEYS.has(key)) { + fail(`${where}.${id}: unknown key "${key}"`); + } + } + + // §13 #3/#4: legal values and enums. + const priority = + entry['priority'] === undefined + ? 0 + : typeof entry['priority'] === 'number' && + Number.isFinite(entry['priority']) + ? entry['priority'] + : fail(`${where}.${id}.priority: must be a finite number`); + + const rawMediaTypes = entry['mediaTypes']; + if (!Array.isArray(rawMediaTypes) || rawMediaTypes.length === 0) { + fail(`${where}.${id}.mediaTypes: must be a non-empty array`); + } + for (const m of rawMediaTypes) { + if (!MODALITIES.includes(m as OmniModality)) { + fail( + `${where}.${id}.mediaTypes: unknown modality ${JSON.stringify(m)} ` + + `(expected ${MODALITIES.join(', ')})`, + ); + } + } + const mediaTypes = [...new Set(rawMediaTypes as OmniModality[])]; + + const rawOrigins = entry['origins'] ?? ['user', 'tool']; + if (!Array.isArray(rawOrigins) || rawOrigins.length === 0) { + fail(`${where}.${id}.origins: must be a non-empty array`); + } + for (const o of rawOrigins) { + if (!ORIGINS.includes(o as FixedPolicyOrigin)) { + fail( + `${where}.${id}.origins: unknown origin ${JSON.stringify(o)} ` + + `(expected ${ORIGINS.join(', ')})`, + ); + } + } + const origins = [...new Set(rawOrigins as FixedPolicyOrigin[])]; + + const onConditionUnavailable = entry['onConditionUnavailable'] ?? 'skip'; + if (onConditionUnavailable === 'abortTurn') { + fail( + `${where}.${id}.onConditionUnavailable: "abortTurn" is not yet ` + + `supported (design reserves it for a later stage); use "skip" or "run"`, + ); + } + if (onConditionUnavailable !== 'skip' && onConditionUnavailable !== 'run') { + fail( + `${where}.${id}.onConditionUnavailable: must be "skip" or "run" ` + + `(got ${JSON.stringify(onConditionUnavailable)})`, + ); + } + + const onFailure = entry['onFailure'] ?? 'continue'; + if (onFailure !== 'continue' && onFailure !== 'abort') { + fail( + `${where}.${id}.onFailure: must be "continue" or "abort" ` + + `(got ${JSON.stringify(onFailure)})`, + ); + } + + const maxRunsPerLineage = + entry['maxRunsPerLineage'] === undefined + ? 1 + : requirePositiveInteger( + entry['maxRunsPerLineage'], + `${where}.${id}.maxRunsPerLineage`, + ); + + const rawOutput = entry['output'] ?? {}; + if (!isPlainRecord(rawOutput)) { + fail(`${where}.${id}.output: must be an object`); + } + // §13 #23: output fields are a closed set with concrete defaults. + for (const key of Object.keys(rawOutput)) { + if (!OUTPUT_KEYS.has(key)) { + fail(`${where}.${id}.output: unknown key "${key}"`); + } + } + const reprocessMedia = rawOutput['reprocessMedia'] ?? false; + if (typeof reprocessMedia !== 'boolean') { + fail(`${where}.${id}.output.reprocessMedia: must be a boolean`); + } + const source = rawOutput['source'] ?? 'omit'; + if (source !== 'keep' && source !== 'omit') { + fail( + `${where}.${id}.output.source: must be "keep" or "omit" ` + + `(got ${JSON.stringify(source)})`, + ); + } + // §13 #17: transport-guard outputs must replace the offending source — + // keeping it would re-deliver the very media the guard rejected. + if (stage === 'transport_guard' && source !== 'omit') { + fail( + `${where}.${id}.output.source: transport guard policies must use ` + + `"omit" (the over-limit source cannot stay in the delivery set)`, + ); + } + + // §13 #5: when-condition structure and field names. + let when: FixedPolicyCondition | undefined; + if (entry['when'] !== undefined) { + if (stage === 'transport_guard') { + // Guard policies are triggered by the limit breach itself, never by + // conditions; a `when` here would silently punch a hole in coverage. + fail( + `${where}.${id}.when: transport guard policies must not declare ` + + `"when" (they run exactly when transport limits are exceeded)`, + ); + } + const errors = validateFixedPolicyCondition( + entry['when'], + `${where}.${id}.when`, + ); + if (errors.length > 0) { + fail(errors.join('; ')); + } + when = entry['when'] as FixedPolicyCondition; + } + + // §13 #6 (+#14): the referenced tool must be registered — an excluded + // (tools.disabled) tool is absent from the registry — and be a + // media-policy tool. + const toolName = entry['toolName']; + if (typeof toolName !== 'string' || toolName.length === 0) { + fail(`${where}.${id}.toolName: must be a non-empty string`); + } + const tool = tools.getTool(toolName); + if (!tool) { + fail( + `${where}.${id}.toolName: tool "${toolName}" is not registered ` + + `(unknown name, or excluded by tool filtering)`, + ); + } + const descriptor = tool.mediaPolicyDescriptor; + if (!descriptor || descriptor.kind !== 'media_policy') { + fail( + `${where}.${id}.toolName: tool "${toolName}" is not a media policy ` + + `tool (no media_policy descriptor)`, + ); + } + // §13 #8: the descriptor must declare a deliverable output at all, and + // a lossy media output obligates a disclosure text output — otherwise + // the pipeline could degrade media with no user-visible disclosure. + if (!descriptor.outputs.some((o) => o.required)) { + fail( + `${where}.${id}.toolName: tool "${toolName}" declares no required ` + + `output; a fixed policy cannot rely on it producing anything`, + ); + } + const hasLossyMedia = descriptor.outputs.some( + (o) => o.kind === 'media' && o.lossy, + ); + const hasDisclosure = descriptor.outputs.some( + (o) => o.kind === 'text' && o.role === 'disclosure', + ); + if (hasLossyMedia && !hasDisclosure) { + fail( + `${where}.${id}.toolName: tool "${toolName}" declares a lossy media ` + + `output but no disclosure text output`, + ); + } + // The policy's modalities must be servable by the tool. + for (const m of mediaTypes) { + if (!descriptor.inputMediaTypes.includes(m)) { + fail( + `${where}.${id}.mediaTypes: tool "${toolName}" does not accept ` + + `"${m}" input (accepts ${descriptor.inputMediaTypes.join(', ')})`, + ); + } + } + + // §13 #11: fixed arguments validate against the tool's io-stripped + // tunable schema; the harness-injected io keys are reserved. + const args = entry['arguments'] ?? {}; + if (!isPlainRecord(args)) { + fail(`${where}.${id}.arguments: must be an object`); + } + for (const reserved of RESERVED_ARGUMENT_KEYS) { + if (Object.prototype.hasOwnProperty.call(args, reserved)) { + fail( + `${where}.${id}.arguments: "${reserved}" is injected by the ` + + `orchestrator per invocation and must not be configured`, + ); + } + } + if (descriptor.settingsSchema) { + const schemaError = SchemaValidator.validate( + descriptor.settingsSchema, + args, + ); + if (schemaError) { + fail(`${where}.${id}.arguments: ${schemaError}`); + } + } + + return { + id, + priority, + mediaTypes, + origins, + when, + onConditionUnavailable, + toolName, + arguments: args, + maxRunsPerLineage, + onFailure, + output: { reprocessMedia, source }, + stage, + }; +} + +function normalizeLimits(raw: unknown): NormalizedOmniProcessingLimits { + if (raw === undefined) { + return { ...DEFAULT_OMNI_PROCESSING_LIMITS }; + } + if (!isPlainRecord(raw)) { + fail('omni.processing.limits: must be an object'); + } + const limits = { ...DEFAULT_OMNI_PROCESSING_LIMITS }; + for (const [key, value] of Object.entries(raw)) { + if (!Object.prototype.hasOwnProperty.call(limits, key)) { + fail(`omni.processing.limits: unknown key "${key}"`); + } + limits[key as keyof NormalizedOmniProcessingLimits] = + requirePositiveInteger(value, `omni.processing.limits.${key}`, { + // Reserving zero output tokens is odd but not incoherent. + allowZero: key === 'reservedOutputTokens', + }); + } + return limits; +} + +function validatePolicyTools( + policyTools: OmniPolicyToolsSettings | undefined, + tools: OmniPolicyToolLookup, +): void { + if (policyTools === undefined) return; + if (!isPlainRecord(policyTools)) { + fail('omni.processing.policyTools: must be an object map'); + } + for (const [toolName, entry] of Object.entries(policyTools)) { + const where = `omni.processing.policyTools.${toolName}`; + if (entry === null) continue; // scope-merge tombstone + if (!isPlainRecord(entry)) { + fail(`${where}: must be an object`); + } + const tool = tools.getTool(toolName); + if (!tool?.mediaPolicyDescriptor) { + fail(`${where}: "${toolName}" is not a registered media policy tool`); + } + const descriptor = tool.mediaPolicyDescriptor; + + // §13 #7: tool-level settings validate against the settingsSchema. + if (entry['settings'] !== undefined) { + if (!isPlainRecord(entry['settings'])) { + fail(`${where}.settings: must be an object`); + } + if (descriptor.settingsSchema) { + const error = SchemaValidator.validate( + descriptor.settingsSchema, + entry['settings'], + ); + if (error) { + fail(`${where}.settings: ${error}`); + } + } + } + + if (entry['runtime'] !== undefined) { + if (!isPlainRecord(entry['runtime'])) { + fail(`${where}.runtime: must be an object`); + } + const timeoutMs = entry['runtime']['timeoutMs']; + if (timeoutMs !== undefined) { + requirePositiveInteger(timeoutMs, `${where}.runtime.timeoutMs`); + } + } + + const modelAccess = entry['modelAccess']; + if (modelAccess === undefined) continue; + if (!isPlainRecord(modelAccess)) { + fail(`${where}.modelAccess: must be an object`); + } + const defaults = modelAccess['defaultArguments']; + const locked = modelAccess['lockedArguments']; + if (defaults !== undefined && !isPlainRecord(defaults)) { + fail(`${where}.modelAccess.defaultArguments: must be an object`); + } + if (locked !== undefined && !isPlainRecord(locked)) { + fail(`${where}.modelAccess.lockedArguments: must be an object`); + } + // §13 #21: a key cannot be both defaulted (model may override) and + // locked (model must not name it) — the combination is contradictory. + if (isPlainRecord(defaults) && isPlainRecord(locked)) { + const conflicts = Object.keys(defaults).filter((k) => + Object.prototype.hasOwnProperty.call(locked, k), + ); + if (conflicts.length > 0) { + fail( + `${where}.modelAccess: ${conflicts + .map((k) => `"${k}"`) + .join(', ')} present in both defaultArguments and ` + + `lockedArguments`, + ); + } + } + // §13 #20: the model-visible projection may only narrow the native + // schema — a property the native schema does not declare cannot be + // introduced by projection. + const projection = modelAccess['parameterSchema']; + if (projection !== undefined) { + if (!isPlainRecord(projection)) { + fail(`${where}.modelAccess.parameterSchema: must be an object`); + } + const projectionProps = isPlainRecord(projection['properties']) + ? Object.keys(projection['properties']) + : []; + const nativeSchema = tool.schema?.parametersJsonSchema; + const nativeProps = + isPlainRecord(nativeSchema) && isPlainRecord(nativeSchema['properties']) + ? new Set(Object.keys(nativeSchema['properties'])) + : new Set(); + const introduced = projectionProps.filter((p) => !nativeProps.has(p)); + if (introduced.length > 0) { + fail( + `${where}.modelAccess.parameterSchema: ${introduced + .map((p) => `"${p}"`) + .join(', ')} not present in the tool's native schema ` + + `(projection may only narrow)`, + ); + } + } + } +} + +/** + * Normalize and validate the full `omni.processing` configuration. + * Called once at startup (after the tool registry exists); throws + * {@link OmniPolicyConfigError} on any violation. + */ +export function normalizeOmniProcessingConfig( + raw: RawOmniProcessingSettings, + tools: OmniPolicyToolLookup, +): NormalizedOmniProcessingConfig { + // §13 #18: the upload byte ceiling cannot exceed the channel's own cap. + if (raw.maxUploadFileBytes !== undefined) { + const bytes = requirePositiveInteger( + raw.maxUploadFileBytes, + 'omni.processing.transportGuard.maxUploadFileBytes', + ); + if (bytes > MAX_UPLOAD_FILE_BYTES_CEILING) { + fail( + `omni.processing.transportGuard.maxUploadFileBytes: ${bytes} exceeds ` + + `the DashScope per-file upload cap (${MAX_UPLOAD_FILE_BYTES_CEILING})`, + ); + } + } + // §13 #19: cached URLs must not outlive the channel's 48h validity. + if (raw.urlTtlHours !== undefined) { + const ttl = raw.urlTtlHours; + if ( + typeof ttl !== 'number' || + !Number.isFinite(ttl) || + ttl < 0 || + ttl > MAX_URL_TTL_HOURS + ) { + fail( + `omni.delivery.upload.urlTtlHours: must be a number between 0 and ` + + `${MAX_URL_TTL_HOURS} (got ${JSON.stringify(ttl)})`, + ); + } + } + + const limits = normalizeLimits(raw.limits); + validatePolicyTools(raw.policyTools, tools); + + const fixedMap = mergePolicyMaps( + systemDefaultFixedPolicies(), + raw.fixedPolicies, + 'omni.processing.fixedPolicies', + { allowTombstones: true }, + ); + const guardMap = mergePolicyMaps( + systemDefaultTransportGuardPolicies(), + raw.transportGuardPolicies, + 'omni.processing.transportGuard.policies', + { allowTombstones: false }, + ); + + const fixedPolicies = Object.entries(fixedMap).map(([id, entry]) => + normalizePolicy( + id, + entry, + 'preprocessing', + 'omni.processing.fixedPolicies', + tools, + ), + ); + const transportGuardPolicies = Object.entries(guardMap).map(([id, entry]) => + normalizePolicy( + id, + entry, + 'transport_guard', + 'omni.processing.transportGuard.policies', + tools, + ), + ); + + // §13 #15/#16: the merged guard set must exist and must cover every + // modality the pipeline can deliver — a modality without a guard policy + // would fail closed with no degradation path. + if (transportGuardPolicies.length === 0) { + fail( + 'omni.processing.transportGuard.policies: must not be empty ' + + '(the transport guard is mandatory)', + ); + } + const covered = new Set( + transportGuardPolicies.flatMap((policy) => policy.mediaTypes), + ); + const uncovered = MODALITIES.filter((m) => !covered.has(m)); + if (uncovered.length > 0) { + fail( + `omni.processing.transportGuard.policies: no guard policy covers ` + + `${uncovered.join(', ')} — the merged set must cover image, video, ` + + `and audio`, + ); + } + + return { fixedPolicies, transportGuardPolicies, limits }; +} diff --git a/packages/core/src/omni/policy/types.ts b/packages/core/src/omni/policy/types.ts index 0be21fb900c..cef105e9305 100644 --- a/packages/core/src/omni/policy/types.ts +++ b/packages/core/src/omni/policy/types.ts @@ -80,10 +80,32 @@ export interface NormalizedFixedPolicy { stage: 'preprocessing' | 'transport_guard'; } +/** Normalized `omni.processing.limits` — per-root derivation budgets + * (policy design §12.2). Every field concrete after normalization. */ +export interface NormalizedOmniProcessingLimits { + /** Media resources processed by policies in parallel per request. */ + maxConcurrentResources: number; + /** Tokens reserved for model output when computing + * `session.availableContextTokens` for when-conditions. */ + reservedOutputTokens: number; + /** Maximum derivation chain length from a root resource. */ + maxLineageDepth: number; + /** Maximum policy invocations per root within one orchestrator run. */ + maxPolicyRunsPerRoot: number; + /** Maximum derived artifacts per root within one orchestrator run. */ + maxArtifactsPerRoot: number; + /** Byte budget for derived artifacts per root within one run. */ + maxDerivedBytesPerRoot: number; + /** Maximum transport-guard passes per resource before explicit + * omission. */ + maxTransportPasses: number; +} + /** Normalized `omni.processing` view the pipeline consumes. */ export interface NormalizedOmniProcessingConfig { fixedPolicies: NormalizedFixedPolicy[]; transportGuardPolicies: NormalizedFixedPolicy[]; + limits: NormalizedOmniProcessingLimits; } /** Structural Config view for the processing config accessor (optional so From 5c4b8cffffb875c052bebe646de10f1227075789 Mon Sep 17 00:00:00 2001 From: LaZzyMan Date: Fri, 7 Aug 2026 12:01:40 +0800 Subject: [PATCH 10/62] feat(omni): enforce policy budgets, transport-guard passes and quarantine (Stage B) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - runFixedPolicies enforces maxPolicyRunsPerRoot (checked before each execution), maxArtifactsPerRoot and maxDerivedBytesPerRoot (checked as derivatives land); exhaustion records outcome 'budget_exhausted' while committed deliveries stand; maxLineageDepth clamps reprocessing. - Failed invocations quarantine their staging dir with a reason.json (policyId, toolName, reason, failedAt); user aborts and quarantine failures fall back to plain staging removal. Startup recovery receives quarantine retention/size settings from config. - Transport-guard violations now run modality-matched guard policies for up to maxTransportPasses passes on the final delivery; a still-violating resource is explicitly omitted (【媒体省略】notice replaces the part in both the read path and tool-result funnel) instead of delivered oversized; guard-pass failures stay fail-closed. Configs without a normalized processing config keep the Stage A throw. - Media-policy tools project a model-visible declaration (D6): locked arguments removed from properties/required, optional narrowing-only parameterSchema merge and description override; validation keeps the native schema. --- packages/core/src/config/config.ts | 2 +- packages/core/src/omni/disclosure.ts | 13 + packages/core/src/omni/index.test.ts | 258 +++++++++++++++++- packages/core/src/omni/index.ts | 162 +++++++++-- .../core/src/omni/policy/model-access.test.ts | 185 +++++++++++++ packages/core/src/omni/policy/model-access.ts | 85 ++++++ .../core/src/omni/policy/orchestrator.test.ts | 240 +++++++++++++++- packages/core/src/omni/policy/orchestrator.ts | 131 ++++++++- .../src/omni/policy/tools/downsample-audio.ts | 1 + .../src/omni/policy/tools/downsample-image.ts | 4 +- .../src/omni/policy/tools/downscale-video.ts | 1 + .../policy/tools/media-policy-tool.test.ts | 82 +++++- .../omni/policy/tools/media-policy-tool.ts | 33 +++ .../core/src/omni/tool-result-media.test.ts | 59 ++++ packages/core/src/omni/tool-result-media.ts | 12 +- 15 files changed, 1216 insertions(+), 52 deletions(-) diff --git a/packages/core/src/config/config.ts b/packages/core/src/config/config.ts index 7f2eeec585b..e570165fbb9 100644 --- a/packages/core/src/config/config.ts +++ b/packages/core/src/config/config.ts @@ -8125,7 +8125,7 @@ export class Config { const { OmniDownsampleImageTool } = await import( '../omni/policy/tools/downsample-image.js' ); - return new OmniDownsampleImageTool(); + return new OmniDownsampleImageTool(this); }); await registerLazy(ToolNames.OMNI_DOWNSCALE_VIDEO, async () => { const { OmniDownscaleVideoTool } = await import( diff --git a/packages/core/src/omni/disclosure.ts b/packages/core/src/omni/disclosure.ts index ba4d5a7a9fc..5b0bbcb6ae1 100644 --- a/packages/core/src/omni/disclosure.ts +++ b/packages/core/src/omni/disclosure.ts @@ -31,3 +31,16 @@ export function formatDisclosureText( export function isDisclosureText(text: string): boolean { return text.startsWith(OMNI_DISCLOSURE_TEXT_PREFIX); } + +/** Marks a text Part as an explicit-omission notice: the transport guard + * could not bring a resource within limits, so the media was withheld and + * this text stands in its place (policy design §10.2). */ +export const OMNI_OMISSION_TEXT_PREFIX = '【媒体省略】'; + +/** Model-facing omission notice for one withheld resource. */ +export function formatOmissionText( + displayName: string, + reason: string, +): string { + return `${OMNI_OMISSION_TEXT_PREFIX}${displayName}：${reason}`; +} diff --git a/packages/core/src/omni/index.test.ts b/packages/core/src/omni/index.test.ts index ee96ad3eb8c..2224ff5c71d 100644 --- a/packages/core/src/omni/index.test.ts +++ b/packages/core/src/omni/index.test.ts @@ -708,6 +708,7 @@ describe('processMediaForOmniDelivery fixed-policy integration', () => { vi.doUnmock('./storage.js'); vi.doUnmock('./upload.js'); vi.doUnmock('./policy/orchestrator.js'); + vi.doUnmock('./recovery.js'); await fs.rm(tmpDir, { recursive: true, force: true }); }); @@ -726,10 +727,26 @@ describe('processMediaForOmniDelivery fixed-policy integration', () => { // Only `.length > 0` matters to the pipeline; the mocked orchestrator // never reads the entries. const POLICY_STUB = [{ id: 'img-downsample' }]; + // Mirrors DEFAULT_OMNI_PROCESSING_LIMITS (normalization is unit-tested + // in policy/config.test.ts; the pipeline dereferences maxTransportPasses + // and forwards the object to the orchestrator). + const LIMITS_STUB = { + maxConcurrentResources: 1, + reservedOutputTokens: 8192, + maxLineageDepth: 8, + maxPolicyRunsPerRoot: 64, + maxArtifactsPerRoot: 256, + maxDerivedBytesPerRoot: 1073741824, + maxTransportPasses: 3, + }; function policyConfig(overrides?: { maxUploadFileBytes?: number; policies?: unknown[]; + transportGuardPolicies?: unknown[]; + maxTransportPasses?: number; + /** Simulates a stub/embedder config without the accessor. */ + noProcessingConfig?: boolean; }): Config { return { isOmniEnabled: vi.fn().mockReturnValue(true), @@ -740,10 +757,20 @@ describe('processMediaForOmniDelivery fixed-policy integration', () => { .fn() .mockReturnValue(overrides?.maxUploadFileBytes ?? 0), getOmniMaxEstimatedTokens: vi.fn().mockReturnValue(0), - getOmniProcessingConfig: vi.fn().mockReturnValue({ - fixedPolicies: overrides?.policies ?? POLICY_STUB, - transportGuardPolicies: [], - }), + getOmniProcessingConfig: vi.fn().mockReturnValue( + overrides?.noProcessingConfig + ? undefined + : { + fixedPolicies: overrides?.policies ?? POLICY_STUB, + transportGuardPolicies: overrides?.transportGuardPolicies ?? [], + limits: { + ...LIMITS_STUB, + ...(overrides?.maxTransportPasses !== undefined + ? { maxTransportPasses: overrides.maxTransportPasses } + : {}), + }, + }, + ), storage: { getQwenDir: () => tmpDir }, } as unknown as Config; } @@ -880,7 +907,10 @@ describe('processMediaForOmniDelivery fixed-policy integration', () => { expect(result.degraded).toBe(true); }); - it('still rejects when the FINAL delivery exceeds the byte cap', async () => { + it('explicitly omits an over-cap FINAL delivery when no guard policy matches its modality', async () => { + // Stage B (policy design §10.2): with a processing config present, a + // persisting violation is an explicit OMISSION, not a throw. The audio + // guard policy does not match an image, so no guard pass runs. const runMock = vi.fn().mockResolvedValue({ deliveries: [ { @@ -892,13 +922,41 @@ describe('processMediaForOmniDelivery fixed-policy integration', () => { ], records: [], }); + const { putFileMock, uploadFileMock, mod } = await armPipeline(runMock); + const result = await mod.processMediaForOmniDelivery( + await realFile('pic.png'), + policyConfig({ + maxUploadFileBytes: 500, + transportGuardPolicies: [{ id: 'guard-audio', mediaTypes: ['audio'] }], + }), + ); + // Only the fixed-policy stage ran — never a guard pass. + expect(runMock).toHaveBeenCalledTimes(1); + expect(result).toMatchObject({ + fileUri: '', + sha256: 'b'.repeat(64), + deduped: false, + uploadCacheHit: false, + degraded: true, + }); + expect(result.omission?.reason).toContain('900 bytes > 500 bytes'); + // Nothing was stored or uploaded for an omitted resource. + expect(putFileMock).not.toHaveBeenCalled(); + expect(uploadFileMock).not.toHaveBeenCalled(); + }); + + it('keeps the fail-closed throw when there is no processing config at all', async () => { + // Stub configs / embedders skipping initialize have no normalized + // processing config; the Stage A guard behavior must survive for them. + const runMock = vi.fn(); const { mod } = await armPipeline(runMock); await expect( mod.processMediaForOmniDelivery( await realFile('pic.png'), - policyConfig({ maxUploadFileBytes: 500 }), + policyConfig({ maxUploadFileBytes: 500, noProcessingConfig: true }), ), ).rejects.toMatchObject({ name: 'OmniTransportGuardError' }); + expect(runMock).not.toHaveBeenCalled(); }); it('wraps orchestrator failures into a sanitized OmniDeliveryError', async () => { @@ -966,4 +1024,192 @@ describe('processMediaForOmniDelivery fixed-policy integration', () => { }, }); }); + + // ── Stage B transport-guard pass loop ──────────────────────────────── + // With `policies: []` the fixed-policy stage is skipped entirely, so + // every runFixedPolicies call in these tests is a GUARD pass on the + // 5000-byte source (cap 500 → violation). + const IMG_GUARD = { id: 'img-guard', mediaTypes: ['image'] }; + + it('runs a matching guard policy on a violation and delivers the compliant result', async () => { + const guardedPath = path.join(tmpDir, 'objects', 'guarded.jpg'); + const runMock = vi.fn().mockResolvedValue({ + deliveries: [ + { + filePath: guardedPath, + recognized: DEGRADED_RECOGNIZED, + sha256: 'b'.repeat(64), + disclosure: 'downsampled to 1568px', + degraded: true, + }, + ], + records: [], + }); + const { mod } = await armPipeline(runMock); + const filePath = await realFile('pic.png'); + const config = policyConfig({ + policies: [], + maxUploadFileBytes: 500, + transportGuardPolicies: [IMG_GUARD], + }); + + const result = await mod.processMediaForOmniDelivery(filePath, config); + + // One guard pass over the SOURCE, restricted to the matching policies. + expect(runMock).toHaveBeenCalledTimes(1); + expect(runMock).toHaveBeenCalledWith( + config, + { + filePath, + recognized: SOURCE_RECOGNIZED, + displayName: 'pic.png', + origin: 'user', + }, + expect.objectContaining({ + policies: [IMG_GUARD], + limits: expect.objectContaining({ maxTransportPasses: 3 }), + }), + ); + expect(result.fileUri).toBe('oss://bucket/degraded'); + expect(result.omission).toBeUndefined(); + expect(result.degraded).toBe(true); + expect(result.disclosure).toBe('downsampled to 1568px'); + }); + + it('stops after maxTransportPasses passes and omits when still violating', async () => { + let call = 0; + const runMock = vi.fn().mockImplementation(async () => { + call += 1; + return { + deliveries: [ + { + // A NEW path every pass: progress is being made, so only the + // pass counter can end the loop. + filePath: path.join(tmpDir, 'objects', `pass-${call}.jpg`), + recognized: { ...DEGRADED_RECOGNIZED, sizeBytes: 900 }, + sha256: String(call).repeat(64).slice(0, 64), + degraded: true, + }, + ], + records: [], + }; + }); + const { mod } = await armPipeline(runMock); + const result = await mod.processMediaForOmniDelivery( + await realFile('pic.png'), + policyConfig({ + policies: [], + maxUploadFileBytes: 500, + maxTransportPasses: 2, + transportGuardPolicies: [IMG_GUARD], + }), + ); + expect(runMock).toHaveBeenCalledTimes(2); + expect(result.omission?.reason).toContain('900 bytes > 500 bytes'); + expect(result.fileUri).toBe(''); + }); + + it('breaks out of the guard loop when a pass makes no progress', async () => { + // Every guard policy no_op'd: the delivery IS the input resource. A + // second pass would repeat identical work forever. + const runMock = vi.fn().mockImplementation(async (_config, resource) => ({ + deliveries: [ + { filePath: resource.filePath, recognized: resource.recognized }, + ], + records: [], + })); + const { mod } = await armPipeline(runMock); + const result = await mod.processMediaForOmniDelivery( + await realFile('pic.png'), + policyConfig({ + policies: [], + maxUploadFileBytes: 500, + transportGuardPolicies: [IMG_GUARD], + }), + ); + expect(runMock).toHaveBeenCalledTimes(1); + expect(result.omission?.reason).toContain('5000 bytes > 500 bytes'); + }); + + it('fails closed when a guard pass itself fails', async () => { + // A guard configuration error must never degrade into sending + // over-limit media (policy design §10.2). + const runMock = vi.fn().mockRejectedValue(new Error('guard blew up')); + const { mod } = await armPipeline(runMock); + await expect( + mod.processMediaForOmniDelivery( + await realFile('pic.png'), + policyConfig({ + policies: [], + maxUploadFileBytes: 500, + transportGuardPolicies: [IMG_GUARD], + }), + ), + ).rejects.toMatchObject({ + name: 'OmniDeliveryError', + message: 'Transport-guard processing failed for pic.png: guard blew up', + }); + }); + + it('readMediaViaOmniDelivery renders an omission as the notice text, not an error', async () => { + const runMock = vi.fn().mockResolvedValue({ + deliveries: [ + { + filePath: path.join(tmpDir, 'objects', 'deadbeef.jpg'), + recognized: { ...DEGRADED_RECOGNIZED, sizeBytes: 900 }, + sha256: 'b'.repeat(64), + degraded: true, + }, + ], + records: [], + }); + const { mod } = await armPipeline(runMock); + const result = await mod.readMediaViaOmniDelivery({ + filePath: await realFile('pic.png'), + config: policyConfig({ maxUploadFileBytes: 500 }), + displayName: 'pic.png', + relativePathForDisplay: 'pic.png', + expectedModality: 'image', + }); + expect(typeof result.llmContent).toBe('string'); + expect(result.llmContent).toMatch(/^【媒体省略】pic\.png：/); + expect(result.llmContent).toContain('900 bytes > 500 bytes'); + expect(result.returnDisplay).toBe( + 'Media omitted by the omni transport guard: pic.png', + ); + expect(result.error).toBeUndefined(); + expect(result.errorType).toBeUndefined(); + }); + + it('threads the quarantine retention settings into startup recovery', async () => { + const recoveryMock = vi.fn().mockResolvedValue(undefined); + vi.doMock('./recovery.js', () => ({ + runStartupRecoveryOnce: recoveryMock, + resetRecoveryLatchForTests: vi.fn(), + })); + const runMock = vi.fn().mockResolvedValue({ + deliveries: [ + { + filePath: path.join(tmpDir, 'objects', 'deadbeef.jpg'), + recognized: DEGRADED_RECOGNIZED, + sha256: 'b'.repeat(64), + }, + ], + records: [], + }); + const { mod } = await armPipeline(runMock); + const config = { + ...policyConfig(), + getOmniQuarantineRetentionDays: () => 3, + getOmniQuarantineMaxBytes: () => 1024, + } as unknown as Config; + + await mod.processMediaForOmniDelivery(await realFile('pic.png'), config); + + expect(recoveryMock).toHaveBeenCalledWith( + expect.anything(), + expect.anything(), + { quarantineRetentionDays: 3, quarantineMaxBytes: 1024 }, + ); + }); }); diff --git a/packages/core/src/omni/index.ts b/packages/core/src/omni/index.ts index dab019d37c6..a0880bd5276 100644 --- a/packages/core/src/omni/index.ts +++ b/packages/core/src/omni/index.ts @@ -14,11 +14,15 @@ import { ToolErrorType } from '../tools/tool-error.js'; import { createDebugLogger } from '../utils/debugLogger.js'; import { isAbortError } from '../utils/errors.js'; import { isFfmpegAvailable, isFfprobeAvailable } from './ffmpeg.js'; -import type { OmniTokenEstimate } from './estimation.js'; +import { + estimateRawResourceTokens, + type OmniTokenEstimate, +} from './estimation.js'; import { assertWithinByteLimit, assertWithinTokenLimit, effectiveMaxUploadFileBytes, + OmniTransportGuardError, } from './guard.js'; import { extensionForMime, @@ -34,7 +38,7 @@ import { DEFAULT_UPLOAD_CACHE_TTL_HOURS, } from './upload-cache.js'; import { runStartupRecoveryOnce } from './recovery.js'; -import { formatDisclosureText } from './disclosure.js'; +import { formatDisclosureText, formatOmissionText } from './disclosure.js'; import { runFixedPolicies, type PolicyDeliveryResource, @@ -86,7 +90,9 @@ export { export { resetCredentialCacheForTests } from './upload.js'; export { OMNI_DISCLOSURE_TEXT_PREFIX, + OMNI_OMISSION_TEXT_PREFIX, formatDisclosureText, + formatOmissionText, isDisclosureText, } from './disclosure.js'; export { @@ -168,6 +174,11 @@ export interface OmniMediaDelivery { /** True when a fixed policy replaced the source with a lossy * derivative. */ degraded?: boolean; + /** Present when the transport guard could not bring the resource within + * limits even after the transport-guard policies ran: the media was NOT + * uploaded (`fileUri` is empty) and callers must materialize an + * explicit-omission text Part in its place (policy design §10.2). */ + omission?: { reason: string }; } /** Thrown for omni pipeline failures. The pipeline fails closed: callers @@ -227,6 +238,31 @@ export function isOmniDeliveryActive(config: Config): boolean { return DashScopeOpenAICompatibleProvider.isDashScopeProvider(cgc); } +/** Non-throwing transport-limit check: runs both guard dimensions and + * reports the first violation as a message instead of an exception, so + * the Stage B guard loop can react (run guard policies / omit) while + * configs without a processing config keep the fail-closed throw. */ +function evaluateTransportLimits( + config: Config, + recognized: RecognizedMedia, + displayName: string, +): { estimate: OmniTokenEstimate; violation?: string } { + try { + assertWithinByteLimit(config, recognized.sizeBytes, displayName); + return { + estimate: assertWithinTokenLimit(config, recognized, displayName), + }; + } catch (err) { + if (err instanceof OmniTransportGuardError) { + return { + estimate: estimateRawResourceTokens(recognized), + violation: err.message, + }; + } + throw err; + } +} + /** * Omni pipeline: recognize → fixed policies (degradation) → transport * guard → hash → promote into the content-addressed store → upload via the @@ -323,18 +359,23 @@ export async function processMediaForOmniDelivery( cacheScope, ); // Lazy one-time hygiene scan (expired .part files, promotion orphans, - // sampled object verification). MUST run before the orchestrator: the - // scan deletes staging/ wholesale, which would race live invocations. - await runStartupRecoveryOnce(store, uploadCache); + // quarantine retention/size sweeps, sampled object verification). MUST + // run before the orchestrator: the scan deletes staging/ wholesale, + // which would race live invocations. + await runStartupRecoveryOnce(store, uploadCache, { + quarantineRetentionDays: config.getOmniQuarantineRetentionDays?.(), + quarantineMaxBytes: config.getOmniQuarantineMaxBytes?.(), + }); // Fixed-policy preprocessing (decision D5: this single site covers // @-commands, tool results, the URL funnel and ACP). Structural view — - // the real accessor arrives with config normalization; a config without - // it (or with no policies) changes nothing. + // a config without the accessor (stub configs, embedders skipping + // initialize) or with no policies changes nothing. + const processingConfig = ( + config as OmniProcessingConfigView + ).getOmniProcessingConfig?.(); let final: PolicyDeliveryResource = { filePath, recognized }; - const policies = - (config as OmniProcessingConfigView).getOmniProcessingConfig?.() - ?.fixedPolicies ?? []; + const policies = processingConfig?.fixedPolicies ?? []; if (policies.length > 0) { let deliveries: PolicyDeliveryResource[]; try { @@ -346,7 +387,7 @@ export async function processMediaForOmniDelivery( displayName, origin: options?.origin ?? 'user', }, - { store, policies, signal }, + { store, policies, signal, limits: processingConfig?.limits }, )); } catch (err) { if (signal?.aborted) throw err; @@ -368,13 +409,88 @@ export async function processMediaForOmniDelivery( } // Transport guard on the FINAL delivery set (decision D1): the bytes - // and token estimate judged are the ones actually delivered. - assertWithinByteLimit(config, final.recognized.sizeBytes, displayName); - const tokenEstimate = assertWithinTokenLimit( - config, - final.recognized, - displayName, - ); + // and token estimate judged are the ones actually delivered. Stage B: + // a violation first runs the transport-guard policies (matched by + // modality only — no `when`, coverage of all three modalities is + // enforced at config normalization) for up to + // `limits.maxTransportPasses` passes; a still-over-limit resource is + // explicitly OMITTED (policy design §10.2) rather than delivered + // oversized. Without a normalized processing config (stub configs, + // embedders skipping initialize) the guard keeps its fail-closed throw. + let guard = evaluateTransportLimits(config, final.recognized, displayName); + if (guard.violation && processingConfig) { + const guardPolicies = processingConfig.transportGuardPolicies.filter((p) => + p.mediaTypes.includes(final.recognized.modality), + ); + const maxPasses = processingConfig.limits.maxTransportPasses; + for ( + let pass = 0; + guard.violation && guardPolicies.length > 0 && pass < maxPasses; + pass++ + ) { + let deliveries: PolicyDeliveryResource[]; + try { + ({ deliveries } = await runFixedPolicies( + config, + { + filePath: final.filePath, + recognized: final.recognized, + displayName, + origin: options?.origin ?? 'user', + }, + { + store, + policies: guardPolicies, + signal, + limits: processingConfig.limits, + }, + )); + } catch (err) { + if (signal?.aborted) throw err; + // Guard-policy failure with no compliant alternative: fail closed + // — a guard configuration error must never degrade into sending + // over-limit media (policy design §10.2). + throw new OmniDeliveryError( + `Transport-guard processing failed for ${displayName}: ` + + `${sanitizeErrorMessage(err, [final.filePath, store.getOmniRootDir()])}`, + { cause: err }, + ); + } + if (deliveries.length !== 1) { + throw new OmniDeliveryError( + `Transport-guard policies produced ${deliveries.length} deliverables for ${displayName}; exactly one is supported.`, + ); + } + if (deliveries[0].filePath === final.filePath) { + // No progress (every guard policy was a no_op for this input) — + // further passes would repeat the same work. + break; + } + final = deliveries[0]; + guard = evaluateTransportLimits(config, final.recognized, displayName); + } + } + if (guard.violation) { + if (!processingConfig) { + throw new OmniTransportGuardError(guard.violation); + } + debugLogger.debug( + `omni ${final.recognized.modality} explicitly omitted (transport guard): ${guard.violation}`, + ); + return { + fileUri: '', + mimeType: final.recognized.detectedMimeType, + sha256: final.sha256 ?? '', + recognized: final.recognized, + tokenEstimate: guard.estimate, + deduped: false, + uploadCacheHit: false, + disclosure: final.disclosure, + degraded: final.degraded, + omission: { reason: guard.violation }, + }; + } + const tokenEstimate = guard.estimate; // Content hash: identity of the stored object. Derivatives arrive with // their hash from promotion; sources are hashed here, after all guards. @@ -525,6 +641,16 @@ export async function readMediaViaOmniDelivery(params: { expectedModality, signal, }); + if (delivery.omission) { + // Explicit omission (policy design §10.2): the media is withheld and + // the omission notice text stands in its place. Not an error — the + // read succeeded; the transport guard's verdict is the content. + return { + llmContent: formatOmissionText(displayName, delivery.omission.reason), + returnDisplay: `Media omitted by the omni transport guard: ${relativePathForDisplay}`, + tokenEstimate: delivery.tokenEstimate, + }; + } const fileDataPart = { fileData: { fileUri: delivery.fileUri, diff --git a/packages/core/src/omni/policy/model-access.test.ts b/packages/core/src/omni/policy/model-access.test.ts index df38629e06d..75384d2d2a4 100644 --- a/packages/core/src/omni/policy/model-access.test.ts +++ b/packages/core/src/omni/policy/model-access.test.ts @@ -10,6 +10,7 @@ import type { OmniPolicyToolsSettings } from './types.js'; import { evaluateMediaPolicyToolCall, isMediaPolicyToolHiddenFromModel, + projectMediaPolicyToolDeclaration, resolveMediaPolicyModelAccess, type MediaPolicyConfigView, } from './model-access.js'; @@ -102,6 +103,190 @@ describe('resolveMediaPolicyModelAccess', () => { resolveMediaPolicyModelAccess(config, 'omni_compress_image'), ).toEqual({ enabled: true, defaultArguments: {}, lockedArguments: {} }); }); + + it('reads description and parameterSchema when well-formed', () => { + const config = configWith({ + omni_compress_image: { + modelAccess: { + enabled: true, + description: 'Compress an image.', + parameterSchema: { properties: { quality: { maximum: 90 } } }, + }, + }, + }); + const access = resolveMediaPolicyModelAccess(config, 'omni_compress_image'); + expect(access.description).toBe('Compress an image.'); + expect(access.parameterSchema).toEqual({ + properties: { quality: { maximum: 90 } }, + }); + }); + + it.each([ + ['empty description', { description: '' }], + ['non-string description', { description: 42 }], + ['array parameterSchema', { parameterSchema: [] }], + ['string parameterSchema', { parameterSchema: '{}' }], + ])('drops a malformed declaration projection: %s', (_label, modelAccess) => { + const config = configWith({ + omni_compress_image: { modelAccess }, + } as unknown as OmniPolicyToolsSettings); + const access = resolveMediaPolicyModelAccess(config, 'omni_compress_image'); + expect(access.description).toBeUndefined(); + expect(access.parameterSchema).toBeUndefined(); + }); +}); + +describe('projectMediaPolicyToolDeclaration', () => { + const NATIVE = { + name: 'omni_compress_image', + description: 'Native description.', + parametersJsonSchema: { + type: 'object', + properties: { + inputPath: { type: 'string', description: 'Source path.' }, + outputDir: { type: 'string' }, + maxDimension: { type: 'number', minimum: 1 }, + quality: { type: 'number', minimum: 1, maximum: 100 }, + }, + required: ['inputPath', 'outputDir'], + additionalProperties: false, + }, + }; + + it('returns the native declaration unchanged without modelAccess settings', () => { + expect(projectMediaPolicyToolDeclaration({}, NATIVE)).toEqual(NATIVE); + }); + + it('removes lockedArguments keys from properties AND required', () => { + const config = configWith({ + omni_compress_image: { + modelAccess: { + enabled: true, + lockedArguments: { outputDir: '/staging' }, + }, + }, + }); + expect(projectMediaPolicyToolDeclaration(config, NATIVE)).toEqual({ + name: 'omni_compress_image', + description: 'Native description.', + parametersJsonSchema: { + type: 'object', + properties: { + inputPath: { type: 'string', description: 'Source path.' }, + maxDimension: { type: 'number', minimum: 1 }, + quality: { type: 'number', minimum: 1, maximum: 100 }, + }, + required: ['inputPath'], + additionalProperties: false, + }, + }); + }); + + it('narrows to parameterSchema properties, merging overrides over native constraints', () => { + const config = configWith({ + omni_compress_image: { + modelAccess: { + enabled: true, + lockedArguments: { inputPath: '/x', outputDir: '/y' }, + parameterSchema: { + properties: { + maxDimension: { maximum: 4096, description: 'Longest edge.' }, + }, + }, + }, + }, + }); + expect(projectMediaPolicyToolDeclaration(config, NATIVE)).toEqual({ + name: 'omni_compress_image', + description: 'Native description.', + parametersJsonSchema: { + type: 'object', + properties: { + maxDimension: { + type: 'number', // native constraint preserved… + minimum: 1, + maximum: 4096, // …override merged on top + description: 'Longest edge.', + }, + }, + required: [], + additionalProperties: false, + }, + }); + }); + + it('is narrowing-only: a projection property with no native counterpart is ignored', () => { + const config = configWith({ + omni_compress_image: { + modelAccess: { + enabled: true, + parameterSchema: { + properties: { + quality: {}, + madeUp: { type: 'string' }, + }, + }, + }, + }, + }); + const declaration = projectMediaPolicyToolDeclaration(config, NATIVE); + const schema = declaration.parametersJsonSchema as { + properties: Record; + }; + expect(Object.keys(schema.properties)).toEqual(['quality']); + }); + + it('never re-adds a locked key even when parameterSchema names it', () => { + const config = configWith({ + omni_compress_image: { + modelAccess: { + enabled: true, + lockedArguments: { outputDir: '/staging' }, + parameterSchema: { + properties: { outputDir: {}, quality: {} }, + }, + }, + }, + }); + const declaration = projectMediaPolicyToolDeclaration(config, NATIVE); + const schema = declaration.parametersJsonSchema as { + properties: Record; + }; + expect(Object.keys(schema.properties)).toEqual(['quality']); + }); + + it('overrides the description when configured', () => { + const config = configWith({ + omni_compress_image: { + modelAccess: { enabled: true, description: 'Model-facing text.' }, + }, + }); + expect(projectMediaPolicyToolDeclaration(config, NATIVE).description).toBe( + 'Model-facing text.', + ); + }); + + it('passes a non-record native schema through, still applying the description override', () => { + const config = configWith({ + omni_compress_image: { + modelAccess: { + enabled: true, + description: 'Overridden.', + lockedArguments: { outputDir: '/staging' }, + }, + }, + }); + const native = { + name: 'omni_compress_image', + description: 'Native description.', + parametersJsonSchema: undefined, + }; + expect(projectMediaPolicyToolDeclaration(config, native)).toEqual({ + name: 'omni_compress_image', + description: 'Overridden.', + parametersJsonSchema: undefined, + }); + }); }); describe('isMediaPolicyToolHiddenFromModel', () => { diff --git a/packages/core/src/omni/policy/model-access.ts b/packages/core/src/omni/policy/model-access.ts index bd169ab10db..cf6db20fd9d 100644 --- a/packages/core/src/omni/policy/model-access.ts +++ b/packages/core/src/omni/policy/model-access.ts @@ -4,6 +4,7 @@ * SPDX-License-Identifier: Apache-2.0 */ +import type { FunctionDeclaration } from '@google/genai'; import type { ToolExecutionOrigin } from '../../core/turn.js'; import type { MediaPolicyToolDescriptor } from '../../tools/tools.js'; import type { @@ -35,6 +36,10 @@ export interface ResolvedMediaPolicyModelAccess { enabled: boolean; defaultArguments: Record; lockedArguments: Record; + /** Model-facing description override for the declaration projection. */ + description?: string; + /** Narrowing-only projection over the native parameter schema. */ + parameterSchema?: Record; } const isPlainRecord = (value: unknown): value is Record => @@ -62,6 +67,86 @@ export function resolveMediaPolicyModelAccess( lockedArguments: isPlainRecord(modelAccess?.lockedArguments) ? modelAccess.lockedArguments : {}, + description: + typeof modelAccess?.description === 'string' && + modelAccess.description !== '' + ? modelAccess.description + : undefined, + parameterSchema: isPlainRecord(modelAccess?.parameterSchema) + ? modelAccess.parameterSchema + : undefined, + }; +} + +/** + * Model-visible declaration for a media-policy tool (decision D6, policy + * design §9.4): the projection is applied at the SINGLE `schema` getter + * the declaration surfaces read, while validation keeps using the native + * schema (the harness-injected arguments the projection hides must stay + * valid). + * + * Shape: the native parametersJsonSchema minus every + * `modelAccess.lockedArguments` key (removed from `properties` and + * `required` — the model must not see arguments it is forbidden to pass), + * then — when `modelAccess.parameterSchema` is configured — narrowed to + * the properties it names, with each named property's constraints merged + * over the native ones. A projection property with no native counterpart + * is ignored (narrowing-only: the projection can never ADD surface). + * `modelAccess.description` overrides the tool description when set. + */ +export function projectMediaPolicyToolDeclaration( + config: MediaPolicyConfigView, + native: { + name: string; + description: string; + parametersJsonSchema: unknown; + }, +): FunctionDeclaration { + const access = resolveMediaPolicyModelAccess(config, native.name); + const description = access.description ?? native.description; + const schema = isPlainRecord(native.parametersJsonSchema) + ? native.parametersJsonSchema + : undefined; + const nativeProps = + schema && isPlainRecord(schema['properties']) + ? schema['properties'] + : undefined; + if (!schema || !nativeProps) { + return { + name: native.name, + description, + parametersJsonSchema: native.parametersJsonSchema, + }; + } + const lockedKeys = new Set(Object.keys(access.lockedArguments)); + const narrowProps = + access.parameterSchema && + isPlainRecord(access.parameterSchema['properties']) + ? (access.parameterSchema['properties'] as Record) + : undefined; + const properties: Record = {}; + for (const [key, value] of Object.entries(nativeProps)) { + if (lockedKeys.has(key)) continue; + if (narrowProps && !(key in narrowProps)) continue; + const override = narrowProps?.[key]; + properties[key] = + isPlainRecord(override) && isPlainRecord(value) + ? { ...value, ...override } + : value; + } + const required = Array.isArray(schema['required']) + ? (schema['required'] as unknown[]).filter( + (key): key is string => typeof key === 'string' && key in properties, + ) + : undefined; + return { + name: native.name, + description, + parametersJsonSchema: { + ...schema, + properties, + ...(required !== undefined ? { required } : {}), + }, }; } diff --git a/packages/core/src/omni/policy/orchestrator.test.ts b/packages/core/src/omni/policy/orchestrator.test.ts index be8a263086c..740e7f206ab 100644 --- a/packages/core/src/omni/policy/orchestrator.test.ts +++ b/packages/core/src/omni/policy/orchestrator.test.ts @@ -23,7 +23,10 @@ import { runFixedPolicies, type PolicySourceResource, } from './orchestrator.js'; -import type { NormalizedFixedPolicy } from './types.js'; +import type { + NormalizedFixedPolicy, + NormalizedOmniProcessingLimits, +} from './types.js'; // The orchestrator resolves the executor with a dynamic import; vitest // intercepts it the same as a static one. @@ -107,6 +110,22 @@ function makeConfig( } as unknown as Config; } +/** System defaults (P §12.2) with per-test overrides. */ +function limitsWith( + overrides: Partial, +): NormalizedOmniProcessingLimits { + return { + maxConcurrentResources: 1, + reservedOutputTokens: 8192, + maxLineageDepth: 8, + maxPolicyRunsPerRoot: 64, + maxArtifactsPerRoot: 256, + maxDerivedBytesPerRoot: 1073741824, + maxTransportPasses: 3, + ...overrides, + }; +} + describe('runFixedPolicies', () => { let tmpDir: string; let store: OmniObjectStore; @@ -616,4 +635,223 @@ describe('runFixedPolicies', () => { }); expect(order).toEqual(['a-high', 'a-low', 'b-low']); }); + + it('stops BEFORE executing once maxPolicyRunsPerRoot is spent, recording budget_exhausted', async () => { + mockToolSuccess(); + // Distinct arguments: identical ones would fingerprint identically and + // make the second policy a (budget-free) degradation-cache hit. + const { deliveries, records } = await runFixedPolicies(config, source, { + store, + policies: [ + makePolicy({ + id: 'a-first', + arguments: { maxDimension: 100 }, + output: { reprocessMedia: false, source: 'keep' }, + }), + makePolicy({ + id: 'b-second', + arguments: { maxDimension: 200 }, + output: { reprocessMedia: false, source: 'keep' }, + }), + ], + limits: limitsWith({ maxPolicyRunsPerRoot: 1 }), + }); + expect(executeToolCallMock).toHaveBeenCalledTimes(1); + expect(records).toEqual([ + { + policyId: 'a-first', + toolName: 'omni_downsample_image', + outcome: 'succeeded', + resource: 'photo.png', + }, + { + policyId: 'b-second', + toolName: 'omni_downsample_image', + outcome: 'budget_exhausted', + resource: 'photo.png', + error: 'maxPolicyRunsPerRoot (1) reached', + }, + ]); + // The committed delivery stands (no rollback): source + derivative. + expect(deliveries.map((d) => d.filePath)).toEqual([ + sourcePath, + store.objectPathFor(sha256Of(DEGRADED_BYTES), '.jpg'), + ]); + }); + + it('stops deriving when maxArtifactsPerRoot is exceeded but keeps the committed delivery', async () => { + mockToolSuccess(); + const { deliveries, records } = await runFixedPolicies(config, source, { + store, + policies: [ + makePolicy(), + makePolicy({ id: 'never-runs', arguments: { maxDimension: 300 } }), + ], + limits: limitsWith({ maxArtifactsPerRoot: 0 }), + }); + expect(executeToolCallMock).toHaveBeenCalledTimes(1); + expect(records).toEqual([ + { + policyId: 'img-downsample', + toolName: 'omni_downsample_image', + outcome: 'succeeded', + resource: 'photo.png', + }, + { + policyId: 'img-downsample', + toolName: 'omni_downsample_image', + outcome: 'budget_exhausted', + resource: 'photo.png', + error: 'maxArtifactsPerRoot (0) exceeded', + }, + ]); + // source: 'omit' already applied — the derivative alone is delivered. + expect(deliveries).toHaveLength(1); + expect(deliveries[0].degraded).toBe(true); + }); + + it('stops deriving when maxDerivedBytesPerRoot is exceeded', async () => { + mockToolSuccess(); // DEGRADED_BYTES is 20 bytes > the 10-byte budget + const { deliveries, records } = await runFixedPolicies(config, source, { + store, + policies: [ + makePolicy(), + makePolicy({ id: 'never-runs', arguments: { maxDimension: 300 } }), + ], + limits: limitsWith({ maxDerivedBytesPerRoot: 10 }), + }); + expect(executeToolCallMock).toHaveBeenCalledTimes(1); + expect(records[1]).toEqual({ + policyId: 'img-downsample', + toolName: 'omni_downsample_image', + outcome: 'budget_exhausted', + resource: 'photo.png', + error: 'maxDerivedBytesPerRoot (10) exceeded', + }); + expect(deliveries).toHaveLength(1); + expect(deliveries[0].degraded).toBe(true); + }); + + it('clamps reprocessing at maxLineageDepth even when lineage runs remain', async () => { + // Distinct bytes per run: identical output would end the chain as a + // no_op fixed point before the depth clamp could matter. + let round = 0; + executeToolCallMock.mockImplementation( + async (_config: Config, request: ToolCallRequestInfo) => { + round++; + const outputDir = request.args['outputDir'] as string; + await fs.writeFile(path.join(outputDir, 'out.jpg'), `round-${round}`); + return { + callId: request.callId, + responseParts: [], + resultDisplay: undefined, + error: undefined, + errorType: undefined, + policyArtifacts: { + toolName: request.name, + invocationId: request.callId, + executionOrigin: request.executionOrigin, + artifacts: [ + { + kind: 'image', + storage: 'workspace', + title: 'out.jpg', + workspacePath: 'out.jpg', + mimeType: 'image/jpeg', + metadata: { omniDisclosure: `round ${round}` }, + }, + ], + }, + }; + }, + ); + const { deliveries } = await runFixedPolicies(config, source, { + store, + policies: [ + makePolicy({ + origins: ['user', 'tool', 'policy'], + maxRunsPerLineage: 10, + output: { reprocessMedia: true, source: 'omit' }, + }), + ], + limits: limitsWith({ maxLineageDepth: 2 }), + }); + // root(depth 0) → run 1 → depth-1 child re-enters → run 2 → the + // depth-2 child delivers but does NOT re-enter (2 is not < 2). The + // lineage cap alone (10) would have allowed further runs. + expect(executeToolCallMock).toHaveBeenCalledTimes(2); + expect(deliveries).toHaveLength(1); + expect(deliveries[0].sha256).toBe(sha256Of('round-2')); + }); + + it('quarantines the staging dir with a reason.json when the invocation fails (D10 Stage B)', async () => { + executeToolCallMock.mockResolvedValue({ + callId: 'x', + responseParts: [], + resultDisplay: undefined, + error: new Error('ffmpeg exploded'), + errorType: undefined, + }); + await runFixedPolicies(config, source, { + store, + policies: [makePolicy()], + }); + await expect(fs.readdir(store.getStagingDir())).resolves.toEqual([]); + const quarantined = await fs.readdir(store.getQuarantineDir()); + expect(quarantined).toHaveLength(1); + const reason = JSON.parse( + await fs.readFile( + path.join(store.getQuarantineDir(), quarantined[0], 'reason.json'), + 'utf8', + ), + ) as Record; + expect(reason).toMatchObject({ + policyId: 'img-downsample', + toolName: 'omni_downsample_image', + reason: 'ffmpeg exploded', + }); + expect(typeof reason['failedAt']).toBe('string'); + }); + + it('removes (not quarantines) staging when the failure is a user abort', async () => { + const controller = new AbortController(); + executeToolCallMock.mockImplementation(async () => { + controller.abort(); + throw new Error('aborted mid-flight'); + }); + await expect( + runFixedPolicies(config, source, { + store, + policies: [makePolicy()], + signal: controller.signal, + }), + ).rejects.toThrow('aborted mid-flight'); + await expect(fs.readdir(store.getStagingDir())).resolves.toEqual([]); + await expect( + fs.readdir(store.getQuarantineDir()).catch(() => []), + ).resolves.toEqual([]); + }); + + it('falls back to plain staging removal when quarantining itself fails', async () => { + vi.spyOn(store, 'quarantineInvocation').mockRejectedValue( + new Error('quarantine disk full'), + ); + executeToolCallMock.mockResolvedValue({ + callId: 'x', + responseParts: [], + resultDisplay: undefined, + error: new Error('ffmpeg exploded'), + errorType: undefined, + }); + const { records } = await runFixedPolicies(config, source, { + store, + policies: [makePolicy()], + }); + expect(records[0]).toMatchObject({ + outcome: 'failed', + error: 'ffmpeg exploded', + }); + // The failed invocation still never leaves live staging state behind. + await expect(fs.readdir(store.getStagingDir())).resolves.toEqual([]); + }); }); diff --git a/packages/core/src/omni/policy/orchestrator.ts b/packages/core/src/omni/policy/orchestrator.ts index ccfc328c42c..d3363aa117e 100644 --- a/packages/core/src/omni/policy/orchestrator.ts +++ b/packages/core/src/omni/policy/orchestrator.ts @@ -31,7 +31,12 @@ import { computePolicyFingerprint, OmniDegradationCache, } from './degradation-cache.js'; -import type { FixedPolicyOrigin, NormalizedFixedPolicy } from './types.js'; +import { DEFAULT_OMNI_PROCESSING_LIMITS } from './config.js'; +import type { + FixedPolicyOrigin, + NormalizedFixedPolicy, + NormalizedOmniProcessingLimits, +} from './types.js'; const debugLogger = createDebugLogger('omni:policy'); @@ -62,7 +67,8 @@ export interface PolicyRunRecord { | 'cache_hit' | 'no_op' | 'failed' - | 'condition_unavailable'; + | 'condition_unavailable' + | 'budget_exhausted'; /** Display label of the resource the policy ran against. */ resource: string; /** Fields that made a `when` condition undecidable. */ @@ -79,6 +85,9 @@ export interface RunFixedPoliciesOptions { conditionContext?: Pick; /** Injectable for tests; defaults to the store-rooted cache. */ degradationCache?: OmniDegradationCache; + /** Per-root derivation budgets (decision D11); the system defaults + * apply when the caller has no normalized processing config. */ + limits?: NormalizedOmniProcessingLimits; } /** Root resource entering the orchestrator. */ @@ -115,6 +124,8 @@ interface WorkItem { /** Per-derivation-chain run counts (policy id → runs). Copied — never * shared — on derivation, so sibling branches cap independently. */ lineageRuns: Map; + /** Derivation-chain length from the root (root = 0). */ + depth: number; deliver: boolean; /** Whether the item enters policy matching (`output.reprocessMedia`). */ process: boolean; @@ -183,15 +194,23 @@ function sortPolicies( * descriptor, promote them into the content-addressed store, and return * the final delivery set plus records of the work performed. * - * Termination is structural: each policy runs at most `maxRunsPerLineage` - * times per derivation chain and the policy set is finite, so the derived - * tree is finite (global budgets are the next commit's backstop). + * Termination is structural AND budgeted: each policy runs at most + * `maxRunsPerLineage` times per derivation chain and the policy set is + * finite, so the derived tree is finite; on top of that the per-root + * budgets (decision D11 — `maxPolicyRunsPerRoot`, `maxArtifactsPerRoot`, + * `maxDerivedBytesPerRoot`, `maxLineageDepth`) stop further derivation + * when exceeded. A budget stop is not a failure: already-committed + * delivery decisions stand (no rollback), the stop is recorded with the + * exhausted budget as its reason, and the transport guard still judges + * the final set. * * Failure semantics (decision D10): a failed invocation never leaves - * partial state (its staging dir is removed); `onFailure: 'continue'` - * keeps the source in the delivery set (the transport guard remains the - * backstop), while `'abort'` — and any transport-guard-stage failure — - * throws {@link OmniPolicyExecutionError}. + * partial state in staging/ — its staging directory is moved to + * quarantine/ with a `reason.json` for postmortem (Stage B; sweeps apply + * retention). `onFailure: 'continue'` keeps the source in the delivery + * set (the transport guard remains the backstop), while `'abort'` — and + * any transport-guard-stage failure — throws + * {@link OmniPolicyExecutionError}. */ export async function runFixedPolicies( config: Config, @@ -202,6 +221,7 @@ export async function runFixedPolicies( records: PolicyRunRecord[]; }> { const policies = sortPolicies(options.policies); + const limits = options.limits ?? DEFAULT_OMNI_PROCESSING_LIMITS; const cache = options.degradationCache ?? new OmniDegradationCache(options.store.getOmniRootDir()); @@ -213,13 +233,39 @@ export async function runFixedPolicies( label: source.displayName, origin: source.origin, lineageRuns: new Map(), + depth: 0, deliver: true, process: true, }, ]; + // Per-root budget counters (decision D11). One runFixedPolicies call + // processes exactly one root, so the counters live here. + let runsUsed = 0; + let artifactsProduced = 0; + let derivedBytesProduced = 0; + let budgetExhausted = false; + const stopOnBudget = ( + policy: NormalizedFixedPolicy, + item: WorkItem, + reason: string, + ): void => { + budgetExhausted = true; + records.push({ + policyId: policy.id, + toolName: policy.toolName, + outcome: 'budget_exhausted', + resource: item.label, + error: reason, + }); + debugLogger.debug( + `per-root policy budget exhausted on ${item.label}: ${reason}; ` + + `no further derivation for this root (committed deliveries stand)`, + ); + }; + // Index-based: executions append derived items behind the cursor. - for (let i = 0; i < items.length; i++) { + for (let i = 0; i < items.length && !budgetExhausted; i++) { const item = items[i]; if (!item.process) continue; for (const policy of policies) { @@ -248,6 +294,15 @@ export async function runFixedPolicies( } // 'unavailable' + onConditionUnavailable 'run' falls through. } + if (runsUsed >= limits.maxPolicyRunsPerRoot) { + stopOnBudget( + policy, + item, + `maxPolicyRunsPerRoot (${limits.maxPolicyRunsPerRoot}) reached`, + ); + break; + } + runsUsed++; item.lineageRuns.set(policy.id, runs + 1); try { const execution = await executePolicy( @@ -266,16 +321,43 @@ export async function runFixedPolicies( }); if (execution.outcome === 'no_op') continue; if (policy.output.source === 'omit') item.deliver = false; + const childDepth = item.depth + 1; + const depthAllowsReprocess = childDepth < limits.maxLineageDepth; + if (policy.output.reprocessMedia && !depthAllowsReprocess) { + debugLogger.debug( + `maxLineageDepth (${limits.maxLineageDepth}) reached under ${item.label}; ` + + `derivatives deliver but do not re-enter policy matching`, + ); + } for (const derived of execution.derived) { + artifactsProduced++; + derivedBytesProduced += derived.recognized.sizeBytes; items.push({ ...derived, label: `${item.label} → ${policy.id}`, origin: 'policy', lineageRuns: new Map(item.lineageRuns), + depth: childDepth, deliver: true, - process: policy.output.reprocessMedia, + process: policy.output.reprocessMedia && depthAllowsReprocess, }); } + if (artifactsProduced > limits.maxArtifactsPerRoot) { + stopOnBudget( + policy, + item, + `maxArtifactsPerRoot (${limits.maxArtifactsPerRoot}) exceeded`, + ); + break; + } + if (derivedBytesProduced > limits.maxDerivedBytesPerRoot) { + stopOnBudget( + policy, + item, + `maxDerivedBytesPerRoot (${limits.maxDerivedBytesPerRoot}) exceeded`, + ); + break; + } } catch (err) { if (options.signal?.aborted) throw err; const message = err instanceof Error ? err.message : String(err); @@ -386,6 +468,7 @@ async function executePolicy( const invocationId = randomBytes(8).toString('hex'); const stagingDir = await store.createStagingDir(invocationId); + let failure: unknown; try { const request: ToolCallRequestInfo = { callId: invocationId, @@ -471,10 +554,30 @@ async function executePolicy( }); } return { outcome: 'succeeded', derived }; + } catch (err) { + failure = err; + throw err; } finally { - // Success and failure both end without a staging dir (this commit's - // Stage A behavior; quarantine-on-failure is the Stage B follow-up). - await store.removeStagingDir(invocationId).catch(() => {}); + if (failure === undefined || signal?.aborted) { + // Success, no_op, and user aborts end without a staging dir — there + // is nothing to diagnose. + await store.removeStagingDir(invocationId).catch(() => {}); + } else { + // Failure (decision D10 Stage B): move the staging dir — partial + // outputs included — into quarantine/ with a reason.json for + // postmortem; the startup sweeps apply retention/size budgets. If + // quarantining itself fails, fall back to plain removal so a failed + // invocation still never leaves live staging state behind. + try { + await store.quarantineInvocation(invocationId, { + policyId: policy.id, + toolName: policy.toolName, + reason: failure instanceof Error ? failure.message : String(failure), + }); + } catch { + await store.removeStagingDir(invocationId).catch(() => {}); + } + } } } diff --git a/packages/core/src/omni/policy/tools/downsample-audio.ts b/packages/core/src/omni/policy/tools/downsample-audio.ts index 9f0cf639fe2..037d256274a 100644 --- a/packages/core/src/omni/policy/tools/downsample-audio.ts +++ b/packages/core/src/omni/policy/tools/downsample-audio.ts @@ -201,6 +201,7 @@ export class OmniDownsampleAudioTool extends BaseMediaPolicyTool { - constructor() { + constructor(config: MediaPolicyToolConfigView = {}) { super( OMNI_DOWNSAMPLE_IMAGE_TOOL_NAME, 'DownsampleImage', @@ -221,6 +222,7 @@ export class OmniDownsampleImageTool extends BaseMediaPolicyTool { } class TestPolicyTool extends BaseMediaPolicyTool { - constructor() { - super('test_policy_tool', 'TestPolicyTool', 'test', Kind.Other, { - type: 'object', - properties: { - inputPath: { type: 'string' }, - outputDir: { type: 'string' }, - level: { type: 'number', minimum: 1 }, + constructor(view: MediaPolicyToolConfigView = {}) { + super( + 'test_policy_tool', + 'TestPolicyTool', + 'test', + Kind.Other, + { + type: 'object', + properties: { + inputPath: { type: 'string' }, + outputDir: { type: 'string' }, + level: { type: 'number', minimum: 1 }, + }, + required: ['inputPath', 'outputDir'], + additionalProperties: false, }, - required: ['inputPath', 'outputDir'], - additionalProperties: false, - }); + view, + ); } override get mediaPolicyDescriptor(): MediaPolicyToolDescriptor { return { @@ -234,4 +242,58 @@ describe('BaseMediaPolicyTool validation', () => { /absolute/, ); }); + + describe('model-visible schema projection (decision D6)', () => { + it('declares the native schema unchanged without modelAccess settings', () => { + expect(tool.schema).toEqual({ + name: 'test_policy_tool', + description: 'test', + parametersJsonSchema: { + type: 'object', + properties: { + inputPath: { type: 'string' }, + outputDir: { type: 'string' }, + level: { type: 'number', minimum: 1 }, + }, + required: ['inputPath', 'outputDir'], + additionalProperties: false, + }, + }); + }); + + it('projects the declaration while validation keeps the native schema', () => { + const configured = new TestPolicyTool({ + getOmniPolicyToolsSettings: () => ({ + test_policy_tool: { + modelAccess: { + enabled: true, + description: 'Model-facing description.', + lockedArguments: { inputPath: '/x', outputDir: '/y' }, + parameterSchema: { properties: { level: { maximum: 9 } } }, + }, + }, + }), + }); + // The model sees ONLY the tunable, with the override merged in. + expect(configured.schema).toEqual({ + name: 'test_policy_tool', + description: 'Model-facing description.', + parametersJsonSchema: { + type: 'object', + properties: { level: { type: 'number', minimum: 1, maximum: 9 } }, + required: [], + additionalProperties: false, + }, + }); + // …but the harness-injected io arguments the projection hides must + // remain valid: validation runs on the NATIVE schema (§9.4). + expect( + configured.validateToolParams({ + inputPath: '/a/in.png', + outputDir: '/b/staging', + level: 3, + }), + ).toBeNull(); + }); + }); }); diff --git a/packages/core/src/omni/policy/tools/media-policy-tool.ts b/packages/core/src/omni/policy/tools/media-policy-tool.ts index c6d3b6e9cfe..55a2b6e6b94 100644 --- a/packages/core/src/omni/policy/tools/media-policy-tool.ts +++ b/packages/core/src/omni/policy/tools/media-policy-tool.ts @@ -6,15 +6,18 @@ import fs from 'node:fs/promises'; import path from 'node:path'; +import type { FunctionDeclaration } from '@google/genai'; import type { MediaPolicyToolDescriptor, ToolArtifact, ToolArtifactKind, ToolResult, } from '../../../tools/tools.js'; +import type { Kind } from '../../../tools/tools.js'; import { BaseDeclarativeTool } from '../../../tools/tools.js'; import { ToolErrorType } from '../../../tools/tool-error.js'; import { SchemaValidator } from '../../../utils/schemaValidator.js'; +import { projectMediaPolicyToolDeclaration } from '../model-access.js'; import type { OmniPolicyToolsSettings } from '../types.js'; /** Default transcode timeout when `policyTools..runtime.timeoutMs` @@ -84,8 +87,38 @@ export function resolvePolicyToolTimeoutMs( export abstract class BaseMediaPolicyTool< TParams extends object, > extends BaseDeclarativeTool { + constructor( + name: string, + displayName: string, + description: string, + kind: Kind, + parameterSchema: unknown, + /** Config view feeding the modelAccess declaration projection; tools + * constructed without one (tests, embedders) declare their native + * schema unchanged. */ + private readonly modelAccessView: MediaPolicyToolConfigView = {}, + ) { + super(name, displayName, description, kind, parameterSchema); + } + abstract override get mediaPolicyDescriptor(): MediaPolicyToolDescriptor; + /** + * Model-visible declaration (decision D6): the single projection point + * every declaration surface reads — the native schema minus + * `modelAccess.lockedArguments` keys, narrowed to + * `modelAccess.parameterSchema` when configured, with the optional + * description override applied. Validation deliberately does NOT use + * this projection (see {@link validateToolParams}). + */ + override get schema(): FunctionDeclaration { + return projectMediaPolicyToolDeclaration(this.modelAccessView, { + name: this.name, + description: this.description, + parametersJsonSchema: this.parameterSchema, + }); + } + /** * Validate against the tool's NATIVE parameter schema, never the * model-visible `schema` getter: Stage B's modelAccess projection makes diff --git a/packages/core/src/omni/tool-result-media.test.ts b/packages/core/src/omni/tool-result-media.test.ts index 251d6624a38..9dc11cbeff5 100644 --- a/packages/core/src/omni/tool-result-media.test.ts +++ b/packages/core/src/omni/tool-result-media.test.ts @@ -196,6 +196,65 @@ describe('processToolResultOmniMedia', () => { expect(result[1]!.fileData?.fileUri).toBe('oss://bucket/key3'); }); + it('replaces an explicitly omitted delivery with the omission notice text', async () => { + // Stage B (policy design §10.2): the pipeline itself withheld the media + // after the guard policies could not bring it within limits. Not an + // error — the notice stands in for the part. + deliverMock.mockResolvedValueOnce({ + fileUri: '', + mimeType: 'image/png', + sha256: '', + recognized: { modality: 'image' }, + tokenEstimate: { + estimatedTokenCount: 1, + method: 'raw-resource-v1', + status: 'ok', + }, + deduped: false, + omission: { reason: 'still 900 bytes over the upload limit' }, + }); + const parts = [inlinePart('image/png', PNG_BYTES)]; + const result = await processToolResultOmniMedia( + parts, + cfg({ image: true }), + signal, + ); + expect(result).not.toBe(parts); + expect(result).toHaveLength(1); + expect(result[0]).toEqual({ + text: '【媒体省略】tool-media.image：still 900 bytes over the upload limit', + }); + }); + + it('an omission does not consume the per-result upload budgets', async () => { + // Nothing was uploaded for an omitted part, so all 8 upload slots must + // remain for the following parts. + deliverMock.mockResolvedValueOnce({ + fileUri: '', + mimeType: 'image/png', + sha256: '', + recognized: { modality: 'image' }, + tokenEstimate: { + estimatedTokenCount: 1, + method: 'raw-resource-v1', + status: 'ok', + }, + deduped: false, + omission: { reason: 'over limit' }, + }); + const parts = Array.from({ length: 9 }, () => + inlinePart('image/png', PNG_BYTES), + ); + const result = await processToolResultOmniMedia( + parts, + cfg({ image: true }), + signal, + ); + expect(deliverMock).toHaveBeenCalledTimes(9); + expect(result.filter((p) => p.fileData).length).toBe(8); + expect(result.filter((p) => p.inlineData).length).toBe(0); + }); + it('keeps the part inline when staging-dir setup itself fails', async () => { // ~/.qwen/omni existing as a regular FILE makes mkdir fail with ENOTDIR. // That failure must degrade THIS part to inline like any other delivery diff --git a/packages/core/src/omni/tool-result-media.ts b/packages/core/src/omni/tool-result-media.ts index 185e778c981..b549c95d9b5 100644 --- a/packages/core/src/omni/tool-result-media.ts +++ b/packages/core/src/omni/tool-result-media.ts @@ -11,7 +11,7 @@ import type { Part } from '@google/genai'; import type { Config } from '../config/config.js'; import { createDebugLogger } from '../utils/debugLogger.js'; import { isOmniDeliveryActive, processMediaForOmniDelivery } from './index.js'; -import { formatDisclosureText } from './disclosure.js'; +import { formatDisclosureText, formatOmissionText } from './disclosure.js'; import { OmniTransportGuardError } from './guard.js'; import { OmniObjectStore } from './storage.js'; import { sniffMediaType } from './recognition.js'; @@ -109,6 +109,16 @@ export async function processToolResultOmniMedia( displayName, origin: 'tool', }); + if (delivery.omission) { + // Explicit omission (policy design §10.2): the transport guard + // could not bring the part within limits even after the guard + // policies ran — the media is withheld, the notice stands in for + // it, and the upload budgets are untouched (nothing was uploaded). + changed = true; + return [ + { text: formatOmissionText(displayName, delivery.omission.reason) }, + ]; + } changed = true; uploadsRemaining--; uploadBytesRemaining -= bytes.length; From b639f5b67c9a143b0f697a3c449cac4a79d58666 Mon Sep 17 00:00:00 2001 From: LaZzyMan Date: Fri, 7 Aug 2026 12:12:59 +0800 Subject: [PATCH 11/62] fix(omni): declare the disclosure text output in degradation tool descriptors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All three degradation tools emit a disclosure at runtime, but their descriptors never declared the text output, so the system default policies failed the lossy-requires-disclosure validation (config §13 #8) at real startup. Stub-based unit tests missed the drift; config.test.ts now normalizes the defaults against the real tool instances. --- packages/core/src/omni/policy/config.test.ts | 20 +++++++++++++++++++ .../policy/tools/downsample-audio.test.ts | 1 + .../src/omni/policy/tools/downsample-audio.ts | 1 + .../policy/tools/downsample-image.test.ts | 1 + .../src/omni/policy/tools/downsample-image.ts | 1 + .../omni/policy/tools/downscale-video.test.ts | 1 + .../src/omni/policy/tools/downscale-video.ts | 1 + 7 files changed, 26 insertions(+) diff --git a/packages/core/src/omni/policy/config.test.ts b/packages/core/src/omni/policy/config.test.ts index 446c080d17d..351163446af 100644 --- a/packages/core/src/omni/policy/config.test.ts +++ b/packages/core/src/omni/policy/config.test.ts @@ -81,6 +81,26 @@ function normalize( describe('normalizeOmniProcessingConfig', () => { describe('system defaults', () => { + it('normalizes against the REAL degradation tools, not just stubs', async () => { + // The stub lookup above can drift from the shipped tool descriptors; + // this is the startup path every real CLI run takes, so a descriptor + // that fails §13 validation (e.g. a lossy output without a declared + // disclosure) must fail HERE, not at first launch. + const [image, video, audio] = await Promise.all([ + import('./tools/downsample-image.js'), + import('./tools/downscale-video.js'), + import('./tools/downsample-audio.js'), + ]); + const real: Record = { + omni_downsample_image: new image.OmniDownsampleImageTool({}), + omni_downscale_video: new video.OmniDownscaleVideoTool({}), + omni_downsample_audio: new audio.OmniDownsampleAudioTool({}), + }; + const config = normalize({}, real); + expect(config.fixedPolicies).toHaveLength(3); + expect(config.transportGuardPolicies).toHaveLength(3); + }); + it('produces the three default fixed policies with when-thresholds', () => { const config = normalize(); expect(config.fixedPolicies.map((p) => p.id).sort()).toEqual([ diff --git a/packages/core/src/omni/policy/tools/downsample-audio.test.ts b/packages/core/src/omni/policy/tools/downsample-audio.test.ts index 9e167e7b42d..65599e3666b 100644 --- a/packages/core/src/omni/policy/tools/downsample-audio.test.ts +++ b/packages/core/src/omni/policy/tools/downsample-audio.test.ts @@ -83,6 +83,7 @@ describe('OmniDownsampleAudioTool', () => { required: true, lossy: true, }, + { kind: 'text', role: 'disclosure', required: true }, ], settingsSchema: expect.objectContaining({ type: 'object' }), }); diff --git a/packages/core/src/omni/policy/tools/downsample-audio.ts b/packages/core/src/omni/policy/tools/downsample-audio.ts index 037d256274a..8f4b7bce34a 100644 --- a/packages/core/src/omni/policy/tools/downsample-audio.ts +++ b/packages/core/src/omni/policy/tools/downsample-audio.ts @@ -75,6 +75,7 @@ const DESCRIPTOR: MediaPolicyToolDescriptor = { required: true, lossy: true, }, + { kind: 'text', role: 'disclosure', required: true }, ], settingsSchema: { type: 'object', diff --git a/packages/core/src/omni/policy/tools/downsample-image.test.ts b/packages/core/src/omni/policy/tools/downsample-image.test.ts index fdc9baf050a..092e1770636 100644 --- a/packages/core/src/omni/policy/tools/downsample-image.test.ts +++ b/packages/core/src/omni/policy/tools/downsample-image.test.ts @@ -94,6 +94,7 @@ describe('OmniDownsampleImageTool', () => { required: true, lossy: true, }, + { kind: 'text', role: 'disclosure', required: true }, ], settingsSchema: expect.objectContaining({ type: 'object' }), }); diff --git a/packages/core/src/omni/policy/tools/downsample-image.ts b/packages/core/src/omni/policy/tools/downsample-image.ts index dfbf59809de..eacdf7b713a 100644 --- a/packages/core/src/omni/policy/tools/downsample-image.ts +++ b/packages/core/src/omni/policy/tools/downsample-image.ts @@ -66,6 +66,7 @@ const DESCRIPTOR: MediaPolicyToolDescriptor = { required: true, lossy: true, }, + { kind: 'text', role: 'disclosure', required: true }, ], settingsSchema: { type: 'object', diff --git a/packages/core/src/omni/policy/tools/downscale-video.test.ts b/packages/core/src/omni/policy/tools/downscale-video.test.ts index 31914e844f9..b5d331fbaa4 100644 --- a/packages/core/src/omni/policy/tools/downscale-video.test.ts +++ b/packages/core/src/omni/policy/tools/downscale-video.test.ts @@ -88,6 +88,7 @@ describe('OmniDownscaleVideoTool', () => { required: true, lossy: true, }, + { kind: 'text', role: 'disclosure', required: true }, ], settingsSchema: expect.objectContaining({ type: 'object' }), }); diff --git a/packages/core/src/omni/policy/tools/downscale-video.ts b/packages/core/src/omni/policy/tools/downscale-video.ts index 433758d99df..daaa7bcc65f 100644 --- a/packages/core/src/omni/policy/tools/downscale-video.ts +++ b/packages/core/src/omni/policy/tools/downscale-video.ts @@ -98,6 +98,7 @@ const DESCRIPTOR: MediaPolicyToolDescriptor = { required: true, lossy: true, }, + { kind: 'text', role: 'disclosure', required: true }, ], settingsSchema: { type: 'object', From d25f3fcb2217b3801f94078c77a23db3e20bafc7 Mon Sep 17 00:00:00 2001 From: LaZzyMan Date: Fri, 7 Aug 2026 13:22:34 +0800 Subject: [PATCH 12/62] fix(omni): harden policy pipeline per review (cache validation, permissions, settings, sweep grace, concurrency) - validate degradation-cache entries and object-store path components (sha256/extension shape) so a poisoned policy-cache.json cannot traverse paths or serve malformed derivatives; malformed entries self-heal - re-hash cache-hit objects before reuse: mismatched bytes trigger a fresh transcode and heal the object store (D2 integrity) - default media policy tools to 'ask' permission via a shared BaseMediaPolicyToolInvocation (model-origin calls confirm outside yolo; fixed_policy runs are unaffected) - consume policyTools settings from config: tool-level settings defaults merge under policy arguments and feed the cache fingerprint so settings edits invalidate cached derivatives - deliver @url omission notices and degradation disclosures: omission replaces the fileData part with the notice text; disclosure text lands immediately before its fileData part (D8) - give staging sweep a 1h multi-process grace window (only entries older than the window are deleted; symlink entries removed regardless of age) - gate concurrent fixed-policy runs per omni root with a FIFO counting semaphore honoring maxConcurrentResources --- .../src/ui/hooks/atCommandProcessor.test.ts | 63 +++++ .../cli/src/ui/hooks/atCommandProcessor.ts | 28 +- packages/core/src/index.ts | 2 + packages/core/src/omni/index.ts | 5 +- .../src/omni/policy/degradation-cache.test.ts | 53 ++++ .../core/src/omni/policy/degradation-cache.ts | 31 +- .../core/src/omni/policy/orchestrator.test.ts | 266 ++++++++++++++++++ packages/core/src/omni/policy/orchestrator.ts | 158 +++++++++-- .../policy/tools/downsample-audio.test.ts | 5 + .../src/omni/policy/tools/downsample-audio.ts | 8 +- .../policy/tools/downsample-image.test.ts | 5 + .../src/omni/policy/tools/downsample-image.ts | 8 +- .../omni/policy/tools/downscale-video.test.ts | 5 + .../src/omni/policy/tools/downscale-video.ts | 8 +- .../omni/policy/tools/media-policy-tool.ts | 23 +- packages/core/src/omni/recovery.test.ts | 44 ++- packages/core/src/omni/recovery.ts | 44 ++- packages/core/src/omni/storage.test.ts | 41 +++ packages/core/src/omni/storage.ts | 28 +- 19 files changed, 763 insertions(+), 62 deletions(-) diff --git a/packages/cli/src/ui/hooks/atCommandProcessor.test.ts b/packages/cli/src/ui/hooks/atCommandProcessor.test.ts index 791164ba7f5..ab017438430 100644 --- a/packages/cli/src/ui/hooks/atCommandProcessor.test.ts +++ b/packages/cli/src/ui/hooks/atCommandProcessor.test.ts @@ -1751,6 +1751,13 @@ describe('handleAtCommand', () => { vi.mock('@qwen-code/qwen-code-core/omni', () => ({ ...omniMocks, + // Deterministic stand-ins for the shared formatters: the tests assert + // the WIRING (which formatter, which arguments, part ordering), while + // the formatters' own wording is covered by their core unit tests. + formatOmissionText: (name: string, reason: string) => + `[omission ${name}: ${reason}]`, + formatDisclosureText: (name: string, disclosure: string) => + `[disclosure ${name}: ${disclosure}]`, OmniObjectStore: class { getOmniRootDir() { return path.join(os.tmpdir(), 'omni-at-test'); @@ -1916,6 +1923,62 @@ describe('handleAtCommand', () => { expect(parts.filter((p) => 'fileData' in p)).toHaveLength(1); }); + it('replaces the media with an omission notice when the transport guard withholds it', async () => { + // Policy design §10.2: an omission is a successful delivery whose + // content IS the notice — no fileData part, no error card. + omniMocks.processMediaForOmniDelivery.mockResolvedValue({ + omission: { reason: 'video exceeds the 500MB transport limit' }, + }); + const result = await handleAtCommand({ + query: 'summarize @https://example.com/clip.mp4 please', + config: omniConfig(true), + onDebugMessage: mockOnDebugMessage, + messageId: 706, + signal: abortController.signal, + }); + + expect(result.shouldProceed).toBe(true); + const parts = result.processedQuery as Array>; + expect(parts).toContainEqual({ + text: '[omission clip.mp4: video exceeds the 500MB transport limit]', + }); + expect(parts.some((p) => 'fileData' in p)).toBe(false); + expect(result.toolDisplays![0]).toMatchObject({ + name: 'Fetch Media URL', + status: ToolCallStatus.Success, + resultDisplay: 'Media omitted by the omni transport guard: clip.mp4', + }); + }); + + it('places the degradation disclosure text immediately before the fileData part (D8)', async () => { + omniMocks.processMediaForOmniDelivery.mockResolvedValue({ + fileUri: 'oss://bucket/clip.mp4', + mimeType: 'video/mp4', + recognized: { modality: 'video', sizeBytes: 2 * 1024 * 1024 }, + degraded: true, + disclosure: '原 1080p → 480p，细节受损', + }); + const result = await handleAtCommand({ + query: 'summarize @https://example.com/clip.mp4 please', + config: omniConfig(true), + onDebugMessage: mockOnDebugMessage, + messageId: 707, + signal: abortController.signal, + }); + + expect(result.shouldProceed).toBe(true); + const parts = result.processedQuery as Array>; + const disclosureIdx = parts.findIndex( + (p) => p['text'] === '[disclosure clip.mp4: 原 1080p → 480p，细节受损]', + ); + const fileDataIdx = parts.findIndex((p) => 'fileData' in p); + expect(disclosureIdx).toBeGreaterThan(-1); + expect(fileDataIdx).toBe(disclosureIdx + 1); + expect( + (result.toolDisplays![0] as { resultDisplay: string }).resultDisplay, + ).toContain('(degraded by media policy)'); + }); + it('ends the turn quietly (shouldProceed=false) on a user abort mid-download', async () => { omniMocks.downloadMediaUrl.mockImplementation(async () => { abortController.abort(); diff --git a/packages/cli/src/ui/hooks/atCommandProcessor.ts b/packages/cli/src/ui/hooks/atCommandProcessor.ts index 8aca3981fc9..83cca0acd24 100644 --- a/packages/cli/src/ui/hooks/atCommandProcessor.ts +++ b/packages/cli/src/ui/hooks/atCommandProcessor.ts @@ -623,6 +623,32 @@ export async function resolveAtCommandQuery({ // the opaque staging path the download landed under. { signal, displayName: urlBase }, ); + if (delivery.omission) { + // Explicit omission (policy design §10.2): the media is withheld + // and the omission notice text stands in its place — mirroring + // readMediaViaOmniDelivery. Not an error: the fetch succeeded; + // the transport guard's verdict is the content. + urlMediaParts.push({ + text: core.formatOmissionText(urlBase, delivery.omission.reason), + }); + urlMediaLabels.push(ref.url); + urlMediaDisplays.push({ + callId, + name: 'Fetch Media URL', + description: `Downloaded ${ref.url}`, + status: ToolCallStatus.Success, + resultDisplay: `Media omitted by the omni transport guard: ${urlBase}`, + confirmationDetails: undefined, + }); + continue; + } + // Disclosure IMMEDIATELY before its media part (decision D8): + // provider converters that relocate media move the pair together. + if (delivery.disclosure) { + urlMediaParts.push({ + text: core.formatDisclosureText(urlBase, delivery.disclosure), + }); + } urlMediaParts.push({ fileData: { fileUri: delivery.fileUri, @@ -636,7 +662,7 @@ export async function resolveAtCommandQuery({ name: 'Fetch Media URL', description: `Downloaded ${ref.url}`, status: ToolCallStatus.Success, - resultDisplay: `Localized ${urlBase} (${delivery.recognized.modality}, ${(delivery.recognized.sizeBytes / 1024 / 1024).toFixed(1)}MB) and delivered via omni upload.`, + resultDisplay: `Localized ${urlBase} (${delivery.recognized.modality}, ${(delivery.recognized.sizeBytes / 1024 / 1024).toFixed(1)}MB) and delivered via omni upload${delivery.degraded ? ' (degraded by media policy)' : ''}.`, confirmationDetails: undefined, }); } catch (error) { diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 1261e24361d..a26462e082c 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -675,6 +675,8 @@ export { downloadMediaUrl, effectiveMaxDownloadFileBytes, recognizeMediaFile, + formatDisclosureText, + formatOmissionText, OmniObjectStore, OmniDeliveryError, OmniDownloadError, diff --git a/packages/core/src/omni/index.ts b/packages/core/src/omni/index.ts index a0880bd5276..af2da993a52 100644 --- a/packages/core/src/omni/index.ts +++ b/packages/core/src/omni/index.ts @@ -360,8 +360,9 @@ export async function processMediaForOmniDelivery( ); // Lazy one-time hygiene scan (expired .part files, promotion orphans, // quarantine retention/size sweeps, sampled object verification). MUST - // run before the orchestrator: the scan deletes staging/ wholesale, - // which would race live invocations. + // run before the orchestrator: the scan deletes stale staging entries, + // which would race this process's own live invocations. (Other + // processes' live entries are protected by the staging grace window.) await runStartupRecoveryOnce(store, uploadCache, { quarantineRetentionDays: config.getOmniQuarantineRetentionDays?.(), quarantineMaxBytes: config.getOmniQuarantineMaxBytes?.(), diff --git a/packages/core/src/omni/policy/degradation-cache.test.ts b/packages/core/src/omni/policy/degradation-cache.test.ts index 3021d8bd144..a8947df91c4 100644 --- a/packages/core/src/omni/policy/degradation-cache.test.ts +++ b/packages/core/src/omni/policy/degradation-cache.test.ts @@ -197,4 +197,57 @@ describe('OmniDegradationCache', () => { ); expect(Object.keys(raw.entries)).toHaveLength(8); }); + + describe('poisoned cache file (workspace-controlled input is shape-validated)', () => { + /** Plant one raw entry as a hostile repo could ship it. */ + async function plantEntry(entry: Record): Promise { + await fs.writeFile( + path.join(root, 'policy-cache.json'), + JSON.stringify({ + version: 1, + entries: { [`${ORIGINAL}|${fp}`]: entry }, + }), + ); + } + + it.each([ + [ + 'traversal in degradedSha256', + { ...ENTRY, degradedSha256: '../../../../etc/passwd' }, + ], + [ + 'uppercase hex degradedSha256', + { ...ENTRY, degradedSha256: 'A'.repeat(64) }, + ], + ['short degradedSha256', { ...ENTRY, degradedSha256: 'ab12' }], + [ + 'traversal in extension', + { ...ENTRY, extension: '/../../../../tmp/evil' }, + ], + ['multi-dot extension', { ...ENTRY, extension: '.jpg/../x' }], + ['dotless extension', { ...ENTRY, extension: 'jpg' }], + ['non-string extension', { ...ENTRY, extension: 42 }], + ['empty disclosure (D8 invariant)', { ...ENTRY, disclosure: '' }], + ['missing disclosure', { ...ENTRY, disclosure: undefined }], + ['empty mimeType', { ...ENTRY, mimeType: '' }], + ['missing mimeType', { ...ENTRY, mimeType: undefined }], + ])( + 'drops a malformed entry instead of serving it: %s', + async (_label, entry) => { + await plantEntry(entry as Record); + await expect(cache.get(ORIGINAL, fp)).resolves.toBeNull(); + // Self-heal: the malformed entry is deleted, so the next transcode's + // put() rebuilds it from verified data. + const raw = JSON.parse( + await fs.readFile(path.join(root, 'policy-cache.json'), 'utf8'), + ); + expect(raw.entries).toEqual({}); + }, + ); + + it('still serves a planted entry when every field is well-formed', async () => { + await plantEntry({ ...ENTRY, createdAt: new Date().toISOString() }); + await expect(cache.get(ORIGINAL, fp)).resolves.toMatchObject(ENTRY); + }); + }); }); diff --git a/packages/core/src/omni/policy/degradation-cache.ts b/packages/core/src/omni/policy/degradation-cache.ts index bc96556347a..4d10ec2d1a7 100644 --- a/packages/core/src/omni/policy/degradation-cache.ts +++ b/packages/core/src/omni/policy/degradation-cache.ts @@ -7,6 +7,7 @@ import { createHash } from 'node:crypto'; import path from 'node:path'; import { OmniJsonCacheFile } from '../json-cache-file.js'; +import { OBJECT_EXTENSION_RE } from '../storage.js'; /** * Identity of one degradation result (decision D2): everything the @@ -104,9 +105,33 @@ export class OmniDegradationCache { originalSha256: string, policyFingerprint: string, ): Promise { - return this.file.access(null, (entries) => ({ - result: entries[this.key(originalSha256, policyFingerprint)] ?? null, - })); + return this.file.access(null, (entries) => { + const key = this.key(originalSha256, policyFingerprint); + const entry = entries[key]; + if (!entry) return { result: null }; + // The cache file sits inside the workspace (`.qwen/omni/`), so a + // hostile repository can ship a crafted one. Entries are only + // trusted when every field that later becomes a filesystem path or + // a model-visible text is well-formed: the hash must be exactly + // 64-hex (it addresses the object store), the extension a single + // dotted component (no traversal segments), and the disclosure + // non-empty (lossy reuse without disclosure would silently break + // the D8 invariant). Malformed entries are dropped, not served — + // the orchestrator then re-derives and overwrites them. + if ( + !/^[0-9a-f]{64}$/.test(entry.degradedSha256) || + typeof entry.extension !== 'string' || + !OBJECT_EXTENSION_RE.test(entry.extension) || + typeof entry.disclosure !== 'string' || + entry.disclosure.length === 0 || + typeof entry.mimeType !== 'string' || + entry.mimeType.length === 0 + ) { + delete entries[key]; + return { result: null, changed: true }; + } + return { result: entry }; + }); } async put( diff --git a/packages/core/src/omni/policy/orchestrator.test.ts b/packages/core/src/omni/policy/orchestrator.test.ts index 740e7f206ab..5cf0ddfbaef 100644 --- a/packages/core/src/omni/policy/orchestrator.test.ts +++ b/packages/core/src/omni/policy/orchestrator.test.ts @@ -99,6 +99,7 @@ function makePolicy( function makeConfig( descriptorByTool: Record, + policyToolsSettings?: unknown, ) { return { getToolRegistry: () => ({ @@ -107,6 +108,7 @@ function makeConfig( ? { mediaPolicyDescriptor: descriptorByTool[name] } : undefined, }), + getOmniPolicyToolsSettings: () => policyToolsSettings, } as unknown as Config; } @@ -854,4 +856,268 @@ describe('runFixedPolicies', () => { // The failed invocation still never leaves live staging state behind. await expect(fs.readdir(store.getStagingDir())).resolves.toEqual([]); }); + + describe('tool-level settings defaults (omni.processing.policyTools..settings)', () => { + it('merges settings under policy arguments in BOTH the tool call and the cache fingerprint', async () => { + mockToolSuccess(); + const configured = makeConfig( + { omni_downsample_image: DESCRIPTOR }, + { omni_downsample_image: { settings: { quality: 60 } } }, + ); + await runFixedPolicies(configured, source, { + store, + policies: [makePolicy()], + }); + + const req = executeToolCallMock.mock.calls[0][1] as ToolCallRequestInfo; + expect(req.args).toMatchObject({ maxDimension: 1568, quality: 60 }); + + // Fingerprint must include the merged tunables — otherwise editing + // settings would keep serving derivatives made under the old values. + const cache = new OmniDegradationCache(store.getOmniRootDir()); + await expect( + cache.get( + sha256Of(SOURCE_BYTES), + computePolicyFingerprint('omni_downsample_image', { + maxDimension: 1568, + quality: 60, + }), + ), + ).resolves.not.toBeNull(); + await expect( + cache.get( + sha256Of(SOURCE_BYTES), + computePolicyFingerprint('omni_downsample_image', { + maxDimension: 1568, + }), + ), + ).resolves.toBeNull(); + }); + + it('policy arguments override colliding settings keys', async () => { + mockToolSuccess(); + const configured = makeConfig( + { omni_downsample_image: DESCRIPTOR }, + { omni_downsample_image: { settings: { maxDimension: 99 } } }, + ); + await runFixedPolicies(configured, source, { + store, + policies: [makePolicy()], // arguments: { maxDimension: 1568 } + }); + const req = executeToolCallMock.mock.calls[0][1] as ToolCallRequestInfo; + expect(req.args['maxDimension']).toBe(1568); + }); + + it.each([ + ['null tombstone', { omni_downsample_image: null }], + ['non-object settings', { omni_downsample_image: { settings: 'evil' } }], + ['array settings', { omni_downsample_image: { settings: [1, 2] } }], + ['absent map', undefined], + ])('ignores malformed settings entries: %s', async (_label, settings) => { + mockToolSuccess(); + const configured = makeConfig( + { omni_downsample_image: DESCRIPTOR }, + settings, + ); + await runFixedPolicies(configured, source, { + store, + policies: [makePolicy()], + }); + const req = executeToolCallMock.mock.calls[0][1] as ToolCallRequestInfo; + expect(req.args).toEqual({ + maxDimension: 1568, + inputPath: sourcePath, + outputDir: expect.stringContaining(store.getStagingDir()), + }); + }); + }); + + it('re-hashes a cache hit before reuse: poisoned object bytes trigger re-transcode (D2 integrity)', async () => { + const degradedSha = sha256Of(DEGRADED_BYTES); + const objectPath = store.objectPathFor(degradedSha, '.jpg'); + await fs.mkdir(path.dirname(objectPath), { recursive: true }); + // A file EXISTS at the addressed path but its bytes do not hash to the + // entry's identity — planted via a crafted policy-cache.json plus a + // foreign object (or plain store corruption). + await fs.writeFile(objectPath, 'not-the-degraded-bytes'); + const cache = new OmniDegradationCache(store.getOmniRootDir()); + const fingerprint = computePolicyFingerprint('omni_downsample_image', { + maxDimension: 1568, + }); + await cache.put(sha256Of(SOURCE_BYTES), fingerprint, { + degradedSha256: degradedSha, + extension: '.jpg', + disclosure: 'poisoned disclosure', + mimeType: 'image/jpeg', + }); + mockToolSuccess(); + + const { deliveries, records } = await runFixedPolicies(config, source, { + store, + policies: [makePolicy()], + }); + + // The mismatching object was never served: the tool re-ran and the + // store now holds verified bytes under the hash. + expect(executeToolCallMock).toHaveBeenCalledTimes(1); + expect(records[0]).toMatchObject({ outcome: 'succeeded' }); + expect(deliveries[0].sha256).toBe(degradedSha); + expect(deliveries[0].disclosure).toBe( + 'Downsampled from 4000x3000 to 1568x1176.', + ); + await expect(fs.readFile(objectPath, 'utf8')).resolves.toBe(DEGRADED_BYTES); + }); + + describe('maxConcurrentResources gates concurrent runs per omni root', () => { + /** Tool mock that parks each invocation on a caller-released latch, + * recording how many invocations are in flight simultaneously. */ + function mockGatedTool() { + let inFlight = 0; + let peak = 0; + const releases: Array<() => void> = []; + executeToolCallMock.mockImplementation( + async (_config: Config, request: ToolCallRequestInfo) => { + inFlight++; + peak = Math.max(peak, inFlight); + await new Promise((resolve) => releases.push(resolve)); + inFlight--; + const outputDir = request.args['outputDir'] as string; + const bytes = `degraded-${request.callId}`; + await fs.writeFile(path.join(outputDir, 'out.jpg'), bytes); + return { + callId: request.callId, + responseParts: [], + resultDisplay: undefined, + error: undefined, + errorType: undefined, + policyArtifacts: { + toolName: request.name, + invocationId: request.callId, + executionOrigin: request.executionOrigin, + artifacts: [ + { + kind: 'image', + storage: 'workspace', + title: 'out.jpg', + workspacePath: 'out.jpg', + mimeType: 'image/jpeg', + metadata: { omniDisclosure: 'gated' }, + }, + ], + }, + }; + }, + ); + return { + peak: () => peak, + started: () => releases.length, + releaseAll: () => { + for (const release of releases.splice(0)) release(); + }, + }; + } + + async function makeSecondSource(): Promise { + const secondPath = path.join(tmpDir, 'photo-2.png'); + await fs.writeFile(secondPath, 'second-image-bytes'); + return { + filePath: secondPath, + recognized: recognizedImage({ sizeBytes: 18 }), + displayName: 'photo-2.png', + origin: 'user', + }; + } + + it('limit 1: the second resource waits until the first fully finishes', async () => { + const gate = mockGatedTool(); + const second = await makeSecondSource(); + const options = { + store, + policies: [makePolicy()], + limits: limitsWith({ maxConcurrentResources: 1 }), + }; + + const run1 = runFixedPolicies(config, source, options); + const run2 = runFixedPolicies(config, second, options); + // Give both runs every chance to start their tool call. + await vi.waitFor(() => expect(gate.started()).toBe(1)); + await new Promise((r) => setTimeout(r, 20)); + expect(gate.started()).toBe(1); // run2 is parked on the gate + + gate.releaseAll(); // finish run1 → slot transfers to run2 + await vi.waitFor(() => expect(gate.started()).toBe(1)); // fresh latch + gate.releaseAll(); + await Promise.all([run1, run2]); + expect(gate.peak()).toBe(1); + expect(executeToolCallMock).toHaveBeenCalledTimes(2); + }); + + it('limit 2: both resources transcode simultaneously', async () => { + const gate = mockGatedTool(); + const second = await makeSecondSource(); + const options = { + store, + policies: [makePolicy()], + limits: limitsWith({ maxConcurrentResources: 2 }), + }; + + const run1 = runFixedPolicies(config, source, options); + const run2 = runFixedPolicies(config, second, options); + await vi.waitFor(() => expect(gate.started()).toBe(2)); + expect(gate.peak()).toBe(2); + gate.releaseAll(); + await Promise.all([run1, run2]); + }); + + it('a failed run releases its slot (no deadlock for the waiter)', async () => { + const second = await makeSecondSource(); + executeToolCallMock + .mockResolvedValueOnce({ + callId: 'x', + responseParts: [], + resultDisplay: undefined, + error: new Error('ffmpeg exploded'), + errorType: undefined, + }) + .mockImplementation( + async (_c: Config, request: ToolCallRequestInfo) => { + const outputDir = request.args['outputDir'] as string; + await fs.writeFile(path.join(outputDir, 'out.jpg'), DEGRADED_BYTES); + return { + callId: request.callId, + responseParts: [], + resultDisplay: undefined, + error: undefined, + errorType: undefined, + policyArtifacts: { + toolName: request.name, + invocationId: request.callId, + executionOrigin: request.executionOrigin, + artifacts: [ + { + kind: 'image', + storage: 'workspace', + title: 'out.jpg', + workspacePath: 'out.jpg', + mimeType: 'image/jpeg', + metadata: { omniDisclosure: 'ok' }, + }, + ], + }, + }; + }, + ); + const options = { + store, + policies: [makePolicy({ onFailure: 'abort' as const })], + limits: limitsWith({ maxConcurrentResources: 1 }), + }; + await expect(runFixedPolicies(config, source, options)).rejects.toThrow( + OmniPolicyExecutionError, + ); + // The waiter (or any later run) must still get the slot. + const { records } = await runFixedPolicies(config, second, options); + expect(records[0]).toMatchObject({ outcome: 'succeeded' }); + }); + }); }); diff --git a/packages/core/src/omni/policy/orchestrator.ts b/packages/core/src/omni/policy/orchestrator.ts index d3363aa117e..1d7460792a7 100644 --- a/packages/core/src/omni/policy/orchestrator.ts +++ b/packages/core/src/omni/policy/orchestrator.ts @@ -186,6 +186,56 @@ function sortPolicies( ); } +/** Per-omni-root counting semaphore bounding how many resources are + * inside fixed-policy processing at once (`maxConcurrentResources`, + * decision D11). Keyed by omni root so distinct stores in one process + * (multi-project setups, tests) never throttle each other. Callers of + * one root share a config, so the per-call limit is stable per key. */ +const resourceGates = new Map< + string, + { active: number; waiters: Array<() => void> } +>(); + +/** + * Take one processing slot for `rootDir`, waiting FIFO when `limit` are + * already taken. Returns an idempotent release function. On release the + * slot transfers directly to the next waiter (no decrement/re-increment + * gap another caller could slip through), so at most `limit` holders + * ever run concurrently. + */ +async function acquireResourceSlot( + rootDir: string, + limit: number, +): Promise<() => void> { + const effectiveLimit = Math.max(1, Math.floor(limit)); + let gate = resourceGates.get(rootDir); + if (!gate) { + gate = { active: 0, waiters: [] }; + resourceGates.set(rootDir, gate); + } + const heldGate = gate; + if (heldGate.active >= effectiveLimit) { + await new Promise((resolve) => heldGate.waiters.push(resolve)); + // Slot transferred by the releaser — `active` already counts us. + } else { + heldGate.active++; + } + let released = false; + return () => { + if (released) return; + released = true; + const next = heldGate.waiters.shift(); + if (next) { + next(); + return; + } + heldGate.active--; + if (heldGate.active === 0 && heldGate.waiters.length === 0) { + resourceGates.delete(rootDir); + } + }; +} + /** * Run the fixed-policy pipeline over one recognized media resource * (decisions D1/D3/D5): match each policy in priority order, execute the @@ -222,6 +272,38 @@ export async function runFixedPolicies( }> { const policies = sortPolicies(options.policies); const limits = options.limits ?? DEFAULT_OMNI_PROCESSING_LIMITS; + // `maxConcurrentResources` (decision D11): each call processes one + // root, so bounding concurrent calls per omni root bounds simultaneous + // transcode work (ffmpeg/sharp processes, staging disk churn). + const releaseSlot = await acquireResourceSlot( + options.store.getOmniRootDir(), + limits.maxConcurrentResources, + ); + try { + return await runFixedPoliciesUnbounded(config, source, options, { + policies, + limits, + }); + } finally { + releaseSlot(); + } +} + +/** Body of {@link runFixedPolicies}, after the per-root concurrency slot + * has been taken. */ +async function runFixedPoliciesUnbounded( + config: Config, + source: PolicySourceResource, + options: RunFixedPoliciesOptions, + normalized: { + policies: NormalizedFixedPolicy[]; + limits: NormalizedOmniProcessingLimits; + }, +): Promise<{ + deliveries: PolicyDeliveryResource[]; + records: PolicyRunRecord[]; +}> { + const { policies, limits } = normalized; const cache = options.degradationCache ?? new OmniDegradationCache(options.store.getOmniRootDir()); @@ -410,6 +492,25 @@ interface ValidatedArtifact { lossy: boolean; } +/** + * Tool-level tunable defaults from + * `omni.processing.policyTools..settings`. The map is raw settings + * input (values may be null tombstones or malformed — see + * `OmniPolicyToolsSettings` in types.ts), so anything non-conforming + * reads as "no defaults" rather than throwing mid-run. + */ +function resolveToolSettingsDefaults( + config: Config, + toolName: string, +): Record { + const entry = config.getOmniPolicyToolsSettings?.()?.[toolName]; + const settings = entry?.settings; + if (settings && typeof settings === 'object' && !Array.isArray(settings)) { + return settings; + } + return {}; +} + /** * Execute one policy against one work item: degradation-cache lookup, * otherwise a real tool invocation in a fresh staging directory followed @@ -435,34 +536,51 @@ async function executePolicy( // The source hash keys the degradation cache; computed lazily so runs // without matching policies never pay it. item.sha256 ??= await hashFileSha256(item.filePath, signal); + // Effective tunables: tool-level defaults from + // `omni.processing.policyTools..settings` (validated against the + // descriptor's settingsSchema at startup) underneath the policy's own + // arguments. Merged HERE — the single point feeding both the tool call + // and the cache fingerprint — so a settings change also invalidates + // cached derivatives produced under the old values. + const settingsDefaults = resolveToolSettingsDefaults(config, policy.toolName); + const effectiveArguments = { ...settingsDefaults, ...policy.arguments }; const fingerprint = computePolicyFingerprint( policy.toolName, - policy.arguments, + effectiveArguments, ); const hit = await cache.get(item.sha256, fingerprint); if (hit) { const objectPath = store.objectPathFor(hit.degradedSha256, hit.extension); const stat = await fs.lstat(objectPath).catch(() => undefined); if (stat?.isFile() && !stat.isSymbolicLink()) { - const recognized = await recognizeMediaFile(objectPath, { signal }); - debugLogger.debug( - `degradation cache hit: policy=${policy.id} sha256=${item.sha256.slice(0, 12)}…`, - ); - return { - outcome: 'cache_hit', - derived: [ - { - filePath: objectPath, - recognized, - sha256: hit.degradedSha256, - disclosure: hit.disclosure, - degraded: true, - }, - ], - }; + // Content verification before reuse: the cache file lives in the + // workspace and is only shape-validated on load, so the bytes at + // the addressed path must actually hash to the entry's identity — + // otherwise a poisoned cache (or a corrupted store) would silently + // substitute foreign media as "the degraded derivative". + const actualSha256 = await hashFileSha256(objectPath, signal); + if (actualSha256 === hit.degradedSha256) { + const recognized = await recognizeMediaFile(objectPath, { signal }); + debugLogger.debug( + `degradation cache hit: policy=${policy.id} sha256=${item.sha256.slice(0, 12)}…`, + ); + return { + outcome: 'cache_hit', + derived: [ + { + filePath: objectPath, + recognized, + sha256: hit.degradedSha256, + disclosure: hit.disclosure, + degraded: true, + }, + ], + }; + } } - // Stale: the derivative left the store (GC, manual deletion). Drop - // every entry pointing at it and re-transcode. + // Stale or mismatching: the derivative left the store (GC, manual + // deletion) or its bytes no longer match the entry. Drop every entry + // pointing at it and re-transcode. await cache.removeByDegradedSha256(hit.degradedSha256); } @@ -474,7 +592,7 @@ async function executePolicy( callId: invocationId, name: policy.toolName, args: { - ...policy.arguments, + ...effectiveArguments, inputPath: item.filePath, outputDir: stagingDir, }, diff --git a/packages/core/src/omni/policy/tools/downsample-audio.test.ts b/packages/core/src/omni/policy/tools/downsample-audio.test.ts index 65599e3666b..bd00518492f 100644 --- a/packages/core/src/omni/policy/tools/downsample-audio.test.ts +++ b/packages/core/src/omni/policy/tools/downsample-audio.test.ts @@ -94,6 +94,11 @@ describe('OmniDownsampleAudioTool', () => { }); }); + it("defaults to 'ask' permission: a model-origin call writes files and must confirm outside yolo", async () => { + const invocation = tool.build({ inputPath, outputDir } as never); + await expect(invocation.getDefaultPermission()).resolves.toBe('ask'); + }); + it('downsamples with the fixed-call defaults and disclosure (D8)', async () => { probe({ bitRate: 320_000, sampleRateHz: 48_000, channels: 2 }); const { result, signal } = await run(); diff --git a/packages/core/src/omni/policy/tools/downsample-audio.ts b/packages/core/src/omni/policy/tools/downsample-audio.ts index 8f4b7bce34a..cb8c10c85d7 100644 --- a/packages/core/src/omni/policy/tools/downsample-audio.ts +++ b/packages/core/src/omni/policy/tools/downsample-audio.ts @@ -11,11 +11,12 @@ import type { ToolInvocation, ToolResult, } from '../../../tools/tools.js'; -import { BaseToolInvocation, Kind } from '../../../tools/tools.js'; +import { Kind } from '../../../tools/tools.js'; import { probeMediaMetadata, runFfmpeg } from '../../ffmpeg.js'; import { assertMediaPolicyIo, BaseMediaPolicyTool, + BaseMediaPolicyToolInvocation, formatBytesShort, MEDIA_POLICY_IO_SCHEMA_PROPERTIES, mediaPolicyToolError, @@ -92,10 +93,7 @@ function describeChannels(channels: number | undefined): string { return ` ${channels}声道`; } -class DownsampleAudioInvocation extends BaseToolInvocation< - DownsampleAudioParams, - ToolResult -> { +class DownsampleAudioInvocation extends BaseMediaPolicyToolInvocation { constructor( params: DownsampleAudioParams, private readonly timeoutMs: number, diff --git a/packages/core/src/omni/policy/tools/downsample-image.test.ts b/packages/core/src/omni/policy/tools/downsample-image.test.ts index 092e1770636..aa78fd908d3 100644 --- a/packages/core/src/omni/policy/tools/downsample-image.test.ts +++ b/packages/core/src/omni/policy/tools/downsample-image.test.ts @@ -104,6 +104,11 @@ describe('OmniDownsampleImageTool', () => { }); }); + it("defaults to 'ask' permission: a model-origin call writes files and must confirm outside yolo", async () => { + const invocation = tool.build({ inputPath, outputDir } as never); + await expect(invocation.getDefaultPermission()).resolves.toBe('ask'); + }); + it('downsamples with the fixed-call defaults and disclosure (D8)', async () => { probe({ width: 4096, height: 3072, frameCount: 1 }); const { result, signal } = await run(); diff --git a/packages/core/src/omni/policy/tools/downsample-image.ts b/packages/core/src/omni/policy/tools/downsample-image.ts index eacdf7b713a..029e346bbd4 100644 --- a/packages/core/src/omni/policy/tools/downsample-image.ts +++ b/packages/core/src/omni/policy/tools/downsample-image.ts @@ -10,11 +10,12 @@ import type { ToolInvocation, ToolResult, } from '../../../tools/tools.js'; -import { BaseToolInvocation, Kind } from '../../../tools/tools.js'; +import { Kind } from '../../../tools/tools.js'; import { probeMediaMetadata } from '../../ffmpeg.js'; import { assertMediaPolicyIo, BaseMediaPolicyTool, + BaseMediaPolicyToolInvocation, formatBytesShort, MEDIA_POLICY_IO_SCHEMA_PROPERTIES, mediaPolicyToolError, @@ -107,10 +108,7 @@ async function loadSharp(): Promise { .default; } -class DownsampleImageInvocation extends BaseToolInvocation< - DownsampleImageParams, - ToolResult -> { +class DownsampleImageInvocation extends BaseMediaPolicyToolInvocation { getDescription(): string { const maxDimension = this.params.maxDimension ?? DOWNSAMPLE_IMAGE_DEFAULTS.maxDimension; diff --git a/packages/core/src/omni/policy/tools/downscale-video.test.ts b/packages/core/src/omni/policy/tools/downscale-video.test.ts index b5d331fbaa4..5f5e43b45e4 100644 --- a/packages/core/src/omni/policy/tools/downscale-video.test.ts +++ b/packages/core/src/omni/policy/tools/downscale-video.test.ts @@ -100,6 +100,11 @@ describe('OmniDownscaleVideoTool', () => { }); }); + it("defaults to 'ask' permission: a model-origin call writes files and must confirm outside yolo", async () => { + const invocation = tool.build({ inputPath, outputDir } as never); + await expect(invocation.getDefaultPermission()).resolves.toBe('ask'); + }); + it('downscales with the fixed-call defaults, audio stream-copied', async () => { probe({ height: 1080, frameRate: 30 }); const { result, signal } = await run(); diff --git a/packages/core/src/omni/policy/tools/downscale-video.ts b/packages/core/src/omni/policy/tools/downscale-video.ts index daaa7bcc65f..204605dc0a5 100644 --- a/packages/core/src/omni/policy/tools/downscale-video.ts +++ b/packages/core/src/omni/policy/tools/downscale-video.ts @@ -11,11 +11,12 @@ import type { ToolInvocation, ToolResult, } from '../../../tools/tools.js'; -import { BaseToolInvocation, Kind } from '../../../tools/tools.js'; +import { Kind } from '../../../tools/tools.js'; import { probeMediaMetadata, runFfmpeg } from '../../ffmpeg.js'; import { assertMediaPolicyIo, BaseMediaPolicyTool, + BaseMediaPolicyToolInvocation, formatBytesShort, MEDIA_POLICY_IO_SCHEMA_PROPERTIES, mediaPolicyToolError, @@ -107,10 +108,7 @@ const DESCRIPTOR: MediaPolicyToolDescriptor = { }, }; -class DownscaleVideoInvocation extends BaseToolInvocation< - DownscaleVideoParams, - ToolResult -> { +class DownscaleVideoInvocation extends BaseMediaPolicyToolInvocation { constructor( params: DownscaleVideoParams, private readonly timeoutMs: number, diff --git a/packages/core/src/omni/policy/tools/media-policy-tool.ts b/packages/core/src/omni/policy/tools/media-policy-tool.ts index 55a2b6e6b94..ea600dde19d 100644 --- a/packages/core/src/omni/policy/tools/media-policy-tool.ts +++ b/packages/core/src/omni/policy/tools/media-policy-tool.ts @@ -14,7 +14,11 @@ import type { ToolResult, } from '../../../tools/tools.js'; import type { Kind } from '../../../tools/tools.js'; -import { BaseDeclarativeTool } from '../../../tools/tools.js'; +import { + BaseDeclarativeTool, + BaseToolInvocation, +} from '../../../tools/tools.js'; +import type { PermissionDecision } from '../../../permissions/types.js'; import { ToolErrorType } from '../../../tools/tool-error.js'; import { SchemaValidator } from '../../../utils/schemaValidator.js'; import { projectMediaPolicyToolDeclaration } from '../model-access.js'; @@ -75,6 +79,23 @@ export function resolvePolicyToolTimeoutMs( : DEFAULT_POLICY_TOOL_TIMEOUT_MS; } +/** + * Base invocation for media-policy tools. These invocations spawn + * ffmpeg/sharp and WRITE files (with overwrite) at the caller-chosen + * `outputDir`, so they are side-effecting: a model-origin call must be + * confirmation-gated like Write/Edit rather than inherit the read-only + * `'allow'` default. The fixed-policy path is unaffected — the scheduler + * skips the permission flow entirely for `fixed_policy` origin, and the + * orchestrator pins `outputDir` to the invocation's staging directory. + */ +export abstract class BaseMediaPolicyToolInvocation< + TParams extends object, +> extends BaseToolInvocation { + override getDefaultPermission(): Promise { + return Promise.resolve('ask'); + } +} + /** * Base class for omni media-policy tools (real DeclarativeTools — the * orchestrator executes them through the ordinary scheduler path, and diff --git a/packages/core/src/omni/recovery.test.ts b/packages/core/src/omni/recovery.test.ts index 56ae2830f4d..a665ff616ae 100644 --- a/packages/core/src/omni/recovery.test.ts +++ b/packages/core/src/omni/recovery.test.ts @@ -242,7 +242,13 @@ describe('runStartupRecoveryOnce', () => { }); describe('staging sweep (storage design §6.1: uncommitted work is deleted)', () => { - it('deletes every staging entry, including nested artifact trees and stray files', async () => { + /** Age a staging entry past the multi-process grace window (1h). */ + async function ageEntry(p: string): Promise { + const when = new Date(Date.now() - 2 * 3600_000); + await fs.utimes(p, when, when); + } + + it('deletes every stale staging entry, including nested artifact trees and stray files', async () => { const stagingDir = store.getStagingDir(); const invocationDir = path.join(stagingDir, '0123456789abcdef'); await fs.mkdir(path.join(invocationDir, 'nested'), { recursive: true }); @@ -250,13 +256,47 @@ describe('runStartupRecoveryOnce', () => { path.join(invocationDir, 'nested', 'artifact.webp'), 'half-written', ); - await fs.writeFile(path.join(stagingDir, 'stray.tmp'), 'stray'); + const stray = path.join(stagingDir, 'stray.tmp'); + await fs.writeFile(stray, 'stray'); + await ageEntry(invocationDir); + await ageEntry(stray); await runStartupRecoveryOnce(store); await expect(fs.readdir(stagingDir)).resolves.toEqual([]); }); + it('keeps entries younger than the grace window (a concurrent process may still be transcoding into them)', async () => { + const stagingDir = store.getStagingDir(); + const liveDir = path.join(stagingDir, 'fedcba9876543210'); + await fs.mkdir(liveDir, { recursive: true }); + await fs.writeFile(path.join(liveDir, 'artifact.mp4'), 'in-flight'); + + await runStartupRecoveryOnce(store); + + await expect( + fs.readFile(path.join(liveDir, 'artifact.mp4'), 'utf8'), + ).resolves.toBe('in-flight'); + }); + + it('removes a symlink ENTRY regardless of age without following it', async () => { + const outside = await fs.mkdtemp(path.join(os.tmpdir(), 'omni-stage-')); + const victim = path.join(outside, 'victim.bin'); + await fs.writeFile(victim, 'external'); + try { + const stagingDir = store.getStagingDir(); + const link = path.join(stagingDir, 'planted-link'); + await fs.symlink(outside, link); + + await runStartupRecoveryOnce(store); + + await expect(fs.lstat(link)).rejects.toThrow(); + await expect(fs.readFile(victim, 'utf8')).resolves.toBe('external'); + } finally { + await fs.rm(outside, { recursive: true, force: true }); + } + }); + it('a symlinked staging ROOT is never swept', async () => { const outside = await fs.mkdtemp(path.join(os.tmpdir(), 'omni-stage-')); const victim = path.join(outside, 'victim.bin'); diff --git a/packages/core/src/omni/recovery.ts b/packages/core/src/omni/recovery.ts index 0362c63b53b..8b354f9e26f 100644 --- a/packages/core/src/omni/recovery.ts +++ b/packages/core/src/omni/recovery.ts @@ -31,6 +31,13 @@ const SAMPLE_VERIFY_MAX_BYTES = 64 * 1024 * 1024; * belong to a promotion in flight in ANOTHER process — deleting it would * fail that process's rename. Older survivors are crash leftovers. */ const TMP_GRACE_MS = 3600_000; +/** Grace window for staging entries, for the same multi-process reason: + * a second CLI process starting while another is mid-transcode must not + * delete the live invocation's work directory out from under its tool. + * One hour comfortably exceeds the 10-minute default policy-tool timeout + * (a directory's mtime is set at creation), so anything older is a crash + * leftover, not an in-flight run. */ +const STAGING_GRACE_MS = 3600_000; /** Default retention for quarantined invocations (storage design §7). */ const QUARANTINE_RETENTION_DAYS = 7; /** Default size budget for the quarantine area (storage design §7). */ @@ -101,12 +108,14 @@ async function sweepDownloads(downloadsDir: string): Promise { } /** - * Delete EVERYTHING under `staging/`. Staging entries belong to policy - * invocations that never committed (a successful commit deletes its own - * staging directory first), so at startup there is nothing to keep - * (storage design §6.1). The staging root itself must be a real directory - * — a symlinked root would redirect the recursive deletes outside the - * omni root. + * Delete crash-orphaned entries under `staging/`. Staging entries belong + * to policy invocations that never committed (a successful commit deletes + * its own staging directory first), so anything past the grace window is + * garbage (storage design §6.1). Entries YOUNGER than the grace window + * are kept: they may be a concurrent process's live invocation, and its + * own commit/quarantine path cleans them up. The staging root itself must + * be a real directory — a symlinked root would redirect the recursive + * deletes outside the omni root. */ async function sweepStaging(stagingDir: string): Promise { if (!(await isRealDirectory(stagingDir))) return; @@ -117,13 +126,18 @@ async function sweepStaging(stagingDir: string): Promise { return; } for (const name of names) { + const p = path.join(stagingDir, name); try { - // rm on a symlink entry removes the link itself without following - // it, so no containment check is needed per entry. - await fs.rm(path.join(stagingDir, name), { - recursive: true, - force: true, - }); + const st = await fs.lstat(p); + // A young REAL entry may belong to an in-flight invocation in + // another process; symlinks are never live invocations (staging + // dirs are created with mkdir) and are removed regardless of age + // (rm on a symlink removes the link itself without following it, + // so no containment check is needed per entry). + if (!st.isSymbolicLink() && Date.now() - st.mtimeMs < STAGING_GRACE_MS) { + continue; + } + await fs.rm(p, { recursive: true, force: true }); debugLogger.debug(`recovery: removed uncommitted staging ${name}`); } catch { // Best-effort sweep. @@ -328,8 +342,10 @@ async function sampleVerifyObjects( * lazily the first time the omni pipeline is touched — zero cost when * omni is unused. * - * 1. everything under `staging/` is deleted — staging entries belong to - * policy invocations that never committed (storage design §6.1); + * 1. staging entries older than the multi-process grace window are + * deleted — they belong to policy invocations that never committed; + * younger entries may be another process's live run (storage design + * §6.1); * 2. crash-orphaned `downloads/*.part` older than the 48h debugging * retention window are removed; * 3. `quarantine/` is trimmed to its retention window and size budget diff --git a/packages/core/src/omni/storage.test.ts b/packages/core/src/omni/storage.test.ts index eb4b152c3af..3fb4fee1be6 100644 --- a/packages/core/src/omni/storage.test.ts +++ b/packages/core/src/omni/storage.test.ts @@ -99,6 +99,47 @@ describe('OmniObjectStore', () => { ); }); + describe('objectPathFor validates cache-sourced components (path traversal)', () => { + const sha256 = 'a'.repeat(64); + + it.each([ + [ + 'traversal hash', + '../../../../etc/passwd', + '.mp4', + /invalid object hash/, + ], + ['uppercase hash', 'A'.repeat(64), '.mp4', /invalid object hash/], + ['short hash', 'abc123', '.mp4', /invalid object hash/], + [ + 'traversal extension', + sha256, + '/../../../../tmp/evil', + /invalid object extension/, + ], + [ + 'multi-segment extension', + sha256, + '.jpg/../x', + /invalid object extension/, + ], + ['dotless extension', sha256, 'jpg', /invalid object extension/], + ['double-dot extension', sha256, '..', /invalid object extension/], + ['overlong extension', sha256, '.abcdefghi', /invalid object extension/], + ])('throws on %s', (_label, hash, ext, message) => { + expect(() => store.objectPathFor(hash, ext)).toThrow(message); + }); + + it('accepts every extension recognition can emit', () => { + for (const ext of ['.mp4', '.webp', '.m4a', '.bin', '.jpg']) { + const p = store.objectPathFor(sha256, ext); + expect(p).toBe( + path.join(store.getObjectsDir(), 'aa', `${sha256}${ext}`), + ); + } + }); + }); + it('propagates copy failures without leaving temp files', async () => { const missing = path.join(qwenDir, 'does-not-exist.mp4'); const sha256 = createHash('sha256').update('missing').digest('hex'); diff --git a/packages/core/src/omni/storage.ts b/packages/core/src/omni/storage.ts index 7a94b078435..f90d33c4f44 100644 --- a/packages/core/src/omni/storage.ts +++ b/packages/core/src/omni/storage.ts @@ -35,6 +35,12 @@ export interface QuarantineReason { * filesystem so staging/quarantine paths can never escape their area. */ const INVOCATION_ID_RE = /^[0-9a-f]{16}$/; +/** Object-store extensions are a single dotted alphanumeric component + * (".jpg", ".m4a", ".bin" — see extensionForMime). Anything else — path + * separators, dots beyond the leading one — is rejected so an extension + * can never smuggle traversal segments into an object path. */ +export const OBJECT_EXTENSION_RE = /^\.[A-Za-z0-9]{1,8}$/; + function assertInvocationId(invocationId: string): void { if (!INVOCATION_ID_RE.test(invocationId)) { throw new Error(`Invalid omni policy invocation id: ${invocationId}`); @@ -92,8 +98,9 @@ export class OmniObjectStore { return path.join(this.omniRoot, 'objects', 'sha256'); } - /** Root of the policy-invocation work area (deleted wholesale by - * startup recovery — anything here belongs to an uncommitted run). */ + /** Root of the policy-invocation work area (stale entries deleted by + * startup recovery — anything past the grace window belongs to an + * uncommitted, crashed run). */ getStagingDir(): string { return path.join(this.omniRoot, 'staging'); } @@ -104,8 +111,21 @@ export class OmniObjectStore { return path.join(this.omniRoot, 'quarantine'); } - /** Compute the final object path for a content hash + extension. */ + /** + * Compute the final object path for a content hash + extension. + * + * Both components are validated here, not just at putFile: callers may + * feed values read back from on-disk cache files (policy-cache.json), + * and a crafted hash or extension ("/../../…") would otherwise turn + * this join into a path-traversal primitive pointing outside the store. + */ objectPathFor(sha256: string, extension: string): string { + if (!/^[0-9a-f]{64}$/.test(sha256)) { + throw new Error(`invalid object hash: ${JSON.stringify(sha256)}`); + } + if (!OBJECT_EXTENSION_RE.test(extension)) { + throw new Error(`invalid object extension: ${JSON.stringify(extension)}`); + } return path.join( this.getObjectsDir(), sha256.slice(0, 2), @@ -182,7 +202,7 @@ export class OmniObjectStore { * a `reason.json` (storage design §4.4). The reason file is written into * the staging directory BEFORE the rename so the quarantine entry appears * complete in one atomic step; a crash in between leaves it in staging, - * which startup recovery deletes wholesale. + * which startup recovery deletes once past the grace window. */ async quarantineInvocation( invocationId: string, From 5cb041c49537c2dbaf49ba65d72aca3680e04cc2 Mon Sep 17 00:00:00 2001 From: LaZzyMan Date: Fri, 7 Aug 2026 15:32:16 +0800 Subject: [PATCH 13/62] fix(omni): harden policy pipeline after self-review round Orchestrator: exclude animated images from policy matching (D9), self-heal unverifiable degradation-cache entries, key cache fingerprints with the descriptor version, and warn when reprocess origins exclude 'policy'. Scheduler: fixed_policy invocations no longer bounce PreToolUse 'ask' to an unanswerable awaiting_approval (deny stays fail-closed), and processToolResultImages short-circuits for fixed_policy results at the method level so all three call sites skip the omni re-delivery / vision bridge re-entrancy. Delivery/CLI: chain preprocessing and guard disclosures instead of replacing (D8), exclude hidden media-policy tools from the /context per-tool breakdown to keep it aligned with getFunctionDeclarations(), annotate fixed-only media-policy tools in /tools, and drop the stale maxConcurrency mention from the policyTools settings description. Tests: add negative coverage for artifact validation (no artifacts, undeclared media type, kind mismatch, missing required output) and for the fixed_policy hook ask/deny and image-funnel paths. --- packages/cli/src/config/settingsSchema.ts | 2 +- .../src/ui/commands/contextCommand.test.ts | 55 +++++ .../cli/src/ui/commands/contextCommand.ts | 8 + .../cli/src/ui/commands/toolsCommand.test.ts | 55 +++++ packages/cli/src/ui/commands/toolsCommand.ts | 9 + .../ui/components/views/ToolsList.test.tsx | 24 ++ .../cli/src/ui/components/views/ToolsList.tsx | 6 + packages/cli/src/ui/types.ts | 3 + .../core/src/core/coreToolScheduler.test.ts | 164 +++++++++++++ packages/core/src/core/coreToolScheduler.ts | 22 +- packages/core/src/omni/index.test.ts | 55 +++++ packages/core/src/omni/index.ts | 13 + packages/core/src/omni/policy/config.test.ts | 57 +++++ packages/core/src/omni/policy/config.ts | 33 +++ .../core/src/omni/policy/orchestrator.test.ts | 227 ++++++++++++++++++ packages/core/src/omni/policy/orchestrator.ts | 89 ++++--- .../policy/tools/downsample-audio.test.ts | 1 + .../src/omni/policy/tools/downsample-audio.ts | 1 + .../policy/tools/downsample-image.test.ts | 1 + .../src/omni/policy/tools/downsample-image.ts | 1 + .../omni/policy/tools/downscale-video.test.ts | 1 + .../src/omni/policy/tools/downscale-video.ts | 1 + packages/core/src/omni/recovery.ts | 6 +- packages/core/src/tools/tools.ts | 7 + .../schemas/settings.schema.json | 2 +- 25 files changed, 809 insertions(+), 34 deletions(-) diff --git a/packages/cli/src/config/settingsSchema.ts b/packages/cli/src/config/settingsSchema.ts index eed1d7a4313..d02c345e02a 100644 --- a/packages/cli/src/config/settingsSchema.ts +++ b/packages/cli/src/config/settingsSchema.ts @@ -3860,7 +3860,7 @@ const SETTINGS_SCHEMA = { default: {} as Record | null>, description: 'Per-tool overrides keyed by policy tool name: settings ' + - '(default arguments), runtime (timeoutMs, maxConcurrency), ' + + '(default arguments), runtime (timeoutMs), ' + 'and modelAccess (enabled, defaultArguments, lockedArguments, ' + 'parameterSchema, output).', showInDialog: false, diff --git a/packages/cli/src/ui/commands/contextCommand.test.ts b/packages/cli/src/ui/commands/contextCommand.test.ts index 45c0714cd8f..4e20dd541a1 100644 --- a/packages/cli/src/ui/commands/contextCommand.test.ts +++ b/packages/cli/src/ui/commands/contextCommand.test.ts @@ -236,6 +236,61 @@ describe('collectContextData (contextCommand)', () => { expect(data.builtinTools[0].name).toBe('web_fetch'); }); + it('excludes fixed-only media-policy tools from the per-tool breakdown (D6)', async () => { + // A media-policy tool without modelAccess.enabled is stripped from + // getFunctionDeclarations() (zero prompt tokens), so listing it in the + // breakdown would make the per-tool sum exceed allToolsTokens. One with + // modelAccess.enabled IS declared to the model and must stay listed. + const descriptor = { + kind: 'media_policy', + inputMediaTypes: ['image'], + outputs: [], + }; + const hiddenPolicyTool = { + name: 'omni_downsample_image', + schema: { name: 'omni_downsample_image', description: 'policy schema' }, + mediaPolicyDescriptor: descriptor, + }; + const exposedPolicyTool = { + name: 'omni_probe_media', + schema: { name: 'omni_probe_media', description: 'probe schema' }, + mediaPolicyDescriptor: descriptor, + }; + const config = { + getModel: vi.fn().mockReturnValue('test-model'), + getContentGeneratorConfig: vi.fn().mockReturnValue({ + contextWindowSize: 32_000, + }), + getToolRegistry: vi.fn().mockReturnValue({ + getAllTools: vi + .fn() + .mockReturnValue([hiddenPolicyTool, exposedPolicyTool]), + getFunctionDeclarations: vi + .fn() + .mockReturnValue([exposedPolicyTool.schema]), + isDeferredAndHidden: vi.fn().mockReturnValue(false), + }), + getOmniPolicyToolsSettings: vi.fn().mockReturnValue({ + omni_probe_media: { modelAccess: { enabled: true } }, + }), + getVisibleTools: vi.fn().mockReturnValue(new Set()), + getUserMemory: vi.fn().mockReturnValue(''), + getAutoMemoryPrompt: vi.fn().mockReturnValue(''), + getSkillManager: vi.fn().mockReturnValue({ + listSkills: vi.fn().mockResolvedValue([]), + }), + getChatCompression: vi.fn().mockReturnValue(undefined), + getAutoCompactThreshold: vi.fn(), + getExperimentalZedIntegration: vi.fn().mockReturnValue(false), + isInteractive: vi.fn().mockReturnValue(true), + } as unknown as Config; + + const data = await collectContextData(config, true); + + expect(data.builtinTools).toHaveLength(1); + expect(data.builtinTools[0].name).toBe('omni_probe_media'); + }); + it('lists the auto-memory section as a separate memory entry (#7651)', async () => { // The managed auto-memory section is no longer part of getUserMemory(); its // tokens are surfaced via getAutoMemoryPrompt(). Exercise the non-empty diff --git a/packages/cli/src/ui/commands/contextCommand.ts b/packages/cli/src/ui/commands/contextCommand.ts index 2f24d17ff87..664cd50b0e0 100644 --- a/packages/cli/src/ui/commands/contextCommand.ts +++ b/packages/cli/src/ui/commands/contextCommand.ts @@ -27,6 +27,7 @@ import { ToolNames, buildSkillLlmContent, computeThresholds, + isMediaPolicyToolHiddenFromModel, type CompactionThresholds, } from '@qwen-code/qwen-code-core'; import { t } from '../../i18n/index.js'; @@ -152,6 +153,13 @@ export async function collectContextData( if (toolRegistry?.isDeferredAndHidden(tool.name)) { continue; } + // Same alignment rule for omni media-policy tools: fixed-only tools + // (declared descriptor, modelAccess not enabled) are stripped from + // getFunctionDeclarations() and cost the model zero prompt tokens, so + // listing them here would make the breakdown sum exceed allToolsTokens. + if (isMediaPolicyToolHiddenFromModel(config, tool)) { + continue; + } const toolJsonStr = JSON.stringify(tool.schema); const tokens = estimateTokens(toolJsonStr); if (tool instanceof DiscoveredMCPTool) { diff --git a/packages/cli/src/ui/commands/toolsCommand.test.ts b/packages/cli/src/ui/commands/toolsCommand.test.ts index 9e1eae83645..25643b3c985 100644 --- a/packages/cli/src/ui/commands/toolsCommand.test.ts +++ b/packages/cli/src/ui/commands/toolsCommand.test.ts @@ -111,4 +111,59 @@ describe('toolsCommand', () => { ); expect(message.tools[1].description).toBe('Edits code files.'); }); + + it('flags hidden media-policy tools as fixedOnly, but not model-enabled ones', async () => { + const mediaTools = [ + { + name: 'omni_downsample_image', + displayName: 'DownsampleImage', + description: 'Downsamples an image.', + schema: {}, + // Media-policy tool with no modelAccess entry → hidden from the + // model's declarations → must surface as fixed-only in /tools. + mediaPolicyDescriptor: { + kind: 'media_policy', + inputMediaTypes: ['image'], + outputs: [], + }, + }, + { + name: 'omni_probe_media', + displayName: 'ProbeMedia', + description: 'Probes media metadata.', + schema: {}, + // Same descriptor, but modelAccess.enabled below re-exposes it to + // the model, so it must NOT carry the fixed-only marker. + mediaPolicyDescriptor: { + kind: 'media_policy', + inputMediaTypes: ['image'], + outputs: [], + }, + }, + ...mockTools, + ] as Tool[]; + const mockContext = createMockCommandContext({ + services: { + config: { + getToolRegistry: () => ({ getAllTools: () => mediaTools }), + getOmniPolicyToolsSettings: () => ({ + omni_probe_media: { modelAccess: { enabled: true } }, + }), + }, + }, + }); + + if (!toolsCommand.action) throw new Error('Action not defined'); + await toolsCommand.action(mockContext, ''); + + const [message] = (mockContext.ui.addItem as vi.Mock).mock.calls[0]; + expect(message.tools).toHaveLength(4); + expect(message.tools[0]).toMatchObject({ + name: 'omni_downsample_image', + fixedOnly: true, + }); + expect(message.tools[1].fixedOnly).toBeUndefined(); + expect(message.tools[2].fixedOnly).toBeUndefined(); + expect(message.tools[3].fixedOnly).toBeUndefined(); + }); }); diff --git a/packages/cli/src/ui/commands/toolsCommand.ts b/packages/cli/src/ui/commands/toolsCommand.ts index 5c6625e6d35..d9698f3b3d9 100644 --- a/packages/cli/src/ui/commands/toolsCommand.ts +++ b/packages/cli/src/ui/commands/toolsCommand.ts @@ -10,6 +10,7 @@ import { CommandKind, } from './types.js'; import { MessageType, type HistoryItemToolsList } from '../types.js'; +import { isMediaPolicyToolHiddenFromModel } from '@qwen-code/qwen-code-core'; import { t } from '../../i18n/index.js'; export const toolsCommand: SlashCommand = { @@ -38,6 +39,7 @@ export const toolsCommand: SlashCommand = { ); return; } + const config = context.services.config!; const tools = toolRegistry.getAllTools(); // Filter out MCP tools by checking for the absence of a serverName property @@ -49,6 +51,13 @@ export const toolsCommand: SlashCommand = { name: tool.name, displayName: tool.displayName, description: tool.description, + // Omni media-policy tools without modelAccess.enabled are stripped + // from the model's declarations but stay listed here for the human; + // the flag renders a "fixed-only" marker so the discrepancy between + // /tools and what the model can call is visible, not confusing. + ...(isMediaPolicyToolHiddenFromModel(config, tool) + ? { fixedOnly: true } + : {}), })), showDescriptions: useShowDescriptions, }; diff --git a/packages/cli/src/ui/components/views/ToolsList.test.tsx b/packages/cli/src/ui/components/views/ToolsList.test.tsx index ae6acd12016..839331942e0 100644 --- a/packages/cli/src/ui/components/views/ToolsList.test.tsx +++ b/packages/cli/src/ui/components/views/ToolsList.test.tsx @@ -55,4 +55,28 @@ describe('', () => { ); expect(lastFrame()).toMatchSnapshot(); }); + + it('marks fixed-only media-policy tools and leaves others unmarked', () => { + const tools: ToolDefinition[] = [ + { + name: 'omni_downsample_image', + displayName: 'DownsampleImage', + fixedOnly: true, + }, + { name: 'read_file', displayName: 'ReadFile' }, + ]; + const { lastFrame } = render( + , + ); + const frame = lastFrame() ?? ''; + // The marker must be attached to the fixed-only tool's line only. + const downsampleLine = frame + .split('\n') + .find((line) => line.includes('DownsampleImage')); + expect(downsampleLine).toContain('[fixed-only'); + const readFileLine = frame + .split('\n') + .find((line) => line.includes('ReadFile')); + expect(readFileLine).not.toContain('fixed-only'); + }); }); diff --git a/packages/cli/src/ui/components/views/ToolsList.tsx b/packages/cli/src/ui/components/views/ToolsList.tsx index d397c1002f7..de9efd4e9d3 100644 --- a/packages/cli/src/ui/components/views/ToolsList.tsx +++ b/packages/cli/src/ui/components/views/ToolsList.tsx @@ -35,6 +35,12 @@ export const ToolsList: React.FC = ({ {tool.displayName} {showDescriptions ? ` (${tool.name})` : ''} + {tool.fixedOnly && ( + + {' '} + {t('[fixed-only: runs via media policies, not the model]')} + + )} {showDescriptions && tool.description && (