diff --git a/docs/design/2026-07-29-omni-multimodal-policy-orchestration.md b/docs/design/2026-07-29-omni-multimodal-policy-orchestration.md index cb0ccb1ca49..eb07ee8591c 100644 --- a/docs/design/2026-07-29-omni-multimodal-policy-orchestration.md +++ b/docs/design/2026-07-29-omni-multimodal-policy-orchestration.md @@ -502,20 +502,25 @@ priority 排序。 ### 8.3 匹配条件 -`when` 使用受限条件 DSL，不接受任意 JavaScript 或 JSONPath。条件由 `all`、`any` -和 comparison 递归组成；comparison 支持 `gt`、`gte`、`lt`、`lte`、`eq`，左右两侧 -都可以是字段或字面量： +`when` 使用受限的表达式条件 DSL（Mapbox style-spec 风格的表达式数组），不接受 +任意 JavaScript 或 JSONPath。表达式形如 `[operator, ...operands]`：comparison +支持 `>`、`>=`、`<`、`<=`、`==`、`!=`，恰好两个操作数，操作数是 +`["field", ""]` 字段引用或裸字面量；组合子 `["all", , ...]`、 +`["any", , ...]`、`["!", ]` 递归嵌套： ```ts -type ConditionOperand = - | { field: FixedPolicyField } - | { value: number | string | boolean }; - -interface ComparisonCondition { - left: ConditionOperand; - operator: 'gt' | 'gte' | 'lt' | 'lte' | 'eq'; - right: ConditionOperand; -} +type ConditionOperand = ['field', FixedPolicyField] | number | string | boolean; + +type ComparisonCondition = [ + '>' | '>=' | '<' | '<=' | '==' | '!=', + ConditionOperand, + ConditionOperand, +]; + +type FixedPolicyCondition = + | ComparisonCondition + | ['all' | 'any', ...FixedPolicyCondition[]] + | ['!', FixedPolicyCondition]; ``` 可读字段分为三个自然命名空间： @@ -547,20 +552,15 @@ availableContextTokens = max( ```jsonc { - "when": { - "all": [ - { - "left": { "field": "resource.estimatedTokenCount" }, - "operator": "gt", - "right": { "field": "session.availableContextTokens" }, - }, - { - "left": { "field": "session.contextWindowTokens" }, - "operator": "gte", - "right": { "value": 131072 }, - }, + "when": [ + "all", + [ + ">", + ["field", "resource.estimatedTokenCount"], + ["field", "session.availableContextTokens"], ], - }, + [">=", ["field", "session.contextWindowTokens"], 131072], + ], } ``` @@ -1204,27 +1204,14 @@ normalizer，并用配置 snapshot 测试防止两处示例漂移。 "mediaTypes": ["image"], }, // 定义触发该 Fixed Policy 的 metadata 条件。 - "when": { + "when": [ // 任意一个条件成立即可执行。 - "any": [ - { - // 检查图片宽度。 - "left": { "field": "resource.width" }, - // 使用大于比较。 - "operator": "gt", - // 宽度超过 2000 像素时命中。 - "right": { "value": 2000 }, - }, - { - // 检查图片高度。 - "left": { "field": "resource.height" }, - // 使用大于比较。 - "operator": "gt", - // 高度超过 2000 像素时命中。 - "right": { "value": 2000 }, - }, - ], - }, + "any", + // 图片宽度超过 2000 像素时命中。 + [">", ["field", "resource.width"], 2000], + // 图片高度超过 2000 像素时命中。 + [">", ["field", "resource.height"], 2000], + ], // 执行图片降采样的 Tool 名。 "tool": "omni_downsample_image", // 固定调用时使用的降采样参数。 @@ -1264,33 +1251,19 @@ normalizer，并用配置 snapshot 测试防止两处示例漂移。 "mediaTypes": ["video"], }, // 比较本轮全部媒体的预估 token 与当前 session 可用上下文。 - "when": { + "when": [ // 所有条件均满足时才执行。 - "all": [ - { - // 左值是本轮待发送媒体的预估 token 总量。 - "left": { - "field": "request.totalEstimatedMediaTokens", - }, - // 当预估媒体 token 大于可用上下文时命中。 - "operator": "gt", - // 可用上下文由窗口、已用 prompt 和预留输出 token 计算。 - "right": { - "field": "session.availableContextTokens", - }, - }, - { - // 确认当前视频本身具有可用的 token 估算值。 - "left": { - "field": "resource.estimatedTokenCount", - }, - // 大于零表示该资源已完成有效估算。 - "operator": "gt", - // 零是估算值有效性的下界。 - "right": { "value": 0 }, - }, + "all", + // 本轮待发送媒体的预估 token 总量大于可用上下文时命中； + // 可用上下文由窗口、已用 prompt 和预留输出 token 计算。 + [ + ">", + ["field", "request.totalEstimatedMediaTokens"], + ["field", "session.availableContextTokens"], ], - }, + // 确认当前视频本身具有可用的 token 估算值（大于零表示有效）。 + [">", ["field", "resource.estimatedTokenCount"], 0], + ], // 条件依赖值不可用时跳过，不误判为条件不成立。 "onConditionUnavailable": "skip", // 执行视频关键帧提取的 Tool 名。 diff --git a/docs/users/features/_meta.ts b/docs/users/features/_meta.ts index 247d04525cd..607f1019fb2 100644 --- a/docs/users/features/_meta.ts +++ b/docs/users/features/_meta.ts @@ -25,4 +25,5 @@ export default { 'status-line': 'Status Line', 'scheduled-tasks': 'Scheduled Tasks', tips: 'Contextual Tips', + 'omni-media-policies': 'Omni Media Policies', }; diff --git a/docs/users/features/omni-media-policies.md b/docs/users/features/omni-media-policies.md new file mode 100644 index 00000000000..1526ab6d2af --- /dev/null +++ b/docs/users/features/omni-media-policies.md @@ -0,0 +1,401 @@ +# Omni 媒体策略编排 + +> 实验特性:仅在 omni 实验分支可用,且仅支持 DashScope OpenAI 兼容模式(`qwen3.5-omni-*` 系列模型)。配置格式可能随实验推进调整,不承诺向后兼容。 + +当你把图片、音频、视频交给多模态模型时,原始文件往往过大(一部 1080p 电影约 1GB)或过长(81 分钟的音轨),直接投递要么超出上传通道限制,要么烧掉整个上下文窗口。**媒体策略编排**让你用配置声明"什么媒体、满足什么条件时、用哪个工具、降质成什么样再投递",整条流水线是: + +``` +识别媒体 → 匹配固定策略 → 派生降质产物(staging) → 内容寻址入库(objects) → 携带披露投递给模型 +``` + +三条铁律贯穿全程: + +1. **有损必披露**——任何降质产物送达模型时,紧邻的文本部件必须携带 `【媒体降质】`(或 `【媒体省略】`/`【媒体转写】`)标记,说明丢了什么; +2. **配置错误启动即失败**——策略引用不存在的工具、条件写错字段、矛盾的组合,一律在启动时报 `OmniPolicyConfigError`,绝不静默降级; +3. **条件不可判定不等于假**——探测不到的元数据字段产生 `unavailable` 三值结果,按策略声明的 `onConditionUnavailable` 处理,永不悄悄当作 `false`。 + +--- + +## 快速开始 + +在项目 `.qwen/settings.json`(或用户级 `~/.qwen/settings.json`)中: + +```json +{ + "omni": { + "enabled": true, + "processing": { + "fixedPolicies": { + "big-image-downsample": { + "mediaTypes": ["image"], + "when": [">", ["field", "resource.width"], 3000], + "toolName": "omni_downsample_image", + "arguments": { "maxDimension": 1568, "quality": 75 } + } + } + } + } +} +``` + +之后任何一张宽度超过 3000px 的图片(来自 `@` 引用、工具结果或 URL 摄取)都会先被降采样,再连同一条 `【媒体降质】原 4096×3072/8.2MB → 1568×1176/…` 披露投递给模型;宽度不超标的图片则原样直达。 + +零配置时 `fixedPolicies` 为空,预处理完全不运行——只有强制的传输守卫(见下文)在媒体超出上传通道限制时兜底。 + +--- + +## 配置总览 + +``` +omni.enabled 总开关 +omni.processing.fixedPolicies. 预处理策略(本文主角) +omni.processing.transportGuard.policies. 传输守卫策略(有系统默认,可覆盖不可删除) +omni.processing.transportGuard.maxUploadFileBytes 单文件上传上限(≤ 1GiB) +omni.processing.policyTools. 工具级设置 / 超时 / 模型可见性 +omni.processing.limits 每根资源的派生预算 +omni.delivery.upload.urlTtlHours 上传 URL 缓存时长(≤ 48) +``` + +--- + +## fixedPolicies:固定策略 + +每条策略是 `策略 id → 配置对象` 的一个键值对。id 需匹配 `^[A-Za-z0-9][A-Za-z0-9._-]*$`;值为 `null` 可删除(墓碑)更低作用域配置合并进来的同名策略。**未知键一律报错**,不会被忽略。 + +| 键 | 类型 / 取值 | 默认 | 说明 | +| ------------------------ | ------------------------------- | -------------------- | ------------------------------------------------------------------------------------------------------------------------ | +| `priority` | 有限数字 | `0` | 执行顺序:priority 降序,同分按 id 升序。 | +| `mediaTypes` | `("image"\|"video"\|"audio")[]` | 必填 | 策略适用的媒体模态,非空。 | +| `origins` | `("user"\|"tool"\|"policy")[]` | `["user","tool"]` | 接受哪些来源的媒体:用户 `@` 引用 / 工具结果 / 其他策略的派生物。 | +| `when` | 条件表达式数组 | 不设(总是匹配) | 见下节 DSL。 | +| `onConditionUnavailable` | `"skip"` \| `"run"` | `"skip"` | 条件字段探测不到时跳过还是照跑(`"abortTurn"` 为设计保留值,当前配置会被拒绝)。 | +| `toolName` | 字符串 | 必填 | 必须是已注册的媒体策略工具(见工具一览);被 `tools.disabled` 排除的工具视为未注册。 | +| `arguments` | 对象 | `{}` | 传给工具的固定参数,按工具的 `settingsSchema` 校验;`inputPath`/`outputDir`/`resourceId` 由编排器逐次注入,配置它们会报错。 | +| `maxRunsPerLineage` | 正整数 | `1` | 同一资源谱系上该策略最多执行几次,防循环。 | +| `onFailure` | `"continue"` \| `"abort"` | `"continue"` | 工具失败后继续匹配后续策略,还是中止本根资源的处理。 | +| `output.reprocessMedia` | 布尔 | `false` | 派生物以 `policy` 来源重新进入策略匹配(策略链)。 | +| `output.source` | `"keep"` \| `"omit"` | `"omit"` | 源媒体是保留在投递集中,还是被派生物替代。 | +| `output.artifacts` | 选择器映射 | `{ "*": "include" }` | 多产物工具的分流:`include` 进投递集,`retain` 只入库不投递。 | + +### output.artifacts 选择器 + +选择器有三种形态,匹配优先级 `role:` > `kind:` > `*`;配置了映射但没有任何选择器命中的产物按 `retain` 处理: + +```json +"output": { + "artifacts": { + "role:transcript": "include", + "kind:image": "retain", + "*": "include" + } +} +``` + +- `kind:` —— 按产物类别; +- `role:` —— 按工具声明的产物角色(如转写工具的 `transcript`); +- `*` —— 兜底。 + +启动校验会核对每个选择器确实是该工具能产出的类别/角色——永远匹配不到的选择器是配置错误,不是静默空操作。 + +--- + +## when 条件 DSL + +条件是 Mapbox 风格的表达式数组:`[操作符, ...操作数]`。 + +```json +[ + "all", + [">", ["field", "resource.durationMs"], 1200000], + ["!", ["<", ["field", "resource.sizeBytes"], 1048576]] +] +``` + +- **比较**:`>` `>=` `<` `<=` `==` `!=`,恰好两个操作数,各为 `["field", "<命名空间.字段>"]` 引用或字面量(数字/字符串/布尔);排序类操作符要求数字。 +- **组合**:`["all", …]`(与)、`["any", …]`(或)、`["!", ]`(非),可任意嵌套。 +- 不支持任意代码、JSONPath 或嵌套取值子表达式。 + +### 可用字段 + +| 命名空间 | 字段 | 含义 | +| ----------- | --------------------------- | ------------------------------------------------------------------------------ | +| `resource.` | `sizeBytes` | 文件字节数 | +| | `durationMs` | 时长(毫秒,音/视频) | +| | `width` / `height` | 像素宽高 | +| | `maxWidth` / `maxHeight` | `width` / `height` 的同值别名 | +| | `frameRate` / `frameCount` | 帧率 / 总帧数 | +| | `bitRate` | 比特率 | +| | `sampleRateHz` / `channels` | 采样率 / 声道数 | +| | `estimatedTokenCount` | 该资源的预估 token 消耗 | +| `request.` | `totalEstimatedMediaTokens` | 当前调度 pass 中该资源及其派生物的预估 token 总量(按资源逐个计算,非跨资源合计) | +| `session.` | `contextWindowTokens` | 模型上下文窗口 | +| | `promptTokenCount` | 当前 prompt 已占 token | +| | `reservedOutputTokens` | 预留输出 token | +| | `availableContextTokens` | 剩余可用上下文 | + +### 三值语义 + +字段探测不到(图片没有 `durationMs`、探针失败等)时,比较结果是 `unavailable` 而非 `false`。组合器用强 Kleene 逻辑传播:`all` 里只要有确定的假就是假,`any` 里只要有确定的真就是真,否则 `unavailable` 上浮,最终由该策略的 `onConditionUnavailable` 决定跳过(默认)还是执行,运行记录会写明缺失字段。 + +--- + +## 策略链:reprocessMedia + origins + +`output.reprocessMedia: true` 让派生物带着 `policy` 来源重新进入匹配,由声明了 `origins: ["policy", …]` 的下游策略接力。典型形态(电影音轨三级流水线): + +``` +extract-audio (origins:[user], reprocessMedia:true, source:keep) + └─▶ audio-downsample (origins:[policy], when: >100MB, reprocessMedia:true, source:omit) + └─▶ transcribe (origins:[policy], when: ≤100MB, source:omit) +``` + +启动校验会拒绝"惰性 reprocessMedia":某策略声明了 `reprocessMedia`,但同一集合里没有任何策略接受 `policy` 来源——派生物将永远无处可去,这是矛盾配置。 + +防失控的多重护栏:`maxRunsPerLineage`(单策略单谱系次数)、`limits.maxLineageDepth`(谱系深度)、`limits.maxPolicyRunsPerRoot`(单根总次数)。 + +--- + +## policyTools:工具级设置与模型可见性 + +```json +"policyTools": { + "omni_transcribe_audio": { + "settings": { "maxInputBytes": 33554432, "chunkSeconds": 300 }, + "runtime": { "timeoutMs": 1800000 }, + "modelAccess": { + "enabled": true, + "defaultArguments": { "chunkSeconds": 180 }, + "lockedArguments": {} + } + } +} +``` + +- **`settings`** —— 该工具的默认可调参数,按工具 `settingsSchema` 校验(如转写的 `maxInputBytes`、关键帧的 `sceneThreshold`)。 +- **`runtime.timeoutMs`** —— 单次调用的墙钟超时。必须小于 staging 清扫宽限窗(3600000ms = 1 小时),否则崩溃恢复清扫可能删掉仍在运行的调用的暂存目录,启动时直接报错。 +- **`modelAccess`** —— 默认所有策略工具**对模型不可见**,只能由固定策略驱动。逐工具打开: + - `enabled: true` —— 工具进入模型的工具列表,同时被写进系统提示的媒体引导小节(见"披露与渐进式理解"); + - `defaultArguments` —— 注入每次调用的默认参数,模型可覆盖; + - `lockedArguments` —— 锁定参数:从模型可见 schema 中剥除,模型执意传入会被硬性拒绝(可重试错误),执行时由 harness 注入。同一键出现在 defaults 和 locked 中是矛盾配置; + - `parameterSchema` —— 可选的可见 schema 投影,只允许收窄原生 schema,不能引入新属性。 + - 运维专用参数(`baseUrl`、`apiKeyEnv` 等)永远不会出现在模型可见 schema 中。 + +--- + +## 策略工具一览 + +八个内置媒体策略工具,全部基于 ffmpeg/sharp,产物均携带强制披露: + +| 工具 | 输入 → 输出 | 关键参数(默认) | 典型用途 | +| ------------------------ | --------------------- | ---------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `omni_downsample_image` | image → JPEG | `maxDimension` 1568、`quality` 75 | 超大截图降采样 | +| `omni_convert_image` | image → JPEG/PNG/WebP | `format` "jpeg"、`quality` 90 | 冷门格式转通用格式 | +| `omni_downscale_video` | video → MP4 | `maxHeight` 480、`fps` 10、`crf` 28 | 分辨率/帧率双降 | +| `omni_clip_video` | video → MP4 片段 | `startSec` 0、`durationSec`(默认到结尾)、`crf` 23 | 截取时间区间 | +| `omni_extract_keyframes` | video → 多张 JPEG | `maxFrames` 8、`sceneThreshold` 0.2、`maxDimension` 768 | **全片分桶采样**:时间轴均分为 maxFrames 桶,每桶开窗场景检测、无场景变化则取桶中点,逐帧披露带绝对时间戳 | +| `omni_extract_audio` | video → WAV/MP3/M4A | `format` "wav"、`sampleRateHz` 16000、`channels` 1 | 抽出音轨供转写 | +| `omni_downsample_audio` | audio → M4A | `bitrateKbps` 64、`sampleRateHz` 16000、`channels` 1 | 压音频体积 | +| `omni_transcribe_audio` | audio → 文本转写 | `chunkSeconds` 180、`maxInputBytes` 10MiB;运维专用:`model`、`baseUrl`、`apiKeyEnv` | **长音频分段转写**:超过 `chunkSeconds` 自动切段(并发 3、共享时间预算),按 `[MM:SS-MM:SS]`/`[H:MM:SS-…]` 拼装;单段失败内联标注不整体失败;自动检测并截断 ASR 重复退化 | + +多产物工具(关键帧)在一次调用事务里逐帧提升为独立产物;转写产物走 `【媒体转写】` 文本通道而非媒体通道。 + +--- + +## transportGuard:传输守卫 + +预处理是纯实验语义,守卫是**强制安全网**:当最终投递集仍超出传输限制(如 DashScope 单文件 100MB/1GiB 上限)时,守卫策略对超限媒体执行降质重试,并在披露中标注 `(服务端输入超限,第 N 次降质重试)`,最多 `limits.maxTransportPasses` 轮;仍不达标则以 `【媒体省略】` 通知替代媒体本身。 + +系统默认三条守卫(可整条覆盖、**不可删除**): + +```json +"transportGuard": { + "policies": { + "image-downsample": { "mediaTypes": ["image"], "toolName": "omni_downsample_image" }, + "video-downscale": { "mediaTypes": ["video"], "toolName": "omni_downscale_video" }, + "audio-downsample": { "mediaTypes": ["audio"], "toolName": "omni_downsample_audio" } + } +} +``` + +守卫策略的三条特殊约束(启动校验强制):合并后的守卫集必须覆盖 image/video/audio 三模态;不得声明 `when`(触发时机就是"超限"本身);`output.source` 必须为 `"omit"`(超限源不能留在投递集里)。 + +--- + +## limits:派生预算 + +| 键 | 默认 | 说明 | +| ------------------------ | ----- | ----------------------------------------- | +| `maxConcurrentResources` | 1 | 并发处理的根资源数 | +| `reservedOutputTokens` | 8192 | 为模型输出预留的 token(唯一允许为 0 的键) | +| `maxLineageDepth` | 8 | 策略链最大深度 | +| `maxPolicyRunsPerRoot` | 64 | 单根资源的策略执行总次数上限 | +| `maxArtifactsPerRoot` | 256 | 单根资源的产物总数上限 | +| `maxDerivedBytesPerRoot` | 1 GiB | 单根资源派生字节预算(电影场景建议调大) | +| `maxTransportPasses` | 3 | 传输守卫降质重试轮数 | + +已提交的投递不受预算追溯影响;预算耗尽只阻止新的派生。 + +--- + +## 披露与渐进式理解 + +投递给模型的每个降质产物都紧邻一条披露文本,三种标记: + +- `【媒体降质】<文件>:原 … → …,<丢失说明>` —— 有损派生物(截取、降采样、抽帧、降码率); +- `【媒体转写】<文件>:[时间段] 文本…` —— 以文本替代媒体本体的转写; +- `【媒体省略】<文件>:…` —— 完全无法投递时的占位通知。 + +同时,系统提示会注入 **Media Delivery (Progressive Understanding)** 小节,向模型说明:降质投递是"概览而非全部内容"、禁止在未获取证据的时间段上外推结论,并列出恰好那些 `modelAccess.enabled` 的工具,引导模型在需要更高保真证据时**主动调用工具取证**(裁剪某个片段、对某区间重新抽帧、转写某段音频)。没有任何工具开放时,该小节改为要求模型明确说明证据缺口。 + +## 缓存与存储 + +- **内容寻址库** `.qwen/omni/objects/sha256//.` —— 所有派生产物按内容哈希入库,同输入同策略永不重复派生(降质缓存键 = 源哈希 + 策略身份); +- **上传缓存** —— DashScope 临时上传的 `oss://` URL 按 `urlTtlHours`(≤ 48h)复用,重复投递不重复上传; +- **隔离区** `quarantine//` —— 失败派生的暂存残留连同 `reason.json` 移入隔离区,按保留天数和体积预算惰性清扫;崩溃留下的未提交 staging 超过 1 小时宽限窗后被回收。 + +--- + +## 完整案例:81 分钟电影拉片 + +目标:给 Qwen Code 一部 1080p 电影(《Breaking Surface》2020,81 分钟,986MB MKV)和一句"制作拉片网页"的 prompt,让模型在**从未看过全片**的情况下产出覆盖全片的交互式分析页面。 + +### 策略设计 + +一部电影拆成三路证据,全部在预处理阶段自动完成: + +1. **听觉全覆盖** —— 抽音轨 → 体积超 100MB 就先压码率 → 分段转写成带时间戳的全文台词; +2. **视觉全覆盖(稀疏)** —— 16 张关键帧按全片分桶采样,每帧披露带绝对时间戳; +3. **视觉细节(稠密,仅开头)** —— 截取前 10 分钟片段,给模型一段"高保真样本"感受影片质感。 + +### 工作区 `.qwen/settings.json` + +```json +{ + "omni": { + "enabled": true, + "processing": { + "fixedPolicies": { + "movie-extract-audio": { + "priority": 100, + "mediaTypes": ["video"], + "origins": ["user"], + "toolName": "omni_extract_audio", + "arguments": { + "format": "wav", + "sampleRateHz": 16000, + "channels": 1 + }, + "output": { "reprocessMedia": true, "source": "keep" } + }, + "movie-audio-downsample": { + "priority": 90, + "mediaTypes": ["audio"], + "origins": ["policy"], + "when": [">", ["field", "resource.sizeBytes"], 104857600], + "toolName": "omni_downsample_audio", + "arguments": { + "bitrateKbps": 24, + "sampleRateHz": 16000, + "channels": 1 + }, + "output": { "reprocessMedia": true, "source": "omit" } + }, + "movie-transcribe": { + "priority": 80, + "mediaTypes": ["audio"], + "origins": ["policy"], + "when": ["<=", ["field", "resource.sizeBytes"], 104857600], + "toolName": "omni_transcribe_audio", + "output": { "source": "omit" } + }, + "movie-clip-opening": { + "priority": 70, + "mediaTypes": ["video"], + "origins": ["user"], + "when": [">", ["field", "resource.durationMs"], 1200000], + "toolName": "omni_clip_video", + "arguments": { "startSec": 0, "durationSec": 600 }, + "output": { "source": "omit" } + }, + "movie-keyframes": { + "priority": 60, + "mediaTypes": ["video"], + "origins": ["user"], + "toolName": "omni_extract_keyframes", + "arguments": { "maxFrames": 16, "maxDimension": 960 }, + "output": { "source": "keep" } + } + }, + "policyTools": { + "omni_extract_audio": { + "modelAccess": { "enabled": true }, + "runtime": { "timeoutMs": 1800000 } + }, + "omni_downsample_audio": { "runtime": { "timeoutMs": 1800000 } }, + "omni_transcribe_audio": { + "settings": { "maxInputBytes": 33554432 }, + "modelAccess": { "enabled": true }, + "runtime": { "timeoutMs": 1800000 } + }, + "omni_clip_video": { + "modelAccess": { "enabled": true }, + "runtime": { "timeoutMs": 1800000 } + }, + "omni_extract_keyframes": { + "modelAccess": { "enabled": true }, + "runtime": { "timeoutMs": 1800000 } + }, + "omni_downscale_video": { + "modelAccess": { "enabled": true }, + "runtime": { "timeoutMs": 1800000 } + } + }, + "limits": { "maxDerivedBytesPerRoot": 4294967296 } + } + } +} +``` + +配置要点逐条对应前文机制: + +- **优先级排布**(100→60)保证抽音轨先于抽帧,音频链按"抽出→压缩→转写"接力; +- **策略链**:`movie-extract-audio` 的 WAV 音轨(约 150MB)带 `policy` 来源重新匹配——超 100MB 命中 `movie-audio-downsample` 压成 24kbps M4A(约 14MB),再次匹配命中 `movie-transcribe`; +- **`source` 取舍**:音轨和中间产物全部 `omit`(模型不需要听原始音频),关键帧 `keep` 源视频让开头片段与关键帧共存; +- **`when` 双向分流**:`>100MB` 与 `≤100MB` 互斥,压缩前后各走一条路,不会重复转写; +- **`modelAccess` 全开 + 引导小节**:模型可以对任意区间自行 clip/keyframes/transcribe 补证据; +- **预算调大**:986MB 源片的派生总量(WAV 音轨 + 片段 + 帧)远超默认 1GiB,`maxDerivedBytesPerRoot` 提到 4GiB; +- **转写 `maxInputBytes` 提到 32MiB**:容纳压缩后 14MB 的 M4A。 + +### 运行 + +```bash +qwen "请对 @tt10081762_1080p.mkv 做一次完整拉片,产出 lapian.html:电影简介、海报、主角剪影 SVG、可交互时间轴、关键事件列表及截图……" \ + --approval-mode yolo +``` + +### 实测结果(qwen3.5-omni-plus,真实 DashScope API) + +预处理阶段(首轮几分钟,之后全部缓存命中秒回): + +- 音频链:4882s 视频 → WAV → 24kbps M4A → **28 段转写、17725 字、0 段失败**,时间戳从 `[0:00:00-0:02:54]` 连续覆盖到 `[1:18:28-1:21:22]`; +- 16 张关键帧时间戳 `@ 152.6s → @ 4729.5s`,均匀铺满全片; +- 开头 600s 片段一段。 + +首个模型请求实际送达:1 段视频 + 16 张图 + 19 条披露 + 内联全文转写,披露样例: + +``` +【媒体降质】tt10081762_1080p.mkv:原 4882s → 片段 [0s–600s] 600s,片段外内容全部丢弃 +【媒体降质】tt10081762_1080p.mkv:原视频 4882s/1920×808 → 关键帧 7/16 @ 1830.8s,静态抽帧(全片分桶采样),时间连续性丢失 +【媒体降质】tt10081762_1080p.mkv:原 4882s 音频 → 分 28 段转写文本 17725 字,语气/音色/非语音信息丢失,识别可能有误 +【媒体转写】tt10081762_1080p.mkv:[0:00:00-0:02:54] Nej! Kom hit! … +``` + +传输守卫也被真实触发:600s 片段(128.8MB)遭服务端拒收后自动两级降质重试(`480p/24.1MB → 360p/13MB`),披露追加 `(服务端输入超限,第 N 次降质重试)`。 + +模型侧行为印证了渐进式理解引导:基于转写和关键帧建立全片骨架后,模型**主动发起了 18 次媒体工具调用**(7 次 clip、5 次 extract_audio、4 次 keyframes、2 次 transcribe,零错误)对关键场景补充取证,最终产出的 `lapian.html` 时间轴覆盖 `00:00 → 81:22` 全片。 + +### 已知边界 + +- 大体量投递对上传通道和 API 速率敏感:上传超时会以 `【媒体省略】`/失败通知降级(模型可靠工具自救),连续多个大请求可能触发 DashScope 速率保护——重试即可,预处理与上传缓存保证重跑近乎零成本; +- 转写基于 ASR,专有名词与多语言混杂段落可能有误,披露文本已固定声明"识别可能有误"; +- 关键帧是稀疏采样,单帧之间的动态信息永久丢失——这正是给模型开放 `omni_clip_video` 的意义。 diff --git a/packages/cli/src/acp-integration/session/Session.ts b/packages/cli/src/acp-integration/session/Session.ts index a34de78dceb..614ef7b0a26 100644 --- a/packages/cli/src/acp-integration/session/Session.ts +++ b/packages/cli/src/acp-integration/session/Session.ts @@ -162,6 +162,7 @@ import { runWithRuntimeContentGenerator, getInvocationContext, runWithInvocationContext, + evaluateMediaPolicyToolCall, } from '@qwen-code/qwen-code-core'; import { NOT_CURRENTLY_GENERATING_CANCEL_MESSAGE } from '@qwen-code/acp-bridge/bridgeErrors'; // Single source of truth shared with the daemon-side answerer (BridgeClient), @@ -7222,6 +7223,28 @@ export class Session implements SessionContext { ); } + // ---- Media-policy modelAccess gate (mirrors CoreToolScheduler) ---- + // Every ACP-originated call is a model call: there is no in-process + // fixed_policy caller on this path, so the origin is pinned rather + // than read from the (untrusted) protocol payload. + const mediaPolicyGate = evaluateMediaPolicyToolCall({ + config: this.config, + tool, + args, + executionOrigin: { kind: 'model' }, + }); + if (mediaPolicyGate.outcome === 'reject') { + return earlyErrorResponse( + new Error(mediaPolicyGate.message), + toolName, + { + recordInvalidToolParams: + mediaPolicyGate.reason === 'invalid_params', + }, + ); + } + args = mediaPolicyGate.args; + // Detect TodoWriteTool early - route to plan updates instead of tool_call events const isTodoWriteTool = tool.name === ToolNames.TODO_WRITE; // Core exposes TodoWriteTool as a type only. The bundle's keepNames diff --git a/packages/cli/src/config/config.ts b/packages/cli/src/config/config.ts index 26fa416e878..90a1f280190 100755 --- a/packages/cli/src/config/config.ts +++ b/packages/cli/src/config/config.ts @@ -38,6 +38,7 @@ import { SchemaValidator, type ConfigParameters, type MCPServerConfig, + type OmniPolicyToolsSettings, type WebSearchSettings, MAX_SUBAGENT_DEPTH_LIMIT, } from '@qwen-code/qwen-code-core'; @@ -2192,10 +2193,27 @@ export async function loadCliConfig( } : undefined, omniEnabled: settings.omni?.enabled ?? false, - omniUploadMaxFileBytes: settings.omni?.upload?.maxFileBytes, - omniMaxEstimatedTokens: settings.omni?.transport?.maxEstimatedTokens, - omniDownloadMaxFileBytes: settings.omni?.download?.maxFileBytes, - omniUploadCacheTtlHours: settings.omni?.upload?.cacheTtlHours, + omniMaxUploadFileBytes: + settings.omni?.processing?.transportGuard?.maxUploadFileBytes, + omniMaxEstimatedTokens: + settings.omni?.processing?.transportGuard?.maxEstimatedTokens, + omniUrlDownloadMaxFileBytes: + settings.omni?.ingestion?.localization?.url?.maxFileBytes, + omniUploadUrlTtlHours: settings.omni?.delivery?.upload?.urlTtlHours, + omniPolicyTools: settings.omni?.processing?.policyTools as + | OmniPolicyToolsSettings + | undefined, + omniFixedPolicies: settings.omni?.processing?.fixedPolicies as + | Record + | undefined, + omniTransportGuardPolicies: settings.omni?.processing?.transportGuard + ?.policies as Record | undefined, + omniProcessingLimits: settings.omni?.processing?.limits as + | Record + | undefined, + omniQuarantineRetentionDays: + settings.omni?.storage?.quarantine?.retentionDays, + omniQuarantineMaxBytes: settings.omni?.storage?.quarantine?.maxBytes, // CDP tunnel (Plan C, #5626): with the tunnel on, browser automation goes // through the CDP tunnel (far lighter than the OS-level computer-use // driver), so disable computer-use to keep the agent off that heavy path. diff --git a/packages/cli/src/config/settingsSchema.ts b/packages/cli/src/config/settingsSchema.ts index 498a6111e6b..fc549616014 100644 --- a/packages/cli/src/config/settingsSchema.ts +++ b/packages/cli/src/config/settingsSchema.ts @@ -3650,107 +3650,371 @@ const SETTINGS_SCHEMA = { 'endpoints. Can also be enabled via QWEN_CODE_ENABLE_OMNI=1.', showInDialog: true, }, - upload: { + processing: { type: 'object', - label: 'Omni Upload', + label: 'Omni Processing', category: 'Experimental', requiresRestart: true, default: {}, - description: 'Upload-channel limits for omni media delivery.', + description: + 'Media policy processing: fixed-policy orchestration, transport ' + + 'guard, per-root derivation limits, and policy tool overrides.', showInDialog: false, properties: { - maxFileBytes: { - type: 'number', - label: 'Max Upload File Bytes', + limits: { + type: 'object', + label: 'Omni Processing Limits', category: 'Experimental', requiresRestart: true, - default: 1073741824, + default: {}, description: - 'Per-file byte ceiling for omni media uploads. Defaults to ' + - '1 GiB, the DashScope temporary-upload per-file cap. Inputs ' + - 'above the limit fail closed with an explanatory error.', + 'Per-invocation derivation budgets. Exceeding a budget stops ' + + 'further derivation for that root resource (already committed ' + + 'artifacts stand).', showInDialog: false, - jsonSchemaOverride: { - type: 'number', - minimum: 1, - default: 1073741824, + properties: { + maxConcurrentResources: { + type: 'number', + label: 'Max Concurrent Resources', + category: 'Experimental', + requiresRestart: true, + default: 1, + description: + 'Number of media resources processed by policies in ' + + 'parallel within one request.', + showInDialog: false, + jsonSchemaOverride: { type: 'number', minimum: 1, default: 1 }, + }, + reservedOutputTokens: { + type: 'number', + label: 'Reserved Output Tokens', + category: 'Experimental', + requiresRestart: true, + default: 8192, + description: + 'Tokens reserved for model output when computing ' + + 'session.availableContextTokens for when-conditions.', + showInDialog: false, + jsonSchemaOverride: { + type: 'number', + minimum: 0, + default: 8192, + }, + }, + maxLineageDepth: { + type: 'number', + label: 'Max Lineage Depth', + category: 'Experimental', + requiresRestart: true, + default: 8, + description: + 'Maximum derivation chain length from a root resource.', + showInDialog: false, + jsonSchemaOverride: { type: 'number', minimum: 1, default: 8 }, + }, + maxPolicyRunsPerRoot: { + type: 'number', + label: 'Max Policy Runs Per Root', + category: 'Experimental', + requiresRestart: true, + default: 64, + description: + 'Maximum policy invocations attributable to one root ' + + 'resource within a single orchestrator run.', + showInDialog: false, + jsonSchemaOverride: { type: 'number', minimum: 1, default: 64 }, + }, + maxArtifactsPerRoot: { + type: 'number', + label: 'Max Artifacts Per Root', + category: 'Experimental', + requiresRestart: true, + default: 256, + description: + 'Maximum derived artifacts attributable to one root ' + + 'resource within a single orchestrator run.', + showInDialog: false, + jsonSchemaOverride: { + type: 'number', + minimum: 1, + default: 256, + }, + }, + maxDerivedBytesPerRoot: { + type: 'number', + label: 'Max Derived Bytes Per Root', + category: 'Experimental', + requiresRestart: true, + default: 1073741824, + description: + 'Byte budget for derived artifacts per root resource ' + + 'within a single orchestrator run. Defaults to 1 GiB.', + showInDialog: false, + jsonSchemaOverride: { + type: 'number', + minimum: 1, + default: 1073741824, + }, + }, + maxTransportPasses: { + type: 'number', + label: 'Max Transport Passes', + category: 'Experimental', + requiresRestart: true, + default: 3, + description: + 'Maximum transport-guard policy passes per resource before ' + + 'the media is removed with an explicit omission note.', + showInDialog: false, + jsonSchemaOverride: { type: 'number', minimum: 1, default: 3 }, + }, }, }, - cacheTtlHours: { - type: 'number', - label: 'Upload Cache TTL (hours)', + fixedPolicies: { + type: 'object', + label: 'Omni Fixed Policies', category: 'Experimental', requiresRestart: true, - default: 47, + default: {} as Record | null>, description: - 'Validity horizon for cached oss:// upload URLs. DashScope ' + - 'temporary uploads live 48h; the default keeps a 1h margin. ' + - '0 disables the upload cache (every delivery re-uploads).', + 'User fixed policies keyed by policy id. There are no ' + + 'built-in default policies: nothing runs unless configured ' + + 'here. Across settings scopes entries merge by id ' + + '(whole-entry replacement); a null entry tombstones a policy ' + + 'from a lower-priority scope. Validated and normalized at ' + + 'startup.', showInDialog: false, - jsonSchemaOverride: { - type: 'number', - minimum: 0, - default: 47, + mergeStrategy: MergeStrategy.SHALLOW_MERGE, + }, + transportGuard: { + type: 'object', + label: 'Omni Transport Guard', + category: 'Experimental', + requiresRestart: true, + default: {}, + description: + 'Delivery-boundary enforcement: hard limits plus mandatory ' + + 'guard policies applied when the final delivery set still ' + + 'exceeds limits. Cannot be disabled.', + showInDialog: false, + properties: { + maxUploadFileBytes: { + type: 'number', + label: 'Max Upload File Bytes', + category: 'Experimental', + requiresRestart: true, + default: 1073741824, + description: + 'Per-file byte ceiling for omni media uploads. Defaults ' + + 'to 1 GiB, the DashScope temporary-upload per-file cap ' + + '(values above it are a startup configuration error). ' + + 'Media still above the limit after guard policies fail ' + + 'closed with an explanatory error.', + showInDialog: false, + jsonSchemaOverride: { + type: 'number', + minimum: 1, + maximum: 1073741824, + default: 1073741824, + }, + }, + maxEstimatedTokens: { + type: 'number', + label: 'Max Estimated Tokens', + category: 'Experimental', + requiresRestart: true, + default: 0, + description: + 'Estimated-token ceiling for a single omni media input, ' + + 'checked at the delivery boundary using the versioned ' + + 'raw-resource estimator. 0 disables the token guard — the ' + + 'estimation formula is pending confirmation with the ' + + 'model provider; set a positive threshold to enforce ' + + 'fail-closed rejection.', + showInDialog: false, + jsonSchemaOverride: { + type: 'number', + minimum: 0, + default: 0, + }, + }, + policies: { + type: 'object', + label: 'Omni Transport Guard Policies', + category: 'Experimental', + requiresRestart: true, + default: {} as Record | null>, + description: + 'Guard policies keyed by policy id, run only when the ' + + 'final delivery set exceeds transport limits. Merged with ' + + 'system defaults by id. The merged set must cover image, ' + + 'video, and audio and must not be empty; every policy ' + + 'output must use source: omit.', + showInDialog: false, + mergeStrategy: MergeStrategy.SHALLOW_MERGE, + }, }, }, + policyTools: { + type: 'object', + label: 'Omni Policy Tools', + category: 'Experimental', + requiresRestart: true, + default: {} as Record | null>, + description: + 'Per-tool overrides keyed by policy tool name: settings ' + + '(default arguments), runtime (timeoutMs), ' + + 'and modelAccess (enabled, defaultArguments, lockedArguments, ' + + 'parameterSchema, output).', + showInDialog: false, + mergeStrategy: MergeStrategy.SHALLOW_MERGE, + }, }, }, - transport: { + delivery: { type: 'object', - label: 'Omni Transport Guard', + label: 'Omni Delivery', category: 'Experimental', requiresRestart: true, default: {}, - description: - 'Transport guard dimensions beyond the byte ceiling for omni ' + - 'media delivery.', + description: 'Model-delivery settings for omni media.', showInDialog: false, properties: { - maxEstimatedTokens: { - type: 'number', - label: 'Max Estimated Tokens', + upload: { + type: 'object', + label: 'Omni Delivery Upload', category: 'Experimental', requiresRestart: true, - default: 0, - description: - 'Estimated-token ceiling for a single omni media input, ' + - 'checked before upload using the versioned raw-resource ' + - 'estimator. 0 disables the token guard — the estimation ' + - 'formula is pending confirmation with the model provider; ' + - 'set a positive threshold to enforce fail-closed rejection.', + default: {}, + description: 'Upload-channel delivery settings.', showInDialog: false, - jsonSchemaOverride: { - type: 'number', - minimum: 0, - default: 0, + properties: { + urlTtlHours: { + type: 'number', + label: 'Upload URL TTL (hours)', + category: 'Experimental', + requiresRestart: true, + default: 47, + description: + 'Validity horizon for cached oss:// upload URLs. ' + + 'DashScope temporary uploads live 48h; the default keeps ' + + 'a 1h margin. 0 disables the upload cache (every ' + + 'delivery re-uploads).', + showInDialog: false, + jsonSchemaOverride: { + type: 'number', + minimum: 0, + default: 47, + }, + }, }, }, }, }, - download: { + ingestion: { type: 'object', - label: 'Omni Download', + label: 'Omni Ingestion', category: 'Experimental', requiresRestart: true, default: {}, - description: 'URL media localization limits for omni delivery.', + description: 'Media input ingestion settings for omni delivery.', showInDialog: false, properties: { - maxFileBytes: { - type: 'number', - label: 'Max Download File Bytes', + localization: { + type: 'object', + label: 'Omni Ingestion Localization', category: 'Experimental', requiresRestart: true, - default: 0, + default: {}, + description: 'Remote-media localization settings.', + showInDialog: false, + properties: { + url: { + type: 'object', + label: 'Omni URL Localization', + category: 'Experimental', + requiresRestart: true, + default: {}, + description: 'URL media download settings.', + showInDialog: false, + properties: { + maxFileBytes: { + type: 'number', + label: 'Max Download File Bytes', + category: 'Experimental', + requiresRestart: true, + default: 0, + description: + 'Byte ceiling for downloading URL media inputs. 0 or ' + + 'unset follows ' + + 'omni.processing.transportGuard.maxUploadFileBytes ' + + '(downloading more than the upload channel can ' + + 'deliver is pointless).', + showInDialog: false, + jsonSchemaOverride: { + type: 'number', + minimum: 0, + default: 0, + }, + }, + }, + }, + }, + }, + }, + }, + storage: { + type: 'object', + label: 'Omni Storage', + category: 'Experimental', + requiresRestart: true, + default: {}, + description: 'Managed storage settings under .qwen/omni/.', + showInDialog: false, + properties: { + quarantine: { + type: 'object', + label: 'Omni Quarantine', + category: 'Experimental', + requiresRestart: true, + default: {}, description: - 'Byte ceiling for downloading URL media inputs. 0 or unset ' + - 'follows omni.upload.maxFileBytes (downloading more than the ' + - 'upload channel can deliver is pointless).', + 'Retention for failed policy invocations moved to ' + + '.qwen/omni/quarantine/ for diagnosis. Quarantined content ' + + 'is never recalled into recognition or delivery.', showInDialog: false, - jsonSchemaOverride: { - type: 'number', - minimum: 0, - default: 0, + properties: { + retentionDays: { + type: 'number', + label: 'Quarantine Retention (days)', + category: 'Experimental', + requiresRestart: true, + default: 7, + description: + 'Days a quarantined invocation directory is kept before ' + + 'startup recovery removes it. Must be at least 1; ' + + 'non-positive values fall back to the default.', + showInDialog: false, + jsonSchemaOverride: { type: 'number', minimum: 1, default: 7 }, + }, + maxBytes: { + type: 'number', + label: 'Quarantine Max Bytes', + category: 'Experimental', + requiresRestart: true, + default: 5368709120, + description: + 'Total byte budget for the quarantine directory. Startup ' + + 'recovery removes oldest entries first until within ' + + 'budget. Defaults to 5 GiB. Must be at least 1; ' + + 'non-positive values fall back to the default.', + showInDialog: false, + jsonSchemaOverride: { + type: 'number', + minimum: 1, + default: 5368709120, + }, + }, }, }, }, diff --git a/packages/cli/src/i18n/locales/en.js b/packages/cli/src/i18n/locales/en.js index 4ee231dbcb9..b58e4f325fc 100644 --- a/packages/cli/src/i18n/locales/en.js +++ b/packages/cli/src/i18n/locales/en.js @@ -223,6 +223,16 @@ export default { 'toolDisplayName.Workflow': 'toolDisplayName.Workflow', 'toolDisplayName.ReadMcpResource': 'toolDisplayName.ReadMcpResource', 'toolDisplayName.ImageGen': 'toolDisplayName.ImageGen', + 'toolDisplayName.DownsampleImage': 'toolDisplayName.DownsampleImage', + 'toolDisplayName.DownscaleVideo': 'toolDisplayName.DownscaleVideo', + 'toolDisplayName.DownsampleAudio': 'toolDisplayName.DownsampleAudio', + 'toolDisplayName.ExtractKeyframes': 'toolDisplayName.ExtractKeyframes', + 'toolDisplayName.ExtractAudio': 'toolDisplayName.ExtractAudio', + 'toolDisplayName.ClipVideo': 'toolDisplayName.ClipVideo', + 'toolDisplayName.ConvertImage': 'toolDisplayName.ConvertImage', + 'toolDisplayName.TranscribeAudio': 'toolDisplayName.TranscribeAudio', + '[fixed-only: runs via media policies, not the model]': + '[fixed-only: runs via media policies, not the model]', // ============================================================================ // Help / UI Components // ============================================================================ diff --git a/packages/cli/src/i18n/locales/zh-TW.js b/packages/cli/src/i18n/locales/zh-TW.js index 08c5e41ad68..87d9a0cedb3 100644 --- a/packages/cli/src/i18n/locales/zh-TW.js +++ b/packages/cli/src/i18n/locales/zh-TW.js @@ -214,6 +214,16 @@ export default { 'toolDisplayName.Workflow': '工作流程', 'toolDisplayName.ReadMcpResource': '讀取 MCP 資源', 'toolDisplayName.ImageGen': '圖像生成', + 'toolDisplayName.DownsampleImage': '降採樣圖像', + 'toolDisplayName.DownscaleVideo': '降採樣影片', + 'toolDisplayName.DownsampleAudio': '降採樣音訊', + 'toolDisplayName.ExtractKeyframes': '擷取關鍵影格', + 'toolDisplayName.ExtractAudio': '擷取音軌', + 'toolDisplayName.ClipVideo': '剪輯影片', + 'toolDisplayName.ConvertImage': '轉換圖像', + 'toolDisplayName.TranscribeAudio': '轉寫音訊', + '[fixed-only: runs via media policies, not the model]': + '［僅固定策略：由媒體策略調用，不開放給模型］', '↑ to manage attachments': '↑ 管理附件', '← → select, Delete to remove, ↓ to exit': '← → 選擇，Delete 刪除，↓ 退出', diff --git a/packages/cli/src/i18n/locales/zh.js b/packages/cli/src/i18n/locales/zh.js index 95d293cece6..f306ad7e38f 100644 --- a/packages/cli/src/i18n/locales/zh.js +++ b/packages/cli/src/i18n/locales/zh.js @@ -215,6 +215,16 @@ export default { 'toolDisplayName.Workflow': '工作流', 'toolDisplayName.ReadMcpResource': '读取 MCP 资源', 'toolDisplayName.ImageGen': '图像生成', + 'toolDisplayName.DownsampleImage': '降采样图像', + 'toolDisplayName.DownscaleVideo': '降采样视频', + 'toolDisplayName.DownsampleAudio': '降采样音频', + 'toolDisplayName.ExtractKeyframes': '提取关键帧', + 'toolDisplayName.ExtractAudio': '提取音轨', + 'toolDisplayName.ClipVideo': '剪辑视频', + 'toolDisplayName.ConvertImage': '转换图像', + 'toolDisplayName.TranscribeAudio': '转写音频', + '[fixed-only: runs via media policies, not the model]': + '［仅固定策略：由媒体策略调用，不开放给模型］', // ============================================================================ // Help / UI Components // ============================================================================ diff --git a/packages/cli/src/ui/commands/contextCommand.test.ts b/packages/cli/src/ui/commands/contextCommand.test.ts index 45c0714cd8f..4e20dd541a1 100644 --- a/packages/cli/src/ui/commands/contextCommand.test.ts +++ b/packages/cli/src/ui/commands/contextCommand.test.ts @@ -236,6 +236,61 @@ describe('collectContextData (contextCommand)', () => { expect(data.builtinTools[0].name).toBe('web_fetch'); }); + it('excludes fixed-only media-policy tools from the per-tool breakdown (D6)', async () => { + // A media-policy tool without modelAccess.enabled is stripped from + // getFunctionDeclarations() (zero prompt tokens), so listing it in the + // breakdown would make the per-tool sum exceed allToolsTokens. One with + // modelAccess.enabled IS declared to the model and must stay listed. + const descriptor = { + kind: 'media_policy', + inputMediaTypes: ['image'], + outputs: [], + }; + const hiddenPolicyTool = { + name: 'omni_downsample_image', + schema: { name: 'omni_downsample_image', description: 'policy schema' }, + mediaPolicyDescriptor: descriptor, + }; + const exposedPolicyTool = { + name: 'omni_probe_media', + schema: { name: 'omni_probe_media', description: 'probe schema' }, + mediaPolicyDescriptor: descriptor, + }; + const config = { + getModel: vi.fn().mockReturnValue('test-model'), + getContentGeneratorConfig: vi.fn().mockReturnValue({ + contextWindowSize: 32_000, + }), + getToolRegistry: vi.fn().mockReturnValue({ + getAllTools: vi + .fn() + .mockReturnValue([hiddenPolicyTool, exposedPolicyTool]), + getFunctionDeclarations: vi + .fn() + .mockReturnValue([exposedPolicyTool.schema]), + isDeferredAndHidden: vi.fn().mockReturnValue(false), + }), + getOmniPolicyToolsSettings: vi.fn().mockReturnValue({ + omni_probe_media: { modelAccess: { enabled: true } }, + }), + getVisibleTools: vi.fn().mockReturnValue(new Set()), + getUserMemory: vi.fn().mockReturnValue(''), + getAutoMemoryPrompt: vi.fn().mockReturnValue(''), + getSkillManager: vi.fn().mockReturnValue({ + listSkills: vi.fn().mockResolvedValue([]), + }), + getChatCompression: vi.fn().mockReturnValue(undefined), + getAutoCompactThreshold: vi.fn(), + getExperimentalZedIntegration: vi.fn().mockReturnValue(false), + isInteractive: vi.fn().mockReturnValue(true), + } as unknown as Config; + + const data = await collectContextData(config, true); + + expect(data.builtinTools).toHaveLength(1); + expect(data.builtinTools[0].name).toBe('omni_probe_media'); + }); + it('lists the auto-memory section as a separate memory entry (#7651)', async () => { // The managed auto-memory section is no longer part of getUserMemory(); its // tokens are surfaced via getAutoMemoryPrompt(). Exercise the non-empty diff --git a/packages/cli/src/ui/commands/contextCommand.ts b/packages/cli/src/ui/commands/contextCommand.ts index 2f24d17ff87..664cd50b0e0 100644 --- a/packages/cli/src/ui/commands/contextCommand.ts +++ b/packages/cli/src/ui/commands/contextCommand.ts @@ -27,6 +27,7 @@ import { ToolNames, buildSkillLlmContent, computeThresholds, + isMediaPolicyToolHiddenFromModel, type CompactionThresholds, } from '@qwen-code/qwen-code-core'; import { t } from '../../i18n/index.js'; @@ -152,6 +153,13 @@ export async function collectContextData( if (toolRegistry?.isDeferredAndHidden(tool.name)) { continue; } + // Same alignment rule for omni media-policy tools: fixed-only tools + // (declared descriptor, modelAccess not enabled) are stripped from + // getFunctionDeclarations() and cost the model zero prompt tokens, so + // listing them here would make the breakdown sum exceed allToolsTokens. + if (isMediaPolicyToolHiddenFromModel(config, tool)) { + continue; + } const toolJsonStr = JSON.stringify(tool.schema); const tokens = estimateTokens(toolJsonStr); if (tool instanceof DiscoveredMCPTool) { diff --git a/packages/cli/src/ui/commands/toolsCommand.test.ts b/packages/cli/src/ui/commands/toolsCommand.test.ts index 9e1eae83645..25643b3c985 100644 --- a/packages/cli/src/ui/commands/toolsCommand.test.ts +++ b/packages/cli/src/ui/commands/toolsCommand.test.ts @@ -111,4 +111,59 @@ describe('toolsCommand', () => { ); expect(message.tools[1].description).toBe('Edits code files.'); }); + + it('flags hidden media-policy tools as fixedOnly, but not model-enabled ones', async () => { + const mediaTools = [ + { + name: 'omni_downsample_image', + displayName: 'DownsampleImage', + description: 'Downsamples an image.', + schema: {}, + // Media-policy tool with no modelAccess entry → hidden from the + // model's declarations → must surface as fixed-only in /tools. + mediaPolicyDescriptor: { + kind: 'media_policy', + inputMediaTypes: ['image'], + outputs: [], + }, + }, + { + name: 'omni_probe_media', + displayName: 'ProbeMedia', + description: 'Probes media metadata.', + schema: {}, + // Same descriptor, but modelAccess.enabled below re-exposes it to + // the model, so it must NOT carry the fixed-only marker. + mediaPolicyDescriptor: { + kind: 'media_policy', + inputMediaTypes: ['image'], + outputs: [], + }, + }, + ...mockTools, + ] as Tool[]; + const mockContext = createMockCommandContext({ + services: { + config: { + getToolRegistry: () => ({ getAllTools: () => mediaTools }), + getOmniPolicyToolsSettings: () => ({ + omni_probe_media: { modelAccess: { enabled: true } }, + }), + }, + }, + }); + + if (!toolsCommand.action) throw new Error('Action not defined'); + await toolsCommand.action(mockContext, ''); + + const [message] = (mockContext.ui.addItem as vi.Mock).mock.calls[0]; + expect(message.tools).toHaveLength(4); + expect(message.tools[0]).toMatchObject({ + name: 'omni_downsample_image', + fixedOnly: true, + }); + expect(message.tools[1].fixedOnly).toBeUndefined(); + expect(message.tools[2].fixedOnly).toBeUndefined(); + expect(message.tools[3].fixedOnly).toBeUndefined(); + }); }); diff --git a/packages/cli/src/ui/commands/toolsCommand.ts b/packages/cli/src/ui/commands/toolsCommand.ts index 5c6625e6d35..d9698f3b3d9 100644 --- a/packages/cli/src/ui/commands/toolsCommand.ts +++ b/packages/cli/src/ui/commands/toolsCommand.ts @@ -10,6 +10,7 @@ import { CommandKind, } from './types.js'; import { MessageType, type HistoryItemToolsList } from '../types.js'; +import { isMediaPolicyToolHiddenFromModel } from '@qwen-code/qwen-code-core'; import { t } from '../../i18n/index.js'; export const toolsCommand: SlashCommand = { @@ -38,6 +39,7 @@ export const toolsCommand: SlashCommand = { ); return; } + const config = context.services.config!; const tools = toolRegistry.getAllTools(); // Filter out MCP tools by checking for the absence of a serverName property @@ -49,6 +51,13 @@ export const toolsCommand: SlashCommand = { name: tool.name, displayName: tool.displayName, description: tool.description, + // Omni media-policy tools without modelAccess.enabled are stripped + // from the model's declarations but stay listed here for the human; + // the flag renders a "fixed-only" marker so the discrepancy between + // /tools and what the model can call is visible, not confusing. + ...(isMediaPolicyToolHiddenFromModel(config, tool) + ? { fixedOnly: true } + : {}), })), showDescriptions: useShowDescriptions, }; diff --git a/packages/cli/src/ui/components/views/ToolsList.test.tsx b/packages/cli/src/ui/components/views/ToolsList.test.tsx index ae6acd12016..839331942e0 100644 --- a/packages/cli/src/ui/components/views/ToolsList.test.tsx +++ b/packages/cli/src/ui/components/views/ToolsList.test.tsx @@ -55,4 +55,28 @@ describe('', () => { ); expect(lastFrame()).toMatchSnapshot(); }); + + it('marks fixed-only media-policy tools and leaves others unmarked', () => { + const tools: ToolDefinition[] = [ + { + name: 'omni_downsample_image', + displayName: 'DownsampleImage', + fixedOnly: true, + }, + { name: 'read_file', displayName: 'ReadFile' }, + ]; + const { lastFrame } = render( + , + ); + const frame = lastFrame() ?? ''; + // The marker must be attached to the fixed-only tool's line only. + const downsampleLine = frame + .split('\n') + .find((line) => line.includes('DownsampleImage')); + expect(downsampleLine).toContain('[fixed-only'); + const readFileLine = frame + .split('\n') + .find((line) => line.includes('ReadFile')); + expect(readFileLine).not.toContain('fixed-only'); + }); }); diff --git a/packages/cli/src/ui/components/views/ToolsList.tsx b/packages/cli/src/ui/components/views/ToolsList.tsx index d397c1002f7..de9efd4e9d3 100644 --- a/packages/cli/src/ui/components/views/ToolsList.tsx +++ b/packages/cli/src/ui/components/views/ToolsList.tsx @@ -35,6 +35,12 @@ export const ToolsList: React.FC = ({ {tool.displayName} {showDescriptions ? ` (${tool.name})` : ''} + {tool.fixedOnly && ( + + {' '} + {t('[fixed-only: runs via media policies, not the model]')} + + )} {showDescriptions && tool.description && ( { vi.mock('@qwen-code/qwen-code-core/omni', () => ({ ...omniMocks, + // Deterministic stand-ins for the shared formatters: the tests assert + // the WIRING (which formatter, which arguments, part ordering), while + // the formatters' own wording is covered by their core unit tests. + formatOmissionText: (name: string, reason: string) => + `[omission ${name}: ${reason}]`, + formatDisclosureText: (name: string, disclosure: string) => + `[disclosure ${name}: ${disclosure}]`, + // Deterministic stand-in for the shared multi-output materializer: + // one marker part per extra. The real builder's output shape is + // covered by its core unit tests; these tests pin the wiring (that + // the funnel calls it and splices its parts after the primary slot). + buildAdditionalMediaParts: (name: string, extras?: unknown[]) => + (extras ?? []).map((_, i) => ({ text: `[extra ${name} ${i}]` })), + // Same wiring-only stand-in for the shared transcript materializer. + buildTranscriptParts: (name: string, transcripts?: unknown[]) => + (transcripts ?? []).map((_, i) => ({ + text: `[transcript ${name} ${i}]`, + })), OmniObjectStore: class { getOmniRootDir() { return path.join(os.tmpdir(), 'omni-at-test'); @@ -1916,6 +1934,115 @@ describe('handleAtCommand', () => { expect(parts.filter((p) => 'fileData' in p)).toHaveLength(1); }); + it('replaces the media with an omission notice when the transport guard withholds it', async () => { + // Policy design §10.2: an omission is a successful delivery whose + // content IS the notice — no fileData part, no error card. + omniMocks.processMediaForOmniDelivery.mockResolvedValue({ + omission: { reason: 'video exceeds the 500MB transport limit' }, + }); + const result = await handleAtCommand({ + query: 'summarize @https://example.com/clip.mp4 please', + config: omniConfig(true), + onDebugMessage: mockOnDebugMessage, + messageId: 706, + signal: abortController.signal, + }); + + expect(result.shouldProceed).toBe(true); + const parts = result.processedQuery as Array>; + expect(parts).toContainEqual({ + text: '[omission clip.mp4: video exceeds the 500MB transport limit]', + }); + expect(parts.some((p) => 'fileData' in p)).toBe(false); + expect(result.toolDisplays![0]).toMatchObject({ + name: 'Fetch Media URL', + status: ToolCallStatus.Success, + resultDisplay: 'Media omitted by the omni transport guard: clip.mp4', + }); + }); + + it('places the degradation disclosure text immediately before the fileData part (D8)', async () => { + omniMocks.processMediaForOmniDelivery.mockResolvedValue({ + fileUri: 'oss://bucket/clip.mp4', + mimeType: 'video/mp4', + recognized: { modality: 'video', sizeBytes: 2 * 1024 * 1024 }, + degraded: true, + disclosure: '原 1080p → 480p，细节受损', + }); + const result = await handleAtCommand({ + query: 'summarize @https://example.com/clip.mp4 please', + config: omniConfig(true), + onDebugMessage: mockOnDebugMessage, + messageId: 707, + signal: abortController.signal, + }); + + expect(result.shouldProceed).toBe(true); + const parts = result.processedQuery as Array>; + const disclosureIdx = parts.findIndex( + (p) => p['text'] === '[disclosure clip.mp4: 原 1080p → 480p，细节受损]', + ); + const fileDataIdx = parts.findIndex((p) => 'fileData' in p); + expect(disclosureIdx).toBeGreaterThan(-1); + expect(fileDataIdx).toBe(disclosureIdx + 1); + expect( + (result.toolDisplays![0] as { resultDisplay: string }).resultDisplay, + ).toContain('(degraded by media policy)'); + }); + + it('splices additionalMedia parts after the primary fileData part (multi-output policies)', async () => { + omniMocks.processMediaForOmniDelivery.mockResolvedValue({ + fileUri: 'oss://bucket/clip.mp4', + mimeType: 'video/mp4', + recognized: { modality: 'video', sizeBytes: 2 * 1024 * 1024 }, + additionalMedia: [ + { fileUri: 'oss://bucket/frame2', mimeType: 'image/jpeg' }, + { fileUri: 'oss://bucket/frame3', mimeType: 'image/jpeg' }, + ], + }); + const result = await handleAtCommand({ + query: 'summarize @https://example.com/clip.mp4 please', + config: omniConfig(true), + onDebugMessage: mockOnDebugMessage, + messageId: 708, + signal: abortController.signal, + }); + + expect(result.shouldProceed).toBe(true); + const parts = result.processedQuery as Array>; + const fileDataIdx = parts.findIndex((p) => 'fileData' in p); + expect(fileDataIdx).toBeGreaterThan(-1); + // The materialized extras follow the primary media part directly. + expect(parts[fileDataIdx + 1]).toEqual({ text: '[extra clip.mp4 0]' }); + expect(parts[fileDataIdx + 2]).toEqual({ text: '[extra clip.mp4 1]' }); + }); + + it('splices additionalMedia parts after the omission notice when the primary is withheld', async () => { + omniMocks.processMediaForOmniDelivery.mockResolvedValue({ + omission: { reason: 'video exceeds the transport limit' }, + additionalMedia: [ + { fileUri: 'oss://bucket/frame2', mimeType: 'image/jpeg' }, + ], + }); + const result = await handleAtCommand({ + query: 'summarize @https://example.com/clip.mp4 please', + config: omniConfig(true), + onDebugMessage: mockOnDebugMessage, + messageId: 709, + signal: abortController.signal, + }); + + expect(result.shouldProceed).toBe(true); + const parts = result.processedQuery as Array>; + const omissionIdx = parts.findIndex( + (p) => + p['text'] === + '[omission clip.mp4: video exceeds the transport limit]', + ); + expect(omissionIdx).toBeGreaterThan(-1); + expect(parts[omissionIdx + 1]).toEqual({ text: '[extra clip.mp4 0]' }); + }); + it('ends the turn quietly (shouldProceed=false) on a user abort mid-download', async () => { omniMocks.downloadMediaUrl.mockImplementation(async () => { abortController.abort(); diff --git a/packages/cli/src/ui/hooks/atCommandProcessor.ts b/packages/cli/src/ui/hooks/atCommandProcessor.ts index 8aca3981fc9..df13e4ee942 100644 --- a/packages/cli/src/ui/hooks/atCommandProcessor.ts +++ b/packages/cli/src/ui/hooks/atCommandProcessor.ts @@ -623,6 +623,69 @@ export async function resolveAtCommandQuery({ // the opaque staging path the download landed under. { signal, displayName: urlBase }, ); + // §6.2/D8 ordering contract documented on buildTranscriptParts. + const transcriptParts = core.buildTranscriptParts( + urlBase, + delivery.transcripts, + ); + // Additional media Parts (multi-output fixed policies): follow the + // primary media slot in every branch below. + const additionalParts = core.buildAdditionalMediaParts( + urlBase, + delivery.additionalMedia, + ); + if (delivery.omission) { + // Explicit omission (policy design §10.2): the media is withheld + // and the omission notice text stands in its place — mirroring + // readMediaViaOmniDelivery. Not an error: the fetch succeeded; + // the transport guard's verdict is the content. + urlMediaParts.push({ + text: core.formatOmissionText(urlBase, delivery.omission.reason), + }); + urlMediaParts.push(...additionalParts); + urlMediaParts.push(...transcriptParts); + urlMediaLabels.push(ref.url); + urlMediaDisplays.push({ + callId, + name: 'Fetch Media URL', + description: `Downloaded ${ref.url}`, + status: ToolCallStatus.Success, + resultDisplay: `Media omitted by the omni transport guard: ${urlBase}`, + confirmationDetails: undefined, + }); + continue; + } + if (!delivery.fileUri && transcriptParts.length > 0) { + // Pure-transcript delivery (§6.2): the policies replaced the + // media with text-only deliverables — no media Part is emitted + // for the primary (additional deliverables, if any, still are). + // The primary disclosure (chained prior lossy steps, decision + // D8) still renders: the transcript was derived through them. + if (delivery.disclosure) { + urlMediaParts.push({ + text: core.formatDisclosureText(urlBase, delivery.disclosure), + }); + } + urlMediaParts.push(...additionalParts); + urlMediaParts.push(...transcriptParts); + urlMediaLabels.push(ref.url); + urlMediaDisplays.push({ + callId, + name: 'Fetch Media URL', + description: `Downloaded ${ref.url}`, + status: ToolCallStatus.Success, + resultDisplay: `Localized ${urlBase} and delivered as transcript (omni policy).`, + confirmationDetails: undefined, + }); + continue; + } + // Disclosure IMMEDIATELY before its media part (decision D8): + // provider converters that relocate media move the pair together. + if (delivery.disclosure) { + urlMediaParts.push({ + text: core.formatDisclosureText(urlBase, delivery.disclosure), + }); + } urlMediaParts.push({ fileData: { fileUri: delivery.fileUri, @@ -630,13 +693,15 @@ export async function resolveAtCommandQuery({ displayName: urlBase, }, }); + urlMediaParts.push(...additionalParts); + urlMediaParts.push(...transcriptParts); urlMediaLabels.push(ref.url); urlMediaDisplays.push({ callId, name: 'Fetch Media URL', description: `Downloaded ${ref.url}`, status: ToolCallStatus.Success, - resultDisplay: `Localized ${urlBase} (${delivery.recognized.modality}, ${(delivery.recognized.sizeBytes / 1024 / 1024).toFixed(1)}MB) and delivered via omni upload.`, + resultDisplay: `Localized ${urlBase} (${delivery.recognized.modality}, ${(delivery.recognized.sizeBytes / 1024 / 1024).toFixed(1)}MB) and delivered via omni upload${delivery.degraded ? ' (degraded by media policy)' : ''}.`, confirmationDetails: undefined, }); } catch (error) { diff --git a/packages/cli/src/ui/hooks/useGeminiStream.test.tsx b/packages/cli/src/ui/hooks/useGeminiStream.test.tsx index aa89113af44..010b57156d7 100644 --- a/packages/cli/src/ui/hooks/useGeminiStream.test.tsx +++ b/packages/cli/src/ui/hooks/useGeminiStream.test.tsx @@ -6765,6 +6765,9 @@ describe('useGeminiStream', () => { name: 'save_memory', args: { fact: 'test fact' }, isClientInitiated: true, + // Slash-command scheduling is the in-process `client` + // channel — the media-policy modelAccess gate keys on this. + executionOrigin: { kind: 'client' }, }), ], expect.any(AbortSignal), diff --git a/packages/cli/src/ui/hooks/useGeminiStream.ts b/packages/cli/src/ui/hooks/useGeminiStream.ts index 72edf85c38d..cc6d8f71c48 100644 --- a/packages/cli/src/ui/hooks/useGeminiStream.ts +++ b/packages/cli/src/ui/hooks/useGeminiStream.ts @@ -1078,6 +1078,11 @@ export const useGeminiStream = ( args: toolArgs, isClientInitiated: true, prompt_id, + // Client-direct provenance (omni policy design): slash + // commands scheduling a tool are the in-process `client` + // channel — subject to the same media-policy modelAccess + // gate as model calls, never to fixed-policy semantics. + executionOrigin: { kind: 'client' }, }; scheduleToolCalls([toolCallRequest], abortSignal); return { queryToSend: null, shouldProceed: false }; diff --git a/packages/cli/src/ui/types.ts b/packages/cli/src/ui/types.ts index 5bac5e8e259..e35bfaa459a 100644 --- a/packages/cli/src/ui/types.ts +++ b/packages/cli/src/ui/types.ts @@ -339,6 +339,9 @@ export interface ToolDefinition { name: string; displayName: string; description?: string; + /** Omni media-policy tool that only runs via fixed policies: it is hidden + * from the model's declarations, so /tools annotates it for the human. */ + fixedOnly?: boolean; } export interface SkillDefinition { diff --git a/packages/core/src/config/config.test.ts b/packages/core/src/config/config.test.ts index 6a35d1a86ce..a3591ad12e1 100644 --- a/packages/core/src/config/config.test.ts +++ b/packages/core/src/config/config.test.ts @@ -592,6 +592,37 @@ describe('Server Config (config.ts)', () => { }); }); + describe('omni quarantine budget getters', () => { + it('passes through positive settings', () => { + const config = new Config({ + ...baseParams, + omniQuarantineRetentionDays: 3, + omniQuarantineMaxBytes: 1024, + }); + expect(config.getOmniQuarantineRetentionDays()).toBe(3); + expect(config.getOmniQuarantineMaxBytes()).toBe(1024); + }); + + it.each([ + ['unset', undefined], + ['zero', 0], + ['negative', -1], + ['NaN', Number.NaN], + ['Infinity', Number.POSITIVE_INFINITY], + ])( + 'falls back to defaults on a %s setting (a bad value must not expire the whole quarantine)', + (_label, bad) => { + const config = new Config({ + ...baseParams, + omniQuarantineRetentionDays: bad, + omniQuarantineMaxBytes: bad, + }); + expect(config.getOmniQuarantineRetentionDays()).toBe(7); + expect(config.getOmniQuarantineMaxBytes()).toBe(5 * 1024 * 1024 * 1024); + }, + ); + }); + describe('getMemoryAgentTimeoutMinutes', () => { it('returns undefined when unset', () => { expect( diff --git a/packages/core/src/config/config.ts b/packages/core/src/config/config.ts index 0d36e74a7f3..9fa678a17c9 100644 --- a/packages/core/src/config/config.ts +++ b/packages/core/src/config/config.ts @@ -28,6 +28,10 @@ import { selectVisionBridgeModel, } from '../services/visionBridge/vision-bridge-service.js'; import type { AnyToolInvocation } from '../tools/tools.js'; +import type { + NormalizedOmniProcessingConfig, + OmniPolicyToolsSettings, +} from '../omni/policy/types.js'; import type { ArenaManager } from '../agents/arena/ArenaManager.js'; import { ArenaAgentClient } from '../agents/arena/ArenaAgentClient.js'; import type { TeamManager } from '../agents/team/TeamManager.js'; @@ -1103,13 +1107,27 @@ export interface ConfigParameters { * pipeline (omni-experiment branch). */ omniEnabled?: boolean; /** Per-file byte ceiling for omni media uploads (default 1 GiB). */ - omniUploadMaxFileBytes?: number; + omniMaxUploadFileBytes?: number; /** Estimated-token ceiling for omni media (0/unset = guard disabled). */ omniMaxEstimatedTokens?: number; /** Byte ceiling for omni URL downloads (unset = follow upload cap). */ - omniDownloadMaxFileBytes?: number; - /** Upload cache TTL in hours (0 disables the cache; default 47). */ - omniUploadCacheTtlHours?: number; + omniUrlDownloadMaxFileBytes?: number; + /** Upload URL TTL in hours (0 disables the cache; default 47). */ + omniUploadUrlTtlHours?: number; + /** Raw `omni.processing.policyTools` map (per-tool settings/runtime/ + * modelAccess). Normalized lazily by the omni policy modules. */ + omniPolicyTools?: OmniPolicyToolsSettings; + /** Raw `omni.processing.fixedPolicies` map (id → policy | null + * tombstone). Normalized at startup against system defaults. */ + omniFixedPolicies?: Record; + /** Raw `omni.processing.transportGuard.policies` map. */ + omniTransportGuardPolicies?: Record; + /** Raw `omni.processing.limits` per-root derivation budgets. */ + omniProcessingLimits?: Record; + /** `omni.storage.quarantine.retentionDays` (default 7). */ + omniQuarantineRetentionDays?: number; + /** `omni.storage.quarantine.maxBytes` (default 5 GiB). */ + omniQuarantineMaxBytes?: number; /** Image generation model selected through `/model --image`. */ imageModel?: string; /** @@ -1936,10 +1954,19 @@ export class Config { private readonly artifactHost?: ArtifactHostConfig; private readonly artifactOss?: ArtifactOssConfig; private readonly omniEnabled: boolean = false; - private readonly omniUploadMaxFileBytes?: number; + private readonly omniMaxUploadFileBytes?: number; private readonly omniMaxEstimatedTokens?: number; - private readonly omniDownloadMaxFileBytes?: number; - private readonly omniUploadCacheTtlHours?: number; + private readonly omniUrlDownloadMaxFileBytes?: number; + private readonly omniUploadUrlTtlHours?: number; + private readonly omniPolicyTools?: OmniPolicyToolsSettings; + private readonly omniFixedPolicies?: Record; + private readonly omniTransportGuardPolicies?: Record; + private readonly omniProcessingLimits?: Record; + private readonly omniQuarantineRetentionDays?: number; + private readonly omniQuarantineMaxBytes?: number; + /** Normalized `omni.processing` view; set once during initialize() + * (after the tool registry exists) when omni is enabled. */ + private omniProcessingConfig?: NormalizedOmniProcessingConfig; private workflowsEnabled = false; private readonly skipWorkflowUsageWarning: boolean = false; private readonly computerUseEnabled: boolean = true; @@ -2216,10 +2243,16 @@ export class Config { this.artifactHost = params.artifactHost; this.artifactOss = params.artifactOss; this.omniEnabled = params.omniEnabled ?? false; - this.omniUploadMaxFileBytes = params.omniUploadMaxFileBytes; + this.omniMaxUploadFileBytes = params.omniMaxUploadFileBytes; this.omniMaxEstimatedTokens = params.omniMaxEstimatedTokens; - this.omniDownloadMaxFileBytes = params.omniDownloadMaxFileBytes; - this.omniUploadCacheTtlHours = params.omniUploadCacheTtlHours; + this.omniUrlDownloadMaxFileBytes = params.omniUrlDownloadMaxFileBytes; + this.omniUploadUrlTtlHours = params.omniUploadUrlTtlHours; + this.omniPolicyTools = params.omniPolicyTools; + this.omniFixedPolicies = params.omniFixedPolicies; + this.omniTransportGuardPolicies = params.omniTransportGuardPolicies; + this.omniProcessingLimits = params.omniProcessingLimits; + this.omniQuarantineRetentionDays = params.omniQuarantineRetentionDays; + this.omniQuarantineMaxBytes = params.omniQuarantineMaxBytes; this.workflowsEnabled = params.workflowsEnabled ?? false; this.skipWorkflowUsageWarning = params.skipWorkflowUsageWarning ?? false; this.computerUseEnabled = params.computerUseEnabled ?? true; @@ -2953,6 +2986,28 @@ export class Config { }); recordStartupEvent('config_initialize_tool_warmup_end'); + // Normalize the omni fixed-policy configuration now that the tool + // registry can resolve policy-tool references. A violation throws + // OmniPolicyConfigError and aborts startup — a mis-configured + // transport guard must never degrade into sending over-limit media. + if (this.isOmniEnabled()) { + const { normalizeOmniProcessingConfig } = await import( + '../omni/policy/config.js' + ); + this.omniProcessingConfig = normalizeOmniProcessingConfig( + { + fixedPolicies: this.omniFixedPolicies, + transportGuardPolicies: this.omniTransportGuardPolicies, + limits: this.omniProcessingLimits, + policyTools: this.omniPolicyTools, + maxUploadFileBytes: this.omniMaxUploadFileBytes, + maxEstimatedTokens: this.omniMaxEstimatedTokens, + urlTtlHours: this.omniUploadUrlTtlHours, + }, + this.toolRegistry, + ); + } + // Fire-and-forget MCP discovery. Each server's tools land in the // registry as it becomes ready; the cli's AppContainer debounces // `setTools()` (~16ms / one frame) so the model sees the new tools @@ -6375,25 +6430,58 @@ export class Config { } isOmniEnabled(): boolean { + // Bare mode means the minimal toolset and no experimental pipelines: + // gating here (the single choke point) keeps every omni surface off — + // tool registration, content normalization, the ffmpeg runtime + // assertion, and the delivery gate — and deliberately wins over the + // env-var opt-in below. + if (this.bareMode) return false; // Omni is experimental and opt-in: enabled via settings or env var. if (process.env['QWEN_CODE_ENABLE_OMNI'] === '1') return true; return this.omniEnabled; } - getOmniUploadMaxFileBytes(): number | undefined { - return this.omniUploadMaxFileBytes; + getOmniMaxUploadFileBytes(): number | undefined { + return this.omniMaxUploadFileBytes; } getOmniMaxEstimatedTokens(): number | undefined { return this.omniMaxEstimatedTokens; } - getOmniDownloadMaxFileBytes(): number | undefined { - return this.omniDownloadMaxFileBytes; + getOmniUrlDownloadMaxFileBytes(): number | undefined { + return this.omniUrlDownloadMaxFileBytes; + } + + getOmniUploadUrlTtlHours(): number | undefined { + return this.omniUploadUrlTtlHours; } - getOmniUploadCacheTtlHours(): number | undefined { - return this.omniUploadCacheTtlHours; + getOmniPolicyToolsSettings(): OmniPolicyToolsSettings | undefined { + return this.omniPolicyTools; + } + + /** Normalized `omni.processing` view. Undefined until initialize() + * completes (or when omni is disabled). */ + getOmniProcessingConfig(): NormalizedOmniProcessingConfig | undefined { + return this.omniProcessingConfig; + } + + getOmniQuarantineRetentionDays(): number { + // A zero/negative/NaN setting would make the recovery sweep treat the + // whole quarantine as expired (or break its cutoff comparisons) — + // fall back to the default instead of propagating nonsense. + const days = this.omniQuarantineRetentionDays; + return typeof days === 'number' && Number.isFinite(days) && days > 0 + ? days + : 7; + } + + getOmniQuarantineMaxBytes(): number { + const bytes = this.omniQuarantineMaxBytes; + return typeof bytes === 'number' && Number.isFinite(bytes) && bytes > 0 + ? bytes + : 5 * 1024 * 1024 * 1024; } resolveImageGenerationModel( @@ -8042,6 +8130,77 @@ export class Config { await registerComputerUseTools(registerLazy, this); } + // Omni media-policy tools: always registered when omni is enabled (the + // fixed-policy orchestrator must be able to find them), but hidden from + // every model-facing surface unless + // `omni.processing.policyTools..modelAccess.enabled` opens them up + // (see omni/policy/model-access.ts). + if (this.isOmniEnabled()) { + // Table-driven: each entry pairs the registered name with a lazy + // import-and-construct factory (the module loads on first use). + const omniPolicyToolFactories: Array<[ToolName, ToolFactory]> = [ + [ + ToolNames.OMNI_DOWNSAMPLE_IMAGE, + async () => + new ( + await import('../omni/policy/tools/downsample-image.js') + ).OmniDownsampleImageTool(this), + ], + [ + ToolNames.OMNI_DOWNSCALE_VIDEO, + async () => + new ( + await import('../omni/policy/tools/downscale-video.js') + ).OmniDownscaleVideoTool(this), + ], + [ + ToolNames.OMNI_DOWNSAMPLE_AUDIO, + async () => + new ( + await import('../omni/policy/tools/downsample-audio.js') + ).OmniDownsampleAudioTool(this), + ], + [ + ToolNames.OMNI_EXTRACT_KEYFRAMES, + async () => + new ( + await import('../omni/policy/tools/extract-keyframes.js') + ).OmniExtractKeyframesTool(this), + ], + [ + ToolNames.OMNI_EXTRACT_AUDIO, + async () => + new ( + await import('../omni/policy/tools/extract-audio.js') + ).OmniExtractAudioTool(this), + ], + [ + ToolNames.OMNI_CLIP_VIDEO, + async () => + new ( + await import('../omni/policy/tools/clip-video.js') + ).OmniClipVideoTool(this), + ], + [ + ToolNames.OMNI_CONVERT_IMAGE, + async () => + new ( + await import('../omni/policy/tools/convert-image.js') + ).OmniConvertImageTool(this), + ], + [ + ToolNames.OMNI_TRANSCRIBE_AUDIO, + async () => + new ( + await import('../omni/policy/tools/transcribe-audio.js') + ).OmniTranscribeAudioTool(this), + ], + ]; + for (const [name, factory] of omniPolicyToolFactories) { + await registerLazy(name, factory); + } + } + // Register monitor tool await registerLazy(ToolNames.MONITOR, async () => { const { MonitorTool } = await import('../tools/monitor.js'); diff --git a/packages/core/src/core/client.ts b/packages/core/src/core/client.ts index f2af5cd93d4..e612cb77029 100644 --- a/packages/core/src/core/client.ts +++ b/packages/core/src/core/client.ts @@ -66,6 +66,7 @@ import { getPlanModeSystemReminder, resolveInteractionMode, } from './prompts.js'; +import { buildOmniMediaGuidanceSection } from '../omni/media-guidance.js'; import { CompressionStatus, GeminiEventType, @@ -967,6 +968,11 @@ export class GeminiClient { ); const stableLayers = { base, + // Progressive media understanding contract: WHY deliveries carry + // 【媒体降质】/【媒体省略】/【媒体转写】 markers and how to fetch + // fuller evidence. Stable — omni config/provider don't change + // in-session — so it belongs inside the cached static prefix. + mediaGuidance: buildOmniMediaGuidanceSection(this.config), contextFiles: this.config.getUserMemory(), appendPrompt: this.config.getAppendSystemPrompt(), }; diff --git a/packages/core/src/core/coreToolScheduler.mediaPolicy.test.ts b/packages/core/src/core/coreToolScheduler.mediaPolicy.test.ts new file mode 100644 index 00000000000..e39edfc4b7b --- /dev/null +++ b/packages/core/src/core/coreToolScheduler.mediaPolicy.test.ts @@ -0,0 +1,456 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Scheduler wiring tests for the omni media-policy protocol: + * modelAccess gate at schedule time, fixed-policy permission bypass, + * and raw policy-artifact capture on the success response. + */ + +import { describe, it, expect, vi } from 'vitest'; +import type { Mock } from 'vitest'; +import { executeToolCall } from './nonInteractiveToolExecutor.js'; +import type { + Config, + MediaPolicyToolDescriptor, + OmniPolicyToolsSettings, + ToolCallRequestInfo, + ToolRegistry, + ToolResult, +} from '../index.js'; +import { + ApprovalMode, + CoreToolScheduler, + DEFAULT_TRUNCATE_TOOL_OUTPUT_LINES, + DEFAULT_TRUNCATE_TOOL_OUTPUT_THRESHOLD, + ToolErrorType, +} from '../index.js'; +import type { ToolCall } from './coreToolScheduler.js'; +import { MockTool } from '../test-utils/mock-tool.js'; + +const DESCRIPTOR: MediaPolicyToolDescriptor = { + kind: 'media_policy', + inputMediaTypes: ['image'], + outputs: [{ kind: 'media', required: true }], +}; + +/** MockTool that reports itself as a media-policy tool (code-registration + * fact — the descriptor getter, never configuration). */ +class MockMediaPolicyTool extends MockTool { + override get mediaPolicyDescriptor(): MediaPolicyToolDescriptor { + return DESCRIPTOR; + } +} + +const FIXED_ORIGIN = { + kind: 'fixed_policy', + policyId: 'image-compress-v1', + stage: 'preprocessing', +} as const; + +function makeConfig(options: { + tool: MockTool; + omniPolicyTools?: OmniPolicyToolsSettings; + approvalMode?: ApprovalMode; + interactive?: boolean; + isToolEnabled?: (name: string) => Promise; +}): Config { + const mockToolRegistry = { + getTool: (name: string) => + name === options.tool.name ? options.tool : undefined, + ensureTool: async (name: string) => + name === options.tool.name ? options.tool : undefined, + getToolByName: (name: string) => + name === options.tool.name ? options.tool : undefined, + getAllToolNames: () => [options.tool.name], + getFunctionDeclarations: () => [], + getAllTools: () => [options.tool], + } as unknown as ToolRegistry; + + return { + getToolRegistry: () => mockToolRegistry, + getApprovalMode: () => options.approvalMode ?? ApprovalMode.DEFAULT, + getAllowedTools: () => [], + getPermissionsAllow: () => [], + getPermissionsDeny: () => undefined, + getSessionId: () => 'test-session-id', + getUsageStatisticsEnabled: () => true, + getDebugMode: () => false, + getContentGeneratorConfig: () => ({ + model: 'test-model', + authType: 'gemini', + }), + getEffectiveInputModalities: () => ({ image: true }), + getShellExecutionConfig: () => ({ + terminalWidth: 90, + terminalHeight: 30, + }), + storage: { + getProjectTempDir: () => '/tmp', + }, + getTruncateToolOutputThreshold: () => + DEFAULT_TRUNCATE_TOOL_OUTPUT_THRESHOLD, + getTruncateToolOutputLines: () => DEFAULT_TRUNCATE_TOOL_OUTPUT_LINES, + getUseModelRouter: () => false, + getGeminiClient: () => null, + getChatRecordingService: () => undefined, + getMessageBus: vi.fn().mockReturnValue(undefined), + getDisableAllHooks: vi.fn().mockReturnValue(true), + getHookSystem: vi.fn().mockReturnValue(undefined), + isInteractive: vi.fn().mockReturnValue(options.interactive ?? false), + getExperimentalZedIntegration: () => false, + getAutoModeDenialState: () => ({ + consecutiveBlock: 0, + consecutiveUnavailable: 0, + totalBlock: 0, + totalUnavailable: 0, + }), + setAutoModeDenialState: vi.fn(), + getAutoModeSettings: () => ({}), + getOmniPolicyToolsSettings: () => options.omniPolicyTools, + ...(options.isToolEnabled + ? { + getPermissionManager: () => ({ + isToolEnabled: options.isToolEnabled, + findMatchingDenyRule: () => undefined, + }), + } + : {}), + } as unknown as Config; +} + +const request = ( + overrides: Partial & { name: string }, +): ToolCallRequestInfo => ({ + callId: 'call-1', + args: {}, + isClientInitiated: false, + prompt_id: 'prompt-1', + ...overrides, +}); + +describe('CoreToolScheduler media-policy modelAccess gate', () => { + it('rejects a call with a missing executionOrigin (fails closed as model) when modelAccess is absent', async () => { + const executeFn = vi.fn(); + const tool = new MockMediaPolicyTool({ + name: 'omni_compress_image', + execute: executeFn, + }); + const config = makeConfig({ tool }); + + const response = await executeToolCall( + config, + request({ name: tool.name }), + new AbortController().signal, + ); + + expect(response.errorType).toBe(ToolErrorType.EXECUTION_DENIED); + expect(response.error?.message).toContain( + '"omni.processing.policyTools.omni_compress_image.modelAccess.enabled": true', + ); + expect(executeFn).not.toHaveBeenCalled(); + }); + + it('executes an enabled tool with defaults + model args + lockedArguments merged', async () => { + const executeFn: Mock = vi.fn().mockResolvedValue({ + llmContent: 'ok', + returnDisplay: 'ok', + } satisfies ToolResult); + const tool = new MockMediaPolicyTool({ + name: 'omni_compress_image', + execute: executeFn, + }); + const config = makeConfig({ + tool, + omniPolicyTools: { + omni_compress_image: { + modelAccess: { + enabled: true, + defaultArguments: { quality: 80, format: 'jpeg' }, + lockedArguments: { output_dir: '/objects' }, + }, + }, + }, + }); + + const response = await executeToolCall( + config, + request({ name: tool.name, args: { quality: 55, source: 'a.png' } }), + new AbortController().signal, + ); + + expect(response.error).toBeUndefined(); + expect(executeFn).toHaveBeenCalledWith({ + quality: 55, + format: 'jpeg', + source: 'a.png', + output_dir: '/objects', + }); + }); + + it('rejects explicit lockedArguments keys as INVALID_TOOL_PARAMS', async () => { + const executeFn = vi.fn(); + const tool = new MockMediaPolicyTool({ + name: 'omni_compress_image', + execute: executeFn, + }); + const config = makeConfig({ + tool, + omniPolicyTools: { + omni_compress_image: { + modelAccess: { + enabled: true, + lockedArguments: { output_dir: '/objects' }, + }, + }, + }, + }); + + const response = await executeToolCall( + config, + request({ name: tool.name, args: { output_dir: '/evil' } }), + new AbortController().signal, + ); + + expect(response.errorType).toBe(ToolErrorType.INVALID_TOOL_PARAMS); + expect(response.error?.message).toContain('"output_dir"'); + expect(executeFn).not.toHaveBeenCalled(); + }); + + it('rejects a forged fixed_policy origin on a non-media-policy tool', async () => { + const executeFn = vi.fn(); + const tool = new MockTool({ + name: 'run_shell_command', + execute: executeFn, + }); + const config = makeConfig({ tool }); + + const response = await executeToolCall( + config, + request({ + name: tool.name, + args: { command: 'rm -rf /' }, + executionOrigin: FIXED_ORIGIN, + }), + new AbortController().signal, + ); + + expect(response.errorType).toBe(ToolErrorType.EXECUTION_DENIED); + expect(response.error?.message).toContain('not a media policy tool'); + expect(executeFn).not.toHaveBeenCalled(); + }); +}); + +describe('CoreToolScheduler fixed_policy execution', () => { + it('executes a fixed_policy call without confirmation even when modelAccess is disabled', async () => { + const executeFn: Mock = vi.fn().mockResolvedValue({ + llmContent: 'compressed', + returnDisplay: 'compressed', + } satisfies ToolResult); + const getDefaultPermission = vi.fn(async () => 'ask' as const); + const tool = new MockMediaPolicyTool({ + name: 'omni_compress_image', + execute: executeFn, + getDefaultPermission, + }); + // No omniPolicyTools at all: modelAccess disabled by default, but the + // fixed-policy orchestrator path must still work. + const config = makeConfig({ tool }); + + const response = await executeToolCall( + config, + request({ + name: tool.name, + args: { source: 'a.png' }, + executionOrigin: FIXED_ORIGIN, + }), + new AbortController().signal, + ); + + expect(response.error).toBeUndefined(); + expect(executeFn).toHaveBeenCalledWith({ source: 'a.png' }); + // The interactive permission flow is skipped entirely. + expect(getDefaultPermission).not.toHaveBeenCalled(); + }); + + it('keeps confirmation for model-origin calls of the same enabled tool (bypass is origin-keyed)', async () => { + const executeFn = vi.fn(); + const tool = new MockMediaPolicyTool({ + name: 'omni_compress_image', + execute: executeFn, + getDefaultPermission: async () => 'ask' as const, + getConfirmationDetails: async () => ({ + type: 'info' as const, + title: 'Confirm compression', + prompt: 'Compress?', + onConfirm: async () => {}, + }), + }); + const config = makeConfig({ + tool, + interactive: true, + omniPolicyTools: { + omni_compress_image: { modelAccess: { enabled: true } }, + }, + }); + + const onToolCallsUpdate = vi.fn(); + const scheduler = new CoreToolScheduler({ + config, + onAllToolCallsComplete: vi.fn(), + onToolCallsUpdate, + getPreferredEditor: () => 'vscode', + onEditorClose: vi.fn(), + }); + + await scheduler.schedule( + [request({ name: tool.name, args: { source: 'a.png' } })], + new AbortController().signal, + ); + + await vi.waitFor(() => { + const statuses = onToolCallsUpdate.mock.calls + .flatMap((call) => call[0] as ToolCall[]) + .map((toolCall) => toolCall.status); + expect(statuses).toContain('awaiting_approval'); + }); + expect(executeFn).not.toHaveBeenCalled(); + }); + + it('still enforces PermissionManager.isToolEnabled for fixed_policy calls', async () => { + const executeFn = vi.fn(); + const tool = new MockMediaPolicyTool({ + name: 'omni_compress_image', + execute: executeFn, + }); + const config = makeConfig({ + tool, + isToolEnabled: async () => false, + }); + + const response = await executeToolCall( + config, + request({ + name: tool.name, + executionOrigin: FIXED_ORIGIN, + }), + new AbortController().signal, + ); + + expect(response.error).toBeDefined(); + expect(executeFn).not.toHaveBeenCalled(); + }); +}); + +describe('CoreToolScheduler policyArtifacts capture', () => { + const ARTIFACTS = [ + { + title: 'compressed.webp', + workspacePath: 'objects/compressed.webp', + mimeType: 'image/webp', + }, + ]; + + it('captures raw artifacts of a successful media-policy call into policyArtifacts', async () => { + const tool = new MockMediaPolicyTool({ + name: 'omni_compress_image', + execute: vi.fn().mockResolvedValue({ + llmContent: 'ok', + returnDisplay: 'ok', + artifacts: ARTIFACTS, + } satisfies ToolResult), + }); + const config = makeConfig({ tool }); + + const response = await executeToolCall( + config, + request({ + name: tool.name, + callId: 'staging-invocation-7', + executionOrigin: FIXED_ORIGIN, + }), + new AbortController().signal, + ); + + expect(response.error).toBeUndefined(); + expect(response.policyArtifacts).toEqual({ + toolName: 'omni_compress_image', + invocationId: 'staging-invocation-7', + executionOrigin: FIXED_ORIGIN, + artifacts: ARTIFACTS, + }); + }); + + it('reports a model origin in policyArtifacts for enabled model-origin calls', async () => { + const tool = new MockMediaPolicyTool({ + name: 'omni_compress_image', + execute: vi.fn().mockResolvedValue({ + llmContent: 'ok', + returnDisplay: 'ok', + artifacts: ARTIFACTS, + } satisfies ToolResult), + }); + const config = makeConfig({ + tool, + omniPolicyTools: { + omni_compress_image: { modelAccess: { enabled: true } }, + }, + }); + + const response = await executeToolCall( + config, + request({ name: tool.name }), + new AbortController().signal, + ); + + expect(response.policyArtifacts?.executionOrigin).toEqual({ + kind: 'model', + }); + }); + + it('does not emit policyArtifacts for ordinary tools with artifacts', async () => { + const tool = new MockTool({ + name: 'ordinary_tool', + execute: vi.fn().mockResolvedValue({ + llmContent: 'ok', + returnDisplay: 'ok', + artifacts: ARTIFACTS, + } satisfies ToolResult), + }); + const config = makeConfig({ tool }); + + const response = await executeToolCall( + config, + request({ name: tool.name }), + new AbortController().signal, + ); + + expect(response.error).toBeUndefined(); + expect(response.policyArtifacts).toBeUndefined(); + // The regular artifacts channel is unaffected. + expect(response.artifacts).toEqual(ARTIFACTS); + }); + + it('does not emit policyArtifacts when a media-policy call produced no artifacts', async () => { + const tool = new MockMediaPolicyTool({ + name: 'omni_compress_image', + execute: vi.fn().mockResolvedValue({ + llmContent: 'nothing to do', + returnDisplay: 'nothing to do', + } satisfies ToolResult), + }); + const config = makeConfig({ tool }); + + const response = await executeToolCall( + config, + request({ name: tool.name, executionOrigin: FIXED_ORIGIN }), + new AbortController().signal, + ); + + expect(response.error).toBeUndefined(); + expect(response.policyArtifacts).toBeUndefined(); + }); +}); diff --git a/packages/core/src/core/coreToolScheduler.test.ts b/packages/core/src/core/coreToolScheduler.test.ts index 9c21fe4a83c..fda8a0389a1 100644 --- a/packages/core/src/core/coreToolScheduler.test.ts +++ b/packages/core/src/core/coreToolScheduler.test.ts @@ -13,6 +13,7 @@ import type { Config, ToolCallConfirmationDetails, ToolConfirmationPayload, + ToolExecutionOrigin, ToolInvocation, ToolResult, ToolResultDisplay, @@ -58,6 +59,7 @@ import { MOCK_TOOL_GET_DEFAULT_PERMISSION, MOCK_TOOL_GET_CONFIRMATION_DETAILS, } from '../test-utils/mock-tool.js'; +import type { MediaPolicyToolDescriptor } from '../tools/tools.js'; import { GeminiChat } from './geminiChat.js'; import { MessageBusType } from '../confirmation-bus/types.js'; import type { HookExecutionResponse } from '../confirmation-bus/types.js'; @@ -89,6 +91,19 @@ import { todoWorkChainContext, } from '../utils/promptIdContext.js'; +/** MockTool that self-identifies as an omni media-policy tool, so a + * `fixed_policy` execution origin passes the scheduler's origin/descriptor + * pairing gate and reaches the code under test. */ +class MockMediaPolicyTool extends MockTool { + override get mediaPolicyDescriptor(): MediaPolicyToolDescriptor { + return { + kind: 'media_policy', + inputMediaTypes: ['image'], + outputs: [], + }; + } +} + type ToolSpanRecord = { name: string; attributes: Record; @@ -3844,6 +3859,80 @@ describe('CoreToolScheduler', () => { expect(runSideQueryMock).not.toHaveBeenCalled(); }); + it('skips the image funnel entirely for a fixed_policy invocation', async () => { + // Same vision-bridge setup that DOES bridge for a model-originated call + // (see the test above) — the only difference is the execution origin. + // A fixed-policy call's result never feeds the model (the orchestrator + // consumes policyArtifacts directly), and running the funnel would + // re-enter media processing from inside a policy run. + runSideQueryMock.mockResolvedValue({ text: 'Screen says READY' }); + const execute = vi.fn().mockResolvedValue({ + llmContent: [ + { text: 'degraded image written' }, + { + inlineData: { + mimeType: 'image/png', + data: 'aW1hZ2U=', + displayName: 'degraded.png', + }, + }, + ], + returnDisplay: 'degraded image written', + }); + const { scheduler, onAllToolCallsComplete } = + createSchedulerForLegacyToolTests({ + toolsByName: new Map([ + [ + 'omni_downsample_image', + new MockMediaPolicyTool({ + name: 'omni_downsample_image', + kind: Kind.Read, + execute, + }), + ], + ]), + visionBridge: true, + }); + + await scheduler.schedule( + [ + { + callId: 'call-policy-image', + name: 'omni_downsample_image', + args: {}, + isClientInitiated: false, + prompt_id: 'prompt-policy-image', + executionOrigin: { + kind: 'fixed_policy', + policyId: 'img-downsample', + stage: 'preprocessing', + }, + }, + ], + new AbortController().signal, + ); + await vi.waitFor(() => { + expect(onAllToolCallsComplete).toHaveBeenCalledOnce(); + }); + + const [completed] = onAllToolCallsComplete.mock.calls[0][0] as ToolCall[]; + if (completed.status !== 'success') { + throw new Error(`Expected success, received ${completed.status}`); + } + // No vision bridge side query, no bridged text, no notice/override. + expect(runSideQueryMock).not.toHaveBeenCalled(); + const functionResponse = + completed.response.responseParts[0].functionResponse; + expect(functionResponse?.response?.['output']).toContain( + 'degraded image written', + ); + expect(functionResponse?.response?.['output']).not.toContain( + 'Screen says READY', + ); + expect(completed.response.visionBridgeNotice).toBeUndefined(); + expect(completed.response.modelOverride).toBeUndefined(); + }); + it('bridges images returned with a tool error', async () => { runSideQueryMock.mockResolvedValue({ text: 'Dialog says access denied' }); const execute = vi.fn().mockResolvedValue({ @@ -10325,6 +10414,7 @@ describe('CoreToolScheduler telemetry spans', () => { args?: Record; abortController?: AbortController; tools?: MockTool[]; + executionOrigin?: ToolExecutionOrigin; }): Promise<{ scheduler: CoreToolScheduler; onAllToolCallsComplete: ReturnType; @@ -10342,6 +10432,9 @@ describe('CoreToolScheduler telemetry spans', () => { args: options.args ?? { input: 'x' }, isClientInitiated: false, prompt_id: 'prompt-ask', + ...(options.executionOrigin + ? { executionOrigin: options.executionOrigin } + : {}), }, ], abortController.signal, @@ -10494,6 +10587,77 @@ describe('CoreToolScheduler telemetry spans', () => { expect(getBlockedSpans()).toHaveLength(0); }); + it('denies a PreToolUse ask (no bounce) for a fixed_policy invocation', async () => { + // Interactive session where a model-originated call WOULD bounce — the + // exclusion must come from the execution origin alone: the orchestrator + // awaits the call headlessly behind the scheduler, so an + // awaiting_approval entry would sit unanswerable. + const execute = vi.fn(); + const messageBus = askMessageBus(); + const { onAllToolCallsComplete, onToolCallsUpdate } = await scheduleWithAsk( + { + messageBus, + // Media-policy tool: a fixed_policy origin on a non-policy tool + // would be rejected by the origin/descriptor gate before the hook + // even fires, which is not the path under test here. + tools: [new MockMediaPolicyTool({ name: 'mockTool', execute })], + executionOrigin: { + kind: 'fixed_policy', + policyId: 'img-downsample', + stage: 'preprocessing', + }, + }, + ); + + await vi.waitFor(() => { + expect(onAllToolCallsComplete).toHaveBeenCalled(); + }); + const completed = onAllToolCallsComplete.mock.calls.at( + -1, + )?.[0] as ToolCall[]; + expect(completed[0].status).toBe('error'); + expect(execute).not.toHaveBeenCalled(); + // Never bounced: no awaiting_approval transition, no blocked span. + const statuses = onToolCallsUpdate.mock.calls.flatMap((call) => + (call[0] as ToolCall[]).map((tc) => tc.status), + ); + expect(statuses).not.toContain('awaiting_approval'); + expect(getBlockedSpans()).toHaveLength(0); + }); + + it('still denies a hard PreToolUse deny for a fixed_policy invocation (fail-closed)', async () => { + // The fixed_policy exemption is scoped to the ask-bounce ONLY: a hook + // that hard-denies must block a policy-originated run exactly like any + // other — policies must not become a hook-bypass channel. + const execute = vi.fn(); + const messageBus = { + request: vi.fn().mockResolvedValue({ + type: MessageBusType.HOOK_EXECUTION_RESPONSE, + correlationId: 'pre-hook', + success: true, + output: { decision: 'deny', reason: 'blocked by policy hook' }, + }), + }; + const { onAllToolCallsComplete } = await scheduleWithAsk({ + messageBus, + tools: [new MockMediaPolicyTool({ name: 'mockTool', execute })], + executionOrigin: { + kind: 'fixed_policy', + policyId: 'img-downsample', + stage: 'preprocessing', + }, + }); + + await vi.waitFor(() => { + expect(onAllToolCallsComplete).toHaveBeenCalled(); + }); + const completed = onAllToolCallsComplete.mock.calls.at( + -1, + )?.[0] as ToolCall[]; + expect(completed[0].status).toBe('error'); + expect(execute).not.toHaveBeenCalled(); + }); + it('cancels a pending ask (no hang) when the signal aborts', async () => { const execute = vi.fn(); const messageBus = askMessageBus(); diff --git a/packages/core/src/core/coreToolScheduler.ts b/packages/core/src/core/coreToolScheduler.ts index c0028afd221..f643ccb349d 100644 --- a/packages/core/src/core/coreToolScheduler.ts +++ b/packages/core/src/core/coreToolScheduler.ts @@ -18,8 +18,11 @@ import type { AnyToolInvocation, ChatRecordingService, ToolArtifact, + PolicyArtifactBatch, + ToolExecutionOrigin, } from '../index.js'; import { createDebugLogger } from '../utils/debugLogger.js'; +import { evaluateMediaPolicyToolCall } from '../omni/policy/model-access.js'; import { sanitizeToolNameForProvider } from '../utils/tool-name-utils.js'; import { compactToolResultDisplayForHistory } from '../utils/toolResultDisplayCompaction.js'; import { @@ -1380,11 +1383,22 @@ export class CoreToolScheduler { private async processToolResultImages( responseParts: Part[], signal: AbortSignal, + executionOrigin?: ToolExecutionOrigin, ): Promise<{ responseParts: Part[]; modelOverride?: string; visionBridgeNotice?: string; }> { + // A fixed_policy invocation's result never feeds the model — the + // orchestrator consumes `policyArtifacts` directly — so the image + // funnel (omni re-delivery + vision bridge) must not run on it: it + // would re-enter media processing from inside a policy run, wasting + // work and re-acquiring the per-root resource gate this very run may + // already hold (self-deadlock at concurrency 1). Applies to every + // call site (success, timeout, tool error) via this single check. + if (executionOrigin?.kind === 'fixed_policy') { + return { responseParts }; + } let modelOverride: string | undefined; const notices: string[] = []; // Omni second normalization trigger point: convert inline tool-result @@ -2309,10 +2323,44 @@ export class CoreToolScheduler { continue; } + // Omni media-policy protocol gate (before buildInvocation, so the + // merged arguments still go through the tool's native schema and + // business validation): + // - model/client-origin calls of a media-policy tool require + // modelAccess.enabled, are rejected when they explicitly name a + // lockedArguments key, and get defaultArguments/lockedArguments + // merged in; + // - a fixed_policy origin on a NON-media-policy tool is rejected + // (defense in depth: a forged origin must not become a + // permission bypass for Shell/Edit/MCP tools); + // - a missing origin fails closed as model. + const policyGate = evaluateMediaPolicyToolCall({ + config: this.config, + tool: toolInstance, + args: reqInfo.args, + executionOrigin: reqInfo.executionOrigin, + }); + if (policyGate.outcome === 'reject') { + newToolCalls.push({ + status: 'error', + request: reqInfo, + tool: toolInstance, + response: createErrorResponse( + reqInfo, + new Error(policyGate.message), + policyGate.reason === 'invalid_params' + ? ToolErrorType.INVALID_TOOL_PARAMS + : ToolErrorType.EXECUTION_DENIED, + ), + durationMs: 0, + }); + continue; + } + const invocationOrError = runInRequestGoalContext(reqInfo, () => this.buildInvocation( toolInstance, - reqInfo.args, + policyGate.args, reqInfo.callId, reqInfo.prompt_id, ), @@ -2436,6 +2484,22 @@ export class CoreToolScheduler { // L3→L4→L5 Permission Flow // ================================================================= + // Fixed-policy orchestrator calls skip the interactive permission + // flow entirely: no PermissionManager ask/deny evaluation, no + // confirmation dialog, no plan/auto classification. The remaining + // guards still hold — PM tool-enablement ran at schedule time, + // origin/descriptor pairing was enforced before buildInvocation, + // and PreToolUse hooks fire (a hook deny fails the call closed) + // at execution time. + if (reqInfo.executionOrigin?.kind === 'fixed_policy') { + this.setToolCallOutcome( + reqInfo.callId, + ToolConfirmationOutcome.ProceedAlways, + ); + this.setStatusInternal(reqInfo.callId, 'scheduled'); + continue; + } + // ---- L3→L4: Shared permission flow ---- let toolParams = invocation.params as Record; const flowResult = await runInRequestGoalContext(reqInfo, () => @@ -4107,10 +4171,15 @@ export class CoreToolScheduler { // prompt, bounce the tool into the existing awaiting_approval flow // instead of denying it. 'denied'/'stop' (and 'ask' in a // non-interactive/background context where we cannot prompt) keep - // the original deny-as-error behavior. + // the original deny-as-error behavior. A fixed_policy invocation + // never bounces: the orchestrator awaits it headlessly behind the + // scheduler, so an awaiting_approval entry would sit unanswerable + // (the confirmation UI belongs to the outer tool call) — an 'ask' + // on a fixed-policy call fails closed as a deny instead. if ( preHookResult.blockType === 'ask' && !signal.aborted && + scheduledCall.request.executionOrigin?.kind !== 'fixed_policy' && this.canPromptForAskBounce() ) { // Mirror the confirmation-phase abort re-check: never open a @@ -4751,6 +4820,7 @@ export class CoreToolScheduler { const processedImages = await this.processToolResultImages( convertedResponse, signal, + scheduledCall.request.executionOrigin, ); const response = processedImages.responseParts; if (response !== convertedResponse) { @@ -4763,6 +4833,22 @@ export class CoreToolScheduler { ...(toolResult.artifacts ?? []), ...(postToolUseArtifacts ?? []), ]; + // Raw media-policy artifacts, captured from the tool's OWN result — + // deliberately excluding the PostToolUse hook artifacts merged into + // `artifacts` above, which must never impersonate policy outputs. + const policyArtifacts: PolicyArtifactBatch | undefined = + scheduledCall.tool.mediaPolicyDescriptor && + toolResult.artifacts && + toolResult.artifacts.length > 0 + ? { + toolName: canonicalName, + invocationId: callId, + executionOrigin: scheduledCall.request.executionOrigin ?? { + kind: 'model', + }, + artifacts: toolResult.artifacts, + } + : undefined; const successResponse: ToolCallResponseInfo = { callId, responseParts: response, @@ -4786,6 +4872,7 @@ export class CoreToolScheduler { ? { visionBridgeNotice: processedImages.visionBridgeNotice } : {}), ...(artifacts.length > 0 ? { artifacts } : {}), + ...(policyArtifacts ? { policyArtifacts } : {}), }; // After an APPROVED exit_plan_mode, swap the large `plan` argument // still sitting in the model turn's functionCall for a pointer to the @@ -4924,6 +5011,7 @@ export class CoreToolScheduler { const processedImages = await this.processToolResultImages( responseParts, signal, + scheduledCall.request.executionOrigin, ); responseParts = processedImages.responseParts; @@ -5037,6 +5125,7 @@ export class CoreToolScheduler { const processedImages = await this.processToolResultImages( imageErrorParts, signal, + scheduledCall.request.executionOrigin, ); const bridgedErrorParts = processedImages.responseParts; if ( diff --git a/packages/core/src/core/geminiChat.ts b/packages/core/src/core/geminiChat.ts index e05ac7f4e8a..b9d72d482eb 100644 --- a/packages/core/src/core/geminiChat.ts +++ b/packages/core/src/core/geminiChat.ts @@ -2436,6 +2436,7 @@ export class GeminiChat { transientInvalidStreamRetryCount + protocolTagLeakRetryCount; let transportStreamRetryCount = 0; let reactiveCompressionAttempted = false; + let omniMediaDegradeAttempts = 0; let suppressNextRetryEvent = false; let streamYieldedAnyChunk = false; @@ -2677,6 +2678,69 @@ export class GeminiChat { const contextOverflow = getContextLengthExceededInfo(error); if (contextOverflow.isExceeded) { + // Server-limit fallback for omni media (server-feedback-driven + // transport guard): a request carrying oss:// media that the + // server rejected as over its input limit is retried with the + // media degraded one guard-ladder rung further. Runs BEFORE + // reactive compression — history compression cannot shrink + // media tokens, which dominate these rejections. Bounded by + // the guard's maxTransportPasses and only armed when a + // normalized omni processing config exists (omni sessions). + const omniDegradeMaxAttempts = + self.config.getOmniProcessingConfig?.()?.limits + .maxTransportPasses ?? 0; + if ( + !exactRoute && + omniMediaDegradeAttempts < omniDegradeMaxAttempts + ) { + const degradeAttempt = omniMediaDegradeAttempts++; + let degradeOutcome: + | { replacedParts: number; degradedResources: number } + | undefined; + try { + // Dynamic import keeps the omni pipeline out of the send + // path for non-omni sessions (mirrors fileUtils). + const { degradeOmniMediaAfterServerReject } = await import( + '../omni/reactive-degrade.js' + ); + degradeOutcome = await degradeOmniMediaAfterServerReject( + self.config, + self.history, + degradeAttempt, + { + signal: params.config?.abortSignal, + observedLimitTokens: contextOverflow.limitTokens, + }, + ); + } catch (degradeError) { + if ( + params.config?.abortSignal?.aborted || + isAbortError(degradeError) + ) { + throw degradeError; + } + debugLogger.warn( + 'Omni media degradation fallback failed.', + degradeError, + ); + } + if (degradeOutcome && degradeOutcome.replacedParts > 0) { + self.popPendingPartialAssistantTurn(); + requestContents = self.getRequestHistoryForRoute( + currentUserContent, + requestModalities, + ); + debugLogger.warn( + `Server input limit exceeded; degraded ` + + `${degradeOutcome.degradedResources} omni media ` + + `resource(s) in place (attempt ${degradeAttempt + 1}/` + + `${omniDegradeMaxAttempts}); retrying.`, + ); + yield { type: StreamEventType.RETRY }; + suppressNextRetryEvent = true; + continue; + } + } if (!exactRoute && !reactiveCompressionAttempted) { reactiveCompressionAttempted = true; const reactiveOriginalTokenCount = diff --git a/packages/core/src/core/openaiContentGenerator/converter.test.ts b/packages/core/src/core/openaiContentGenerator/converter.test.ts index f2168cf8c07..f4611937993 100644 --- a/packages/core/src/core/openaiContentGenerator/converter.test.ts +++ b/packages/core/src/core/openaiContentGenerator/converter.test.ts @@ -1502,6 +1502,126 @@ describe('OpenAIContentConverter', () => { ); }); + it('moves an omni degradation disclosure together with its media part when splitting tool media', () => { + // The omni pipeline emits a disclosure text Part IMMEDIATELY before + // each lossy derivative's media Part. When splitToolMedia relocates + // the media into the follow-up user message, the disclosure must move + // WITH it — stranded in the text-only tool message, the model could + // not attribute it to the media. Ordinary text parts stay behind. + const request: GenerateContentParameters = { + model: 'models/test', + contents: [ + { + role: 'model', + parts: [{ functionCall: { id: 'call_1', name: 'Read', args: {} } }], + }, + { + role: 'user', + parts: [ + { + functionResponse: { + id: 'call_1', + name: 'Read', + response: { output: 'Image content' }, + parts: [ + { text: 'ordinary tool text' }, + { text: '【媒体降质】photo.png：downsampled to 1568px' }, + { + inlineData: { + mimeType: 'image/png', + data: 'base64encodedimagedata', + }, + }, + ] as unknown as Part[], + }, + }, + ], + }, + ], + }; + + const messages = converter.convertGeminiRequestToOpenAI(request, { + ...requestContext, + splitToolMedia: true, + }); + + const toolMessage = messages.find((m) => m.role === 'tool'); + expect(toolMessage?.content).toBe('Image content\nordinary tool text'); + + const userMessage = messages.find((m) => m.role === 'user'); + const userContent = userMessage?.content as Array<{ + type: string; + text?: string; + image_url?: { url: string }; + }>; + expect(userContent.map((p) => p.type)).toEqual([ + 'text', + 'text', + 'image_url', + ]); + expect(userContent[0].text).toBe( + '(attached media from previous tool call)', + ); + // Disclosure sits immediately before its media part. + expect(userContent[1].text).toBe( + '【媒体降质】photo.png：downsampled to 1568px', + ); + expect(userContent[2].image_url?.url).toBe( + 'data:image/png;base64,base64encodedimagedata', + ); + }); + + it('gives a disclosure only to the media part directly following it', () => { + // Two media parts after one disclosure: only the adjacent one owns + // it — the second media part must not pull the disclosure past the + // first (prev-tracking, not "last disclosure seen"). + const request: GenerateContentParameters = { + model: 'models/test', + contents: [ + { + role: 'model', + parts: [{ functionCall: { id: 'call_1', name: 'Read', args: {} } }], + }, + { + role: 'user', + parts: [ + { + functionResponse: { + id: 'call_1', + name: 'Read', + response: { output: 'two images' }, + parts: [ + { text: '【媒体降质】a.png：lossy' }, + { inlineData: { mimeType: 'image/png', data: 'first' } }, + { inlineData: { mimeType: 'image/png', data: 'second' } }, + ] as unknown as Part[], + }, + }, + ], + }, + ], + }; + + const messages = converter.convertGeminiRequestToOpenAI(request, { + ...requestContext, + splitToolMedia: true, + }); + const userMessage = messages.find((m) => m.role === 'user'); + const userContent = userMessage?.content as Array<{ + type: string; + text?: string; + image_url?: { url: string }; + }>; + expect(userContent.map((p) => p.type)).toEqual([ + 'text', + 'text', + 'image_url', + 'image_url', + ]); + expect(userContent[1].text).toBe('【媒体降质】a.png：lossy'); + expect(userContent[2].image_url?.url).toBe('data:image/png;base64,first'); + }); + it('should keep all tool messages contiguous and merge split media into a single follow-up user message for parallel tool calls (issue #3616)', () => { // Two assistant tool calls in parallel. Both responses come back in the // same `user` content as separate functionResponse parts. The first diff --git a/packages/core/src/core/openaiContentGenerator/converter.ts b/packages/core/src/core/openaiContentGenerator/converter.ts index d8dd2dcd038..e0497b98e6b 100644 --- a/packages/core/src/core/openaiContentGenerator/converter.ts +++ b/packages/core/src/core/openaiContentGenerator/converter.ts @@ -40,6 +40,7 @@ import { } from '../tool-call-preparation.js'; import { InvalidStreamError } from '../invalid-stream-error.js'; import { normalizeMcpToolName } from '../../utils/tool-name-utils.js'; +import { isDisclosureText } from '../../omni/disclosure.js'; import { setGenAiUsageProvenance } from '../../telemetry/gen-ai-usage.js'; const debugLogger = createDebugLogger('CONVERTER'); @@ -668,6 +669,15 @@ function processContent( ) { const mediaParts: OpenAIContentPart[] = []; const textParts: OpenAI.Chat.ChatCompletionContentPartText[] = []; + // Track the previous part so an omni media-degradation disclosure + // (emitted immediately before its media part) moves WITH the media + // into the follow-up user message instead of being stranded in the + // text-only tool message, where the model could not attribute it. + // The asymmetry with transcript text (§6.2) is deliberate: + // transcripts FOLLOW their media part and read fine as plain text + // in the tool message — only the disclosure carries the D8 + // adjacency requirement, so only the preceding disclosure migrates. + let prev: OpenAIContentPart | undefined; for (const cp of toolMessage.content as OpenAIContentPart[]) { if ( cp && @@ -676,10 +686,17 @@ function processContent( cp.type === 'video_url' || cp.type === 'file') ) { + if (prev?.type === 'text' && isDisclosureText(prev.text)) { + textParts.pop(); + mediaParts.push(prev); + } mediaParts.push(cp); } else if (cp && cp.type === 'text') { textParts.push(cp); } + // Consecutive media parts after one disclosure must not each + // claim it: only the part directly following the text does. + prev = cp; } if (mediaParts.length > 0) { const textOnly = textParts.map((p) => p.text).join('\n'); diff --git a/packages/core/src/core/prompts.ts b/packages/core/src/core/prompts.ts index bd3b02c345a..c88429af224 100644 --- a/packages/core/src/core/prompts.ts +++ b/packages/core/src/core/prompts.ts @@ -486,6 +486,12 @@ export interface SystemPromptLayers { * fixed for the whole session. */ base: string; + /** + * Stable layer: the omni progressive-media-understanding contract + * (omni/media-guidance.ts) — fixed for the whole session (the omni + * config and provider do not change in-session). + */ + mediaGuidance?: string | null; /** * Context layer: concatenated context files (QWEN.md hierarchy, baseline * rules, extension files). Reloaded only on explicit refresh. @@ -510,6 +516,7 @@ export interface SystemPromptLayers { export function assembleSystemPrompt(layers: SystemPromptLayers): string { return ( layers.base + + buildSystemPromptSuffix(layers.mediaGuidance ?? undefined) + buildSystemPromptSuffix(layers.contextFiles) + buildSystemPromptSuffix(layers.appendPrompt) + (layers.gitStatus ? `\n\n${layers.gitStatus}` : '') + diff --git a/packages/core/src/core/turn.ts b/packages/core/src/core/turn.ts index 488f9662065..d7bee2809a5 100644 --- a/packages/core/src/core/turn.ts +++ b/packages/core/src/core/turn.ts @@ -122,6 +122,49 @@ export interface GeminiFinishedEventValue { usageMetadata: GenerateContentResponseUsageMetadata | undefined; } +/** + * Provenance of a tool-call request. Set exclusively by in-process callers + * when they construct the {@link ToolCallRequestInfo} — it is NEVER parsed + * from tool parameters, wire protocols, or model output, and it is NEVER + * inferred from `isClientInitiated`. A missing origin fails closed as + * `{ kind: 'model' }` (the least-privileged origin). + * + * `fixed_policy` marks calls issued by the omni fixed-policy orchestrator: + * they bypass the interactive permission flow (no confirmation dialog, no + * plan/auto classification) but still honor PreToolUse hooks and the + * PermissionManager tool-enablement check. + */ +export type ToolExecutionOrigin = + | { kind: 'model' } + | { kind: 'client' } + | { + kind: 'fixed_policy'; + /** ID of the fixed policy that issued this call. */ + policyId: string; + /** Pipeline stage the policy ran in. */ + stage: 'preprocessing' | 'transport_guard'; + }; + +/** + * Raw, successful media-policy tool artifacts captured by the scheduler + * BEFORE PostToolUse hook artifacts are merged in — hook-produced artifacts + * must never impersonate policy outputs. Carried on + * {@link ToolCallResponseInfo.policyArtifacts} for the fixed-policy + * orchestrator (and the model-call artifact bridge) to consume. + */ +export interface PolicyArtifactBatch { + /** Canonical tool name that produced the artifacts. */ + toolName: string; + /** The call id of the invocation (the orchestrator uses its staging + * invocation id as the call id, so this keys the staging directory). */ + invocationId: string; + /** Origin the call executed under (missing origins fail closed to model + * before this batch is built, so this is always concrete). */ + executionOrigin: ToolExecutionOrigin; + /** The tool's own `ToolResult.artifacts`, unmerged and in order. */ + artifacts: ToolArtifact[]; +} + export interface ToolCallRequestInfo { callId: string; /** @@ -137,6 +180,12 @@ export interface ToolCallRequestInfo { /** Set to true when the LLM response was truncated due to max_tokens. */ wasOutputTruncated?: boolean; goalContext?: GoalTurnPermit; + /** + * Provenance of this request. Only set by in-process callers; absent on + * every request materialized from model output or a wire protocol. + * Consumers treat a missing value as `{ kind: 'model' }` (fail closed). + */ + executionOrigin?: ToolExecutionOrigin; } export interface ToolCallResponseInfo { @@ -150,6 +199,12 @@ export interface ToolCallResponseInfo { modelOverride?: string; visionBridgeNotice?: string; artifacts?: ToolArtifact[]; + /** + * Raw successful artifacts of a media-policy tool, captured before + * PostToolUse hook artifact merging. Absent for non-media-policy tools, + * failed calls, and calls that produced no artifacts. + */ + policyArtifacts?: PolicyArtifactBatch; } function normalizeRequestParts(req: PartListUnion): Part[] { diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 962bf58b41b..f50b9c401c4 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -675,13 +675,32 @@ export { downloadMediaUrl, effectiveMaxDownloadFileBytes, recognizeMediaFile, + formatDisclosureText, + formatOmissionText, + buildAdditionalMediaParts, + buildTranscriptParts, OmniObjectStore, OmniDeliveryError, OmniDownloadError, OmniTransportGuardError, type OmniModality, type OmniMediaDelivery, + type OmniAdditionalMediaDelivery, + type OmniAdditionalMediaPart, type OmniTokenEstimate, type DownloadedMedia, } from './omni/index.js'; export { processToolResultOmniMedia } from './omni/tool-result-media.js'; +export { + resolveMediaPolicyModelAccess, + isMediaPolicyToolHiddenFromModel, + evaluateMediaPolicyToolCall, + type MediaPolicyConfigView, + type MediaPolicyCallGateResult, + type ResolvedMediaPolicyModelAccess, +} from './omni/policy/model-access.js'; +export type { + OmniPolicyToolSettings, + OmniPolicyToolModelAccessSettings, + OmniPolicyToolsSettings, +} from './omni/policy/types.js'; diff --git a/packages/core/src/omni/delivery-gate.ts b/packages/core/src/omni/delivery-gate.ts new file mode 100644 index 00000000000..b5cb9ae4327 --- /dev/null +++ b/packages/core/src/omni/delivery-gate.ts @@ -0,0 +1,74 @@ +/** + * @license + * Copyright 2026 Qwen Team + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Leaf module for the omni delivery activation check, split out of + * `omni/index.ts` so lightweight consumers (system-prompt assembly in + * client.ts, media-guidance.ts) can evaluate the gate without statically + * pulling in the whole delivery pipeline (storage, upload, ffmpeg, + * policy orchestrator). + */ + +import type { Config } from '../config/config.js'; +import { AuthType } from '../core/contentGenerator.js'; +import { DashScopeOpenAICompatibleProvider } from '../core/openaiContentGenerator/provider/dashscope.js'; +import { createDebugLogger } from '../utils/debugLogger.js'; + +const debugLogger = createDebugLogger('omni:delivery-gate'); + +/** + * Placeholder the model-config resolver assigns under Qwen OAuth; the real + * token is swapped in per-request by QwenContentGenerator and never lands + * in the ContentGeneratorConfig, so it cannot authenticate the uploads + * endpoint. See modelConfigResolver.ts. + */ +const QWEN_OAUTH_PLACEHOLDER_API_KEY = 'QWEN_OAUTH_DYNAMIC_TOKEN'; + +/** + * Gate for the omni delivery path. All conditions must hold: + * + * 1. omni enabled (settings or QWEN_CODE_ENABLE_OMNI=1); + * 2. trusted workspace (the pipeline writes .qwen/omni/ and uploads + * workspace bytes off-machine); + * 3. a usable API key for the uploads endpoint — Qwen OAuth is excluded: + * its ContentGeneratorConfig carries a placeholder, and the OAuth token + * is not accepted by the uploads channel; + * 4. an explicit baseUrl (the uploads origin is derived from it — never + * send the configured credential to an origin the user didn't set); + * 5. a DashScope-compatible provider. + * + * Any failed condition falls back to the pre-omni inline behavior. + * Modality support is checked by the caller (fileUtils) alongside the + * existing modality gate. + */ +export function isOmniDeliveryActive(config: Config): boolean { + // Optional calls so stub Configs in tests (and embedders constructing + // partial configs) don't need the omni accessors to process files. + if (!config.isOmniEnabled?.()) return false; + if (config.isTrustedFolder?.() === false) { + debugLogger.debug('omni delivery inactive: untrusted workspace'); + return false; + } + const cgc = config.getContentGeneratorConfig?.(); + if (!cgc) return false; + if ( + cgc.authType === AuthType.QWEN_OAUTH || + !cgc.apiKey || + cgc.apiKey === QWEN_OAUTH_PLACEHOLDER_API_KEY + ) { + debugLogger.debug( + 'omni delivery inactive: no static API key usable for the uploads endpoint (Qwen OAuth is not supported)', + ); + return false; + } + if (!cgc.baseUrl) { + debugLogger.debug( + 'omni delivery inactive: no explicit baseUrl to derive the uploads origin from', + ); + return false; + } + return DashScopeOpenAICompatibleProvider.isDashScopeProvider(cgc); +} diff --git a/packages/core/src/omni/disclosure.ts b/packages/core/src/omni/disclosure.ts new file mode 100644 index 00000000000..46a1e11f862 --- /dev/null +++ b/packages/core/src/omni/disclosure.ts @@ -0,0 +1,60 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Disclosure text delivery (decision D8): a lossy policy derivative must + * reach the model with its disclosure IMMEDIATELY adjacent to the media + * Part, so provider converters that relocate media (splitToolMedia) can + * move the pair together and the model can attribute the disclosure to + * the right resource. + * + * Deliberately a leaf module — imported by both the omni pipeline and the + * OpenAI converter, so it must not pull in either side. + */ + +/** Marks a text Part as a media-degradation disclosure. Converters key on + * this prefix to keep the disclosure adjacent to its media part. */ +export const OMNI_DISCLOSURE_TEXT_PREFIX = '【媒体降质】'; + +/** Model-facing disclosure text for one degraded resource. */ +export function formatDisclosureText( + displayName: string, + disclosure: string, +): string { + return `${OMNI_DISCLOSURE_TEXT_PREFIX}${displayName}：${disclosure}`; +} + +/** Whether a text is a disclosure emitted by {@link formatDisclosureText}. */ +export function isDisclosureText(text: string): boolean { + return text.startsWith(OMNI_DISCLOSURE_TEXT_PREFIX); +} + +/** Marks a text Part as an explicit-omission notice: the transport guard + * could not bring a resource within limits, so the media was withheld and + * this text stands in its place (policy design §10.2). */ +export const OMNI_OMISSION_TEXT_PREFIX = '【媒体省略】'; + +/** Model-facing omission notice for one withheld resource. */ +export function formatOmissionText( + displayName: string, + reason: string, +): string { + return `${OMNI_OMISSION_TEXT_PREFIX}${displayName}：${reason}`; +} + +/** Marks a text Part as a media transcript: a text derivative (upstream P + * §6.2 transcript protocol, `metadata.omniRole: 'transcript'`) produced by + * a fixed policy and delivered as text instead of (or alongside) the media + * Part. */ +export const OMNI_TRANSCRIPT_TEXT_PREFIX = '【媒体转写】'; + +/** Model-facing transcript text for one media resource. */ +export function formatTranscriptText( + displayName: string, + transcript: string, +): string { + return `${OMNI_TRANSCRIPT_TEXT_PREFIX}${displayName}：${transcript}`; +} diff --git a/packages/core/src/omni/download.test.ts b/packages/core/src/omni/download.test.ts index d880c7c3b2c..f2039c7f8df 100644 --- a/packages/core/src/omni/download.test.ts +++ b/packages/core/src/omni/download.test.ts @@ -477,6 +477,34 @@ describe('downloadMediaUrl', () => { expect(fetchFn).not.toHaveBeenCalled(); }); + it('refuses a symlink planted at the downloads path (no bytes through the link)', async () => { + // mkdir { recursive: true } succeeds silently on a symlink-to-dir, so + // without the lstat guard the streamed bytes would land at an + // attacker-chosen location outside the omni root. + const outside = await fs.mkdtemp(path.join(os.tmpdir(), 'omni-dl-out-')); + const linkDir = path.join(downloadsDir, 'downloads'); + await fs.symlink(outside, linkDir); + const fetchFn = fetchOk('media-bytes'); + try { + const err = await downloadMediaUrl({ + url: 'https://media.example.com/a.mp4', + downloadsDir: linkDir, + maxBytes: 1000, + fetchFn, + resolveTarget: async (u) => pinnedTarget(u), + }).catch((e: Error) => e); + expect(err).toBeInstanceOf(OmniDownloadError); + expect((err as Error).message).toMatch( + /Could not prepare the downloads directory/, + ); + expect(fetchFn).not.toHaveBeenCalled(); + // Nothing was written through the link. + await expect(fs.readdir(outside)).resolves.toEqual([]); + } finally { + await fs.rm(outside, { recursive: true, force: true }); + } + }); + it('pins the connection to the vetted address (real socket)', async () => { // The regression test for the check-then-connect hole: a preflight-only // gate lets `fetch` re-resolve the name, so this asserts the socket is diff --git a/packages/core/src/omni/download.ts b/packages/core/src/omni/download.ts index 2fabf1fdb1e..b0c7e51344f 100644 --- a/packages/core/src/omni/download.ts +++ b/packages/core/src/omni/download.ts @@ -5,6 +5,7 @@ */ import { randomBytes } from 'node:crypto'; +import { once } from 'node:events'; import { createWriteStream } from 'node:fs'; import fs from 'node:fs/promises'; import path from 'node:path'; @@ -17,6 +18,7 @@ import { } from '../extension/network-policy.js'; import { loadUndici, detectRuntime } from '../utils/runtimeFetchOptions.js'; import { createDebugLogger } from '../utils/debugLogger.js'; +import { prepareOmniDownloadsDir } from './storage.js'; const debugLogger = createDebugLogger('omni:download'); @@ -298,9 +300,11 @@ export async function downloadMediaUrl(params: { // Local filesystem failures (ENOTDIR, EROFS, EACCES) must stay inside the // OmniDownloadError contract, and fs error text embeds the absolute - // directory path, which must not reach the UI or the debug log. + // directory path, which must not reach the UI or the debug log. The + // prepare step is also the symlink guard: a link planted at downloads/ + // would redirect the streamed bytes to an attacker-chosen location. try { - await fs.mkdir(downloadsDir, { recursive: true, mode: 0o700 }); + await prepareOmniDownloadsDir(downloadsDir); } catch (err) { throw new OmniDownloadError( `Could not prepare the downloads directory for URL media: ${ @@ -526,11 +530,15 @@ export async function downloadMediaUrl(params: { yield chunk; } }; - await pipeline( - source, - counter, - createWriteStream(partPath, { mode: 0o600 }), - ); + // The fd must be open BEFORE the pipeline runs: createWriteStream + // opens lazily, so a failure on an all-synchronous body (e.g. the + // byte cap tripping on the first chunks) could otherwise reject — + // and reach the outer `.part` cleanup — before the file was even + // created, resurrecting the `.part` after its rm. (On open failure + // the stream self-destructs and `once` rejects into the catch.) + const destination = createWriteStream(partPath, { mode: 0o600 }); + await once(destination, 'open'); + await pipeline(source, counter, destination); } catch (err) { if (signal?.aborted) throw err; if (err instanceof OmniDownloadError) throw err; diff --git a/packages/core/src/omni/ffmpeg.test.ts b/packages/core/src/omni/ffmpeg.test.ts index 72914799c2e..78a380be7a7 100644 --- a/packages/core/src/omni/ffmpeg.test.ts +++ b/packages/core/src/omni/ffmpeg.test.ts @@ -17,6 +17,7 @@ import { isFfprobeAvailable, probeMediaMetadata, resetFfmpegCachesForTests, + runFfmpeg, } from './ffmpeg.js'; type ExecCallback = ( @@ -203,9 +204,69 @@ describe('probeMediaMetadata per-modality branches', () => { formatName: 'mov,mp4', durationMs: 12_500, codec: 'aac', + sampleRateHz: 44_100, + channels: 2, }); }); + it('prefers the format-level bit rate and falls back to the stream', async () => { + mockExecResult(() => ({ + stdout: JSON.stringify({ + format: { format_name: 'mp3', duration: '10', bit_rate: '320000' }, + streams: [ + { codec_type: 'audio', codec_name: 'mp3', bit_rate: '128000' }, + ], + }), + })); + await expect(probeMediaMetadata('/a.mp3', 'audio')).resolves.toMatchObject({ + bitRate: 320_000, + }); + + mockExecResult(() => ({ + stdout: JSON.stringify({ + format: { format_name: 'mp3', duration: '10' }, + streams: [ + { codec_type: 'audio', codec_name: 'mp3', bit_rate: '128000' }, + ], + }), + })); + await expect(probeMediaMetadata('/a.mp3', 'audio')).resolves.toMatchObject({ + bitRate: 128_000, + }); + }); + + it('reports the video bit rate for modality video', async () => { + mockExecResult(() => ({ + stdout: JSON.stringify({ + format: { format_name: 'mp4', duration: '5', bit_rate: '2500000' }, + streams: [{ codec_type: 'video', codec_name: 'h264' }], + }), + })); + await expect(probeMediaMetadata('/v.mp4', 'video')).resolves.toMatchObject({ + bitRate: 2_500_000, + }); + }); + + it('omits bitRate/sampleRateHz/channels when unusable', async () => { + mockExecResult(() => ({ + stdout: JSON.stringify({ + format: { format_name: 'wav', duration: '3', bit_rate: 'N/A' }, + streams: [ + { + codec_type: 'audio', + codec_name: 'pcm_s16le', + sample_rate: 'N/A', + channels: 0, + }, + ], + }), + })); + const result = await probeMediaMetadata('/a.wav', 'audio'); + expect(result.bitRate).toBeUndefined(); + expect(result.sampleRateHz).toBeUndefined(); + expect(result.channels).toBeUndefined(); + }); + it("reads only dimensions for modality 'image' (no duration)", async () => { mockExecResult(() => ({ stdout: JSON.stringify({ @@ -283,3 +344,51 @@ describe('probeMediaMetadata per-modality branches', () => { expect(result.codec).toBeUndefined(); }); }); + +describe('runFfmpeg', () => { + it('invokes ffmpeg with the given args and resolves code 0 on success', async () => { + mockExecResult(() => ({ stderr: 'frame= 100' })); + await expect( + runFfmpeg(['-y', '-i', '/in.mov', '/out.mp4']), + ).resolves.toEqual({ + code: 0, + stderr: 'frame= 100', + }); + const [command, args, options] = execFileMock.mock.calls[0]; + expect(command).toBe('ffmpeg'); + expect(args).toEqual(['-y', '-i', '/in.mov', '/out.mp4']); + expect(options).toMatchObject({ maxBuffer: 16 * 1024 * 1024 }); + expect(options).not.toHaveProperty('timeout'); + }); + + it('threads timeoutMs and signal into execFile options', async () => { + mockExecResult(() => ({})); + const signal = new AbortController().signal; + await runFfmpeg(['-version'], { signal, timeoutMs: 120_000 }); + const options = execFileMock.mock.calls[0][2]; + expect(options).toMatchObject({ timeout: 120_000, signal }); + }); + + it('never rejects: a failing run resolves with the exit code and stderr', async () => { + mockExecResult(() => ({ + error: Object.assign(new Error('exit 187'), { code: 187 }), + stderr: 'Conversion failed!', + })); + await expect(runFfmpeg(['-i', '/in.mov'])).resolves.toEqual({ + code: 187, + stderr: 'Conversion failed!', + }); + }); + + it('maps a non-numeric error code (e.g. ENOENT/abort kill) to 1', async () => { + mockExecResult(() => ({ + error: Object.assign(new Error('spawn ffmpeg ENOENT'), { + code: 'ENOENT', + }), + })); + await expect(runFfmpeg(['-version'])).resolves.toEqual({ + code: 1, + stderr: '', + }); + }); +}); diff --git a/packages/core/src/omni/ffmpeg.ts b/packages/core/src/omni/ffmpeg.ts index 736fbfc5c7f..fead0c19527 100644 --- a/packages/core/src/omni/ffmpeg.ts +++ b/packages/core/src/omni/ffmpeg.ts @@ -121,6 +121,35 @@ export async function assertOmniRuntimeDependencies(): Promise { ); } +/** Outcome of one ffmpeg run (see {@link runFfmpeg}). */ +export interface FfmpegRunResult { + /** Process exit code (non-zero on failure, including timeout kill). */ + code: number; + /** Captured stderr (ffmpeg writes its diagnostics there). */ + stderr: string; +} + +/** + * Run ffmpeg with the given arguments. Never rejects — callers branch on + * the exit code, and MUST check `signal?.aborted` explicitly afterwards + * (an aborted run also surfaces as a non-zero code, but the two need + * different error messages). `timeoutMs` kills the process when exceeded, + * which likewise surfaces as a non-zero exit code. + */ +export async function runFfmpeg( + args: string[], + options?: { signal?: AbortSignal; timeoutMs?: number }, +): Promise { + const { code, stderr } = await execCommand('ffmpeg', args, { + // Transcodes are long-running; stderr carries progress lines, so give + // it more headroom than the probe calls. + maxBuffer: 16 * 1024 * 1024, + ...(options?.timeoutMs !== undefined && { timeout: options.timeoutMs }), + ...(options?.signal && { signal: options.signal }), + }); + return { code, stderr }; +} + /** Media metadata extracted via ffprobe (fields populated per modality). */ export interface MediaProbeResult { /** Container/format name reported by ffprobe (e.g. "mov,mp4,m4a,..."). */ @@ -138,6 +167,12 @@ export interface MediaProbeResult { /** Frame count of the primary video stream (image: >1 means animated — * GIF/APNG/animated WebP; absent when the container does not report it). */ frameCount?: number; + /** Overall bit rate in bits/second (format-level; audio/video). */ + bitRate?: number; + /** Sample rate in Hz of the first audio stream (audio only). */ + sampleRateHz?: number; + /** Channel count of the first audio stream (audio only). */ + channels?: number; } /** Parse an ffprobe rational like "30000/1001" (or plain "25") into fps. */ @@ -153,6 +188,63 @@ function parseFrameRate(raw: string | undefined): number | undefined { return Number.isFinite(fps) && fps > 0 ? fps : undefined; } +/** Image containers/codecs that can hold more than one frame. Only these + * warrant the decode-and-count fallback below — a plain JPEG/BMP without + * nb_frames is single-frame by construction. */ +function isAnimationCapableImage( + formatName: string | undefined, + codecName: string | undefined, +): boolean { + const tokens = new Set([ + ...(formatName?.split(',').map((t) => t.trim()) ?? []), + ...(codecName ? [codecName] : []), + ]); + return ['gif', 'webp', 'png', 'apng'].some((t) => tokens.has(t)); +} + +/** + * Count an image stream's frames by decoding it (`-count_frames` → + * `nb_read_frames`). The fallback for animation-capable containers whose + * headers carry no frame count (animated WebP, APNG). Returns NaN when + * counting fails or aborts — the caller's finite-and-positive guard then + * omits frameCount; the image tools' sharp `pages` check remains as the + * second, independent animated-input backstop. + */ +async function countImageFrames( + filePath: string, + signal?: AbortSignal, +): Promise { + try { + const { stdout, code } = await execCommand( + 'ffprobe', + [ + '-v', + 'error', + '-count_frames', + '-select_streams', + 'v:0', + '-show_entries', + 'stream=nb_read_frames', + '-print_format', + 'json', + filePath, + ], + { + timeout: 15_000, + maxBuffer: 4 * 1024 * 1024, + ...(signal && { signal }), + }, + ); + if (signal?.aborted || code !== 0) return NaN; + const parsed = JSON.parse(stdout) as { + streams?: Array<{ nb_read_frames?: string }>; + }; + return Number(parsed.streams?.[0]?.nb_read_frames); + } catch { + return NaN; + } +} + /** * Probe a local media file with ffprobe. Throws on non-zero exit or * unparseable output — the omni pipeline treats a failed probe as a @@ -188,7 +280,7 @@ export async function probeMediaMetadata( ); } let parsed: { - format?: { format_name?: string; duration?: string }; + format?: { format_name?: string; duration?: string; bit_rate?: string }; streams?: Array<{ codec_type?: string; codec_name?: string; @@ -197,6 +289,9 @@ export async function probeMediaMetadata( avg_frame_rate?: string; r_frame_rate?: string; nb_frames?: string; + sample_rate?: string; + channels?: number; + bit_rate?: string; }>; }; try { @@ -213,6 +308,14 @@ export async function probeMediaMetadata( Number.isFinite(durationSeconds) && durationSeconds >= 0 ? Math.round(durationSeconds * 1000) : undefined; + const parsePositiveInt = (raw: string | undefined): number | undefined => { + const n = Number(raw); + return Number.isFinite(n) && n > 0 ? Math.round(n) : undefined; + }; + // Prefer the format-level bit rate; fall back to the primary stream's. + const bitRateFor = (stream?: { bit_rate?: string }): number | undefined => + parsePositiveInt(parsed.format?.bit_rate) ?? + parsePositiveInt(stream?.bit_rate); const base: MediaProbeResult = { formatName: parsed.format?.format_name }; switch (modality) { @@ -221,7 +324,22 @@ export async function probeMediaMetadata( // video stream; a single-frame image reports 1 or omits it. The token // estimator needs the real count — an animated GIF estimated as one // frame sails under the transport guard at ~1/300 of its real cost. - const nbFrames = Number(videoStream?.nb_frames); + let nbFrames = Number(videoStream?.nb_frames); + if ( + !(Number.isFinite(nbFrames) && nbFrames > 0) && + isAnimationCapableImage( + parsed.format?.format_name, + videoStream?.codec_name, + ) + ) { + // Animation-capable container without a reported nb_frames: + // ffprobe leaves it out for WebP and APNG (their headers carry no + // frame count), so `missing` must not be read as `single-frame` — + // that would fail OPEN through every animated-image gate (the D9 + // still-image exclusion and both image tools' refusals). Decode + // the stream once to count the real frames. + nbFrames = await countImageFrames(filePath, signal); + } return { ...base, width: videoStream?.width, @@ -232,12 +350,17 @@ export async function probeMediaMetadata( : {}), }; } - case 'audio': + case 'audio': { + const channels = audioStream?.channels; return { ...base, durationMs, codec: audioStream?.codec_name, + bitRate: bitRateFor(audioStream), + sampleRateHz: parsePositiveInt(audioStream?.sample_rate), + ...(typeof channels === 'number' && channels > 0 ? { channels } : {}), }; + } case 'video': default: return { @@ -249,6 +372,7 @@ export async function probeMediaMetadata( parseFrameRate(videoStream?.avg_frame_rate) ?? parseFrameRate(videoStream?.r_frame_rate), codec: videoStream?.codec_name, + bitRate: bitRateFor(videoStream), }; } } diff --git a/packages/core/src/omni/guard.test.ts b/packages/core/src/omni/guard.test.ts index d1b1d825374..dc18999788b 100644 --- a/packages/core/src/omni/guard.test.ts +++ b/packages/core/src/omni/guard.test.ts @@ -16,7 +16,7 @@ import type { RecognizedMedia } from './recognition.js'; function cfg(overrides: { maxBytes?: number; maxTokens?: number }): Config { return { - getOmniUploadMaxFileBytes: vi.fn().mockReturnValue(overrides.maxBytes), + getOmniMaxUploadFileBytes: vi.fn().mockReturnValue(overrides.maxBytes), getOmniMaxEstimatedTokens: vi.fn().mockReturnValue(overrides.maxTokens), } as unknown as Config; } @@ -53,7 +53,9 @@ describe('byte guard', () => { it('rejects above the configured limit with an explanatory message', () => { expect(() => assertWithinByteLimit(cfg({ maxBytes: 1000 }), 2000, 'clip.mp4'), - ).toThrow(/clip\.mp4.*2000 bytes > 1000 bytes.*omni\.upload\.maxFileBytes/); + ).toThrow( + /clip\.mp4.*2000 bytes > 1000 bytes.*omni\.processing\.transportGuard\.maxUploadFileBytes/, + ); }); it('passes at or below the limit', () => { @@ -81,7 +83,7 @@ describe('token guard', () => { expect(() => assertWithinTokenLimit(cfg({ maxTokens: 196_608 }), VIDEO_8MIN, 'p3.mp4'), ).toThrow( - /p3\.mp4.*raw-resource-v1.*196608.*omni\.transport\.maxEstimatedTokens/, + /p3\.mp4.*raw-resource-v1.*196608.*omni\.processing\.transportGuard\.maxEstimatedTokens/, ); }); diff --git a/packages/core/src/omni/guard.ts b/packages/core/src/omni/guard.ts index d1fb9ee2069..6acebb0865e 100644 --- a/packages/core/src/omni/guard.ts +++ b/packages/core/src/omni/guard.ts @@ -18,15 +18,15 @@ export const DEFAULT_OMNI_MAX_UPLOAD_FILE_BYTES = 1024 * 1024 * 1024; * surface the message; there is no silent degradation. Messages must stay * free of absolute paths (they can reach model-visible content). */ export class OmniTransportGuardError extends Error { - constructor(message: string) { - super(message); + constructor(message: string, options?: { cause?: unknown }) { + super(message, options); this.name = 'OmniTransportGuardError'; } } /** Resolve the effective byte ceiling (undefined/<=0 config → default). */ export function effectiveMaxUploadFileBytes(config: Config): number { - const configured = config.getOmniUploadMaxFileBytes?.(); + const configured = config.getOmniMaxUploadFileBytes?.(); return configured !== undefined && configured > 0 ? configured : DEFAULT_OMNI_MAX_UPLOAD_FILE_BYTES; @@ -44,7 +44,7 @@ export function assertWithinByteLimit( if (sizeBytes > maxBytes) { throw new OmniTransportGuardError( `${displayName} exceeds the omni upload limit: ${sizeBytes} bytes > ` + - `${maxBytes} bytes (omni.upload.maxFileBytes). ` + + `${maxBytes} bytes (omni.processing.transportGuard.maxUploadFileBytes). ` + `Reduce the file size before retrying.`, ); } @@ -55,7 +55,7 @@ export function assertWithinByteLimit( * BEFORE store/upload, so an oversized input costs one probe — not a copy * and a multi-minute upload. * - * Threshold semantics (`omni.transport.maxEstimatedTokens`): + * Threshold semantics (`omni.processing.transportGuard.maxEstimatedTokens`): * - unset / 0 / negative → guard disabled (the estimation formula is still * pending confirmation with the model provider; estimates are attached * for observability but must not reject until a threshold is set); @@ -77,7 +77,7 @@ export function assertWithinTokenLimit( throw new OmniTransportGuardError( `${displayName} exceeds the omni estimated-token limit: ` + `~${estimate.estimatedTokenCount} tokens (${estimate.method}) > ` + - `${maxTokens} (omni.transport.maxEstimatedTokens). ` + + `${maxTokens} (omni.processing.transportGuard.maxEstimatedTokens). ` + `Reduce duration/resolution or raise the limit.`, ); } diff --git a/packages/core/src/omni/index.test.ts b/packages/core/src/omni/index.test.ts index 8bfa8577b5e..fe74ae809d4 100644 --- a/packages/core/src/omni/index.test.ts +++ b/packages/core/src/omni/index.test.ts @@ -72,8 +72,8 @@ describe('sanitizeErrorMessage', () => { describe('effectiveMaxDownloadFileBytes', () => { const capsConfig = (download?: number, upload?: number): Config => ({ - getOmniDownloadMaxFileBytes: () => download, - getOmniUploadMaxFileBytes: () => upload, + getOmniUrlDownloadMaxFileBytes: () => download, + getOmniMaxUploadFileBytes: () => upload, }) as unknown as Config; it('never exceeds the upload cap, even when configured higher', () => { @@ -186,7 +186,7 @@ describe('readMediaViaOmniDelivery result shape', () => { isTrustedFolder: vi.fn().mockReturnValue(true), getContentGeneratorConfig: vi.fn().mockReturnValue(DASHSCOPE_CGC), getModel: vi.fn().mockReturnValue('qwen3.5-omni-plus'), - getOmniUploadMaxFileBytes: vi.fn().mockReturnValue(0), + getOmniMaxUploadFileBytes: vi.fn().mockReturnValue(0), getOmniMaxEstimatedTokens: vi.fn().mockReturnValue(0), storage: { getQwenDir: () => '/tmp/omni-test-qwen' }, } as unknown as Config; @@ -540,9 +540,9 @@ describe('processMediaForOmniDelivery upload cache integration', () => { .fn() .mockReturnValue(overrides?.cgc ?? DASHSCOPE_CGC), getModel: vi.fn().mockReturnValue('qwen3.5-omni-plus'), - getOmniUploadMaxFileBytes: vi.fn().mockReturnValue(0), + getOmniMaxUploadFileBytes: vi.fn().mockReturnValue(0), getOmniMaxEstimatedTokens: vi.fn().mockReturnValue(0), - getOmniUploadCacheTtlHours: vi.fn().mockReturnValue(overrides?.ttlHours), + getOmniUploadUrlTtlHours: vi.fn().mockReturnValue(overrides?.ttlHours), storage: { getQwenDir: () => tmpDir }, } as unknown as Config; } @@ -688,3 +688,1141 @@ describe('processMediaForOmniDelivery upload cache integration', () => { await expect(fs.stat(expired)).rejects.toMatchObject({ code: 'ENOENT' }); }); }); + +describe('processMediaForOmniDelivery fixed-policy integration', () => { + // The orchestrator itself is unit-tested in policy/orchestrator.test.ts; + // these tests pin the pipeline wiring around it: when it runs, what it + // receives, how its output replaces the source, that the transport guard + // judges the FINAL delivery (decision D1), and how failures surface. + let tmpDir: string; + + beforeEach(async () => { + vi.resetModules(); + tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'omni-policy-int-')); + }); + + afterEach(async () => { + vi.resetAllMocks(); + vi.doUnmock('./ffmpeg.js'); + vi.doUnmock('./recognition.js'); + vi.doUnmock('./storage.js'); + vi.doUnmock('./upload.js'); + vi.doUnmock('./policy/orchestrator.js'); + vi.doUnmock('./recovery.js'); + await fs.rm(tmpDir, { recursive: true, force: true }); + }); + + const SOURCE_RECOGNIZED = { + modality: 'image', + detectedMimeType: 'image/png', + sizeBytes: 5000, + metadata: { width: 4000, height: 3000 }, + }; + const DEGRADED_RECOGNIZED = { + modality: 'image', + detectedMimeType: 'image/jpeg', + sizeBytes: 100, + metadata: { width: 1568, height: 1176 }, + }; + // Only `.length > 0` matters to the pipeline; the mocked orchestrator + // never reads the entries. + const POLICY_STUB = [{ id: 'img-downsample' }]; + // Mirrors DEFAULT_OMNI_PROCESSING_LIMITS (normalization is unit-tested + // in policy/config.test.ts; the pipeline dereferences maxTransportPasses + // and forwards the object to the orchestrator). + const LIMITS_STUB = { + maxConcurrentResources: 1, + reservedOutputTokens: 8192, + maxLineageDepth: 8, + maxPolicyRunsPerRoot: 64, + maxArtifactsPerRoot: 256, + maxDerivedBytesPerRoot: 1073741824, + maxTransportPasses: 3, + }; + + function policyConfig(overrides?: { + maxUploadFileBytes?: number; + policies?: unknown[]; + transportGuardPolicies?: unknown[]; + maxTransportPasses?: number; + /** Simulates a stub/embedder config without the accessor. */ + noProcessingConfig?: boolean; + /** Resolved model window for the session.* snapshot. */ + contextWindowSize?: number; + /** Current chat's last prompt token count for the session.* snapshot. */ + lastPromptTokenCount?: number; + }): Config { + return { + isOmniEnabled: vi.fn().mockReturnValue(true), + isTrustedFolder: vi.fn().mockReturnValue(true), + getContentGeneratorConfig: vi.fn().mockReturnValue( + overrides?.contextWindowSize !== undefined + ? { + ...DASHSCOPE_CGC, + contextWindowSize: overrides.contextWindowSize, + } + : DASHSCOPE_CGC, + ), + ...(overrides?.lastPromptTokenCount !== undefined + ? { + getGeminiClient: () => ({ + getChat: () => ({ + getLastPromptTokenCount: () => overrides.lastPromptTokenCount, + }), + }), + } + : {}), + getModel: vi.fn().mockReturnValue('qwen3.5-omni-plus'), + getOmniMaxUploadFileBytes: vi + .fn() + .mockReturnValue(overrides?.maxUploadFileBytes ?? 0), + getOmniMaxEstimatedTokens: vi.fn().mockReturnValue(0), + getOmniProcessingConfig: vi.fn().mockReturnValue( + overrides?.noProcessingConfig + ? undefined + : { + fixedPolicies: overrides?.policies ?? POLICY_STUB, + transportGuardPolicies: overrides?.transportGuardPolicies ?? [], + limits: { + ...LIMITS_STUB, + ...(overrides?.maxTransportPasses !== undefined + ? { maxTransportPasses: overrides.maxTransportPasses } + : {}), + }, + }, + ), + storage: { getQwenDir: () => tmpDir }, + } as unknown as Config; + } + + async function armPipeline(runFixedPoliciesMock: ReturnType) { + const putFileMock = vi + .fn() + .mockResolvedValue({ objectPath: '/tmp/obj.jpg', deduped: false }); + const uploadFileMock = vi.fn().mockResolvedValue('oss://bucket/degraded'); + const hashFileMock = vi.fn().mockResolvedValue('a'.repeat(64)); + vi.doMock('./ffmpeg.js', () => ({ + isFfmpegAvailable: vi.fn().mockResolvedValue(true), + isFfprobeAvailable: vi.fn().mockResolvedValue(true), + })); + vi.doMock('./recognition.js', () => ({ + recognizeMediaFile: vi.fn().mockResolvedValue(SOURCE_RECOGNIZED), + hashFileSha256: hashFileMock, + extensionForMime: vi.fn().mockReturnValue('.jpg'), + })); + const objectsDir = path.join(tmpDir, 'objects'); + vi.doMock('./storage.js', () => ({ + OmniObjectStore: class { + putFile = putFileMock; + getOmniRootDir() { + return tmpDir; + } + getObjectsDir() { + return objectsDir; + } + }, + })); + vi.doMock('./upload.js', () => ({ + DashScopeUploader: class { + uploadFile = uploadFileMock; + }, + OSS_URL_PREFIX: 'oss://', + })); + vi.doMock('./policy/orchestrator.js', () => ({ + runFixedPolicies: runFixedPoliciesMock, + OmniPolicyExecutionError: class extends Error {}, + })); + const mod = await import('./index.js'); + return { putFileMock, uploadFileMock, hashFileMock, mod }; + } + + async function realFile(name: string): Promise { + const filePath = path.join(tmpDir, name); + await fs.writeFile(filePath, 'not really media'); + return filePath; + } + + it('replaces the source with the policy derivative and carries its disclosure', async () => { + const degradedPath = path.join(tmpDir, 'objects', 'deadbeef.jpg'); + const runMock = vi.fn().mockResolvedValue({ + deliveries: [ + { + filePath: degradedPath, + recognized: DEGRADED_RECOGNIZED, + sha256: 'b'.repeat(64), + disclosure: 'downsampled to 1568px', + degraded: true, + }, + ], + records: [], + fileDeliveries: [], + }); + const { putFileMock, hashFileMock, mod } = await armPipeline(runMock); + const filePath = await realFile('pic.png'); + const config = policyConfig(); + + const result = await mod.processMediaForOmniDelivery(filePath, config); + + // The orchestrator received the source resource with user provenance. + expect(runMock).toHaveBeenCalledTimes(1); + expect(runMock).toHaveBeenCalledWith( + config, + { + filePath, + recognized: SOURCE_RECOGNIZED, + displayName: 'pic.png', + origin: 'user', + }, + expect.objectContaining({ policies: POLICY_STUB }), + ); + // Storage/upload operate on the DERIVATIVE under its promotion hash; + // the source is never re-hashed (the derivative arrived with one). + expect(putFileMock).toHaveBeenCalledWith( + degradedPath, + 'b'.repeat(64), + '.jpg', + undefined, + ); + expect(hashFileMock).not.toHaveBeenCalled(); + expect(result.fileUri).toBe('oss://bucket/degraded'); + expect(result.mimeType).toBe('image/jpeg'); + expect(result.sha256).toBe('b'.repeat(64)); + expect(result.recognized).toBe(DEGRADED_RECOGNIZED); + expect(result.disclosure).toBe('downsampled to 1568px'); + expect(result.degraded).toBe(true); + }); + + // ── session.* condition namespace snapshot (policy design §8.3) ─────── + it('threads a stub-config session snapshot (reserved tokens only) into the orchestrator', async () => { + const runMock = vi.fn().mockResolvedValue({ + deliveries: [], + records: [], + fileDeliveries: [], + }); + const { mod } = await armPipeline(runMock); + await expect( + mod.processMediaForOmniDelivery( + await realFile('pic.png'), + policyConfig(), + ), + ).rejects.toThrow(); + // Window size and prompt count are unknown on the stub config: the + // snapshot must carry ONLY the reserved-output limit — absent fields + // read as `unavailable`, never as zero. + expect(runMock.mock.calls[0][2].conditionContext).toEqual({ + session: { reservedOutputTokens: 8192 }, + }); + }); + + it('snapshots the full session namespace once and reuses it for the guard pass', async () => { + const preprocessedPath = path.join(tmpDir, 'objects', 'pre.jpg'); + const guardedPath = path.join(tmpDir, 'objects', 'guarded.jpg'); + const runMock = vi + .fn() + .mockResolvedValueOnce({ + deliveries: [ + { + filePath: preprocessedPath, + recognized: { ...DEGRADED_RECOGNIZED, sizeBytes: 900 }, + sha256: 'b'.repeat(64), + degraded: true, + }, + ], + records: [], + fileDeliveries: [], + }) + .mockResolvedValueOnce({ + deliveries: [ + { + filePath: guardedPath, + recognized: DEGRADED_RECOGNIZED, + sha256: 'c'.repeat(64), + degraded: true, + }, + ], + records: [], + fileDeliveries: [], + }); + const { mod } = await armPipeline(runMock); + await mod.processMediaForOmniDelivery( + await realFile('pic.png'), + policyConfig({ + maxUploadFileBytes: 500, + transportGuardPolicies: [{ id: 'img-guard', mediaTypes: ['image'] }], + contextWindowSize: 131072, + lastPromptTokenCount: 20000, + }), + ); + + expect(runMock).toHaveBeenCalledTimes(2); + expect(runMock.mock.calls[0][2].conditionContext).toEqual({ + session: { + reservedOutputTokens: 8192, + contextWindowTokens: 131072, + promptTokenCount: 20000, + availableContextTokens: 131072 - 20000 - 8192, + }, + }); + // The guard pass receives the SAME snapshot object — taken once per + // delivery, constant across every pass of that delivery. + expect(runMock.mock.calls[1][2].conditionContext).toBe( + runMock.mock.calls[0][2].conditionContext, + ); + }); + + it('skips the orchestrator entirely when no fixed policies are configured', async () => { + const runMock = vi.fn(); + const { mod } = await armPipeline(runMock); + const result = await mod.processMediaForOmniDelivery( + await realFile('pic.png'), + policyConfig({ policies: [] }), + ); + expect(runMock).not.toHaveBeenCalled(); + expect(result.disclosure).toBeUndefined(); + expect(result.degraded).toBeUndefined(); + }); + + it('judges the transport byte guard on the FINAL delivery, not the source', async () => { + // Source 5000 bytes, cap 500: without policies this delivery would be + // rejected. The derivative is 100 bytes — the reordered pipeline (D1) + // must accept it. + const runMock = vi.fn().mockResolvedValue({ + deliveries: [ + { + filePath: path.join(tmpDir, 'objects', 'deadbeef.jpg'), + recognized: DEGRADED_RECOGNIZED, + sha256: 'b'.repeat(64), + disclosure: 'downsampled to 1568px', + degraded: true, + }, + ], + records: [], + fileDeliveries: [], + }); + const { mod } = await armPipeline(runMock); + const result = await mod.processMediaForOmniDelivery( + await realFile('pic.png'), + policyConfig({ maxUploadFileBytes: 500 }), + ); + expect(result.degraded).toBe(true); + }); + + it('explicitly omits an over-cap FINAL delivery when no guard policy matches its modality', async () => { + // Stage B (policy design §10.2): with a processing config present, a + // persisting violation is an explicit OMISSION, not a throw. The audio + // guard policy does not match an image, so no guard pass runs. + const runMock = vi.fn().mockResolvedValue({ + deliveries: [ + { + filePath: path.join(tmpDir, 'objects', 'deadbeef.jpg'), + recognized: { ...DEGRADED_RECOGNIZED, sizeBytes: 900 }, + sha256: 'b'.repeat(64), + degraded: true, + }, + ], + records: [], + fileDeliveries: [], + }); + const { putFileMock, uploadFileMock, mod } = await armPipeline(runMock); + const result = await mod.processMediaForOmniDelivery( + await realFile('pic.png'), + policyConfig({ + maxUploadFileBytes: 500, + transportGuardPolicies: [{ id: 'guard-audio', mediaTypes: ['audio'] }], + }), + ); + // Only the fixed-policy stage ran — never a guard pass. + expect(runMock).toHaveBeenCalledTimes(1); + expect(result).toMatchObject({ + fileUri: '', + sha256: 'b'.repeat(64), + deduped: false, + uploadCacheHit: false, + degraded: true, + }); + expect(result.omission?.reason).toContain('900 bytes > 500 bytes'); + // Nothing was stored or uploaded for an omitted resource. + expect(putFileMock).not.toHaveBeenCalled(); + expect(uploadFileMock).not.toHaveBeenCalled(); + }); + + it('keeps the fail-closed throw when there is no processing config at all', async () => { + // Stub configs / embedders skipping initialize have no normalized + // processing config; the Stage A guard behavior must survive for them. + const runMock = vi.fn(); + const { mod } = await armPipeline(runMock); + await expect( + mod.processMediaForOmniDelivery( + await realFile('pic.png'), + policyConfig({ maxUploadFileBytes: 500, noProcessingConfig: true }), + ), + ).rejects.toMatchObject({ name: 'OmniTransportGuardError' }); + expect(runMock).not.toHaveBeenCalled(); + }); + + it('wraps orchestrator failures into a sanitized OmniDeliveryError', async () => { + const runMock = vi.fn().mockRejectedValue(new Error('policy blew up')); + const { mod } = await armPipeline(runMock); + await expect( + mod.processMediaForOmniDelivery( + await realFile('pic.png'), + policyConfig(), + ), + ).rejects.toMatchObject({ + name: 'OmniDeliveryError', + message: 'Fixed-policy processing failed for pic.png: policy blew up', + }); + }); + + it('rejects a delivery set that is not exactly one resource', async () => { + const runMock = vi + .fn() + .mockResolvedValue({ deliveries: [], records: [], fileDeliveries: [] }); + const { mod } = await armPipeline(runMock); + await expect( + mod.processMediaForOmniDelivery( + await realFile('pic.png'), + policyConfig(), + ), + ).rejects.toMatchObject({ + name: 'OmniDeliveryError', + message: + 'Fixed policies produced 0 media deliverables for pic.png; exactly one is supported.', + }); + }); + + // ── Multi-output fixed policies (#8187 多产物投递) ─────────────────── + const FRAME_2_RECOGNIZED = { + modality: 'image', + detectedMimeType: 'image/jpeg', + sizeBytes: 120, + metadata: { width: 640, height: 360 }, + }; + const FRAME_3_RECOGNIZED = { ...FRAME_2_RECOGNIZED, sizeBytes: 130 }; + + function keyframeDeliveries(tmp: string) { + return [ + { + filePath: path.join(tmp, 'objects', 'frame1.jpg'), + recognized: DEGRADED_RECOGNIZED, + sha256: 'b'.repeat(64), + disclosure: '帧 1/3', + degraded: true, + }, + { + filePath: path.join(tmp, 'objects', 'frame2.jpg'), + recognized: FRAME_2_RECOGNIZED, + sha256: 'd'.repeat(64), + disclosure: '帧 2/3', + degraded: true, + }, + { + filePath: path.join(tmp, 'objects', 'frame3.jpg'), + recognized: FRAME_3_RECOGNIZED, + sha256: 'e'.repeat(64), + disclosure: '帧 3/3', + degraded: true, + }, + ]; + } + + it('uploads every deliverable of a multi-output policy and carries the extras in additionalMedia', async () => { + const runMock = vi.fn().mockResolvedValue({ + deliveries: keyframeDeliveries(tmpDir), + records: [], + fileDeliveries: [], + }); + const { putFileMock, uploadFileMock, hashFileMock, mod } = + await armPipeline(runMock); + uploadFileMock + .mockResolvedValueOnce('oss://bucket/frame1') + .mockResolvedValueOnce('oss://bucket/frame2') + .mockResolvedValueOnce('oss://bucket/frame3'); + + const result = await mod.processMediaForOmniDelivery( + await realFile('vid.mp4'), + policyConfig(), + ); + + // Primary = first deliverable; the rest ride in additionalMedia, in + // orchestrator order, each with its own URL/hash/disclosure. + expect(result.fileUri).toBe('oss://bucket/frame1'); + expect(result.sha256).toBe('b'.repeat(64)); + expect(result.additionalMedia).toEqual([ + { + fileUri: 'oss://bucket/frame2', + mimeType: 'image/jpeg', + sha256: 'd'.repeat(64), + disclosure: '帧 2/3', + }, + { + fileUri: 'oss://bucket/frame3', + mimeType: 'image/jpeg', + sha256: 'e'.repeat(64), + disclosure: '帧 3/3', + }, + ]); + // Every deliverable went through the SAME store→upload pipeline. + expect(putFileMock).toHaveBeenCalledTimes(3); + expect(uploadFileMock).toHaveBeenCalledTimes(3); + // All arrived with promotion hashes — nothing is re-hashed. + expect(hashFileMock).not.toHaveBeenCalled(); + }); + + it('explicitly omits an over-cap ADDITIONAL deliverable while the rest deliver', async () => { + const deliveries = keyframeDeliveries(tmpDir); + deliveries[1] = { + ...deliveries[1], + recognized: { ...FRAME_2_RECOGNIZED, sizeBytes: 900 }, + }; + const runMock = vi.fn().mockResolvedValue({ + deliveries, + records: [], + fileDeliveries: [], + }); + const { putFileMock, uploadFileMock, mod } = await armPipeline(runMock); + + const result = await mod.processMediaForOmniDelivery( + await realFile('vid.mp4'), + policyConfig({ maxUploadFileBytes: 500 }), + ); + + // The violating extra becomes an explicit omission entry (policy + // design §10.2 — no re-derivation of derivatives); its neighbors and + // the primary are unaffected. + expect(result.fileUri).toBe('oss://bucket/degraded'); + expect(result.additionalMedia).toHaveLength(2); + expect(result.additionalMedia![0]).toMatchObject({ + fileUri: '', + sha256: 'd'.repeat(64), + disclosure: '帧 2/3', + }); + expect(result.additionalMedia![0].omission?.reason).toContain( + '900 bytes > 500 bytes', + ); + expect(result.additionalMedia![1]).toMatchObject({ + fileUri: 'oss://bucket/degraded', + sha256: 'e'.repeat(64), + }); + // The omitted extra never touched the store or the upload channel. + expect(putFileMock).toHaveBeenCalledTimes(2); + expect(uploadFileMock).toHaveBeenCalledTimes(2); + }); + + it('readMediaViaOmniDelivery materializes extras as [disclosure, fileData] pairs after the primary and before transcripts', async () => { + const runMock = vi.fn().mockResolvedValue({ + deliveries: keyframeDeliveries(tmpDir), + records: [], + fileDeliveries: [ + { + filePath: '/tmp/objects/t.txt', + role: 'transcript', + mimeType: 'text/plain', + text: '你好，世界', + sha256: 'c'.repeat(64), + sizeBytes: 15, + }, + ], + }); + const { uploadFileMock, mod } = await armPipeline(runMock); + uploadFileMock + .mockResolvedValueOnce('oss://bucket/frame1') + .mockResolvedValueOnce('oss://bucket/frame2') + .mockResolvedValueOnce('oss://bucket/frame3'); + + const result = await mod.readMediaViaOmniDelivery({ + filePath: await realFile('vid.mp4'), + config: policyConfig(), + displayName: 'vid.mp4', + relativePathForDisplay: 'vid.mp4', + expectedModality: 'video', + }); + + const parts = result.llmContent as Array>; + // [zoom hint, primary disclosure, primary fileData, + // extra1 disclosure, extra1 fileData, extra2 disclosure, + // extra2 fileData, transcript] — D8 adjacency per pair, transcripts + // last. + expect(parts.map((p) => ('fileData' in p ? 'media' : 'text'))).toEqual([ + 'text', + 'text', + 'media', + 'text', + 'media', + 'text', + 'media', + 'text', + ]); + expect(parts[3]!['text']).toBe('【媒体降质】vid.mp4：帧 2/3'); + expect(parts[4]).toEqual({ + fileData: { + fileUri: 'oss://bucket/frame2', + mimeType: 'image/jpeg', + displayName: 'vid.mp4', + }, + }); + expect(parts[5]!['text']).toBe('【媒体降质】vid.mp4：帧 3/3'); + expect(parts[6]).toEqual({ + fileData: { + fileUri: 'oss://bucket/frame3', + mimeType: 'image/jpeg', + displayName: 'vid.mp4', + }, + }); + expect(parts[7]!['text']).toBe('【媒体转写】vid.mp4：你好，世界'); + }); + + it('readMediaViaOmniDelivery still materializes extras when the PRIMARY is omitted', async () => { + const deliveries = keyframeDeliveries(tmpDir).slice(0, 2); + deliveries[0] = { + ...deliveries[0], + recognized: { ...DEGRADED_RECOGNIZED, sizeBytes: 900 }, + }; + const runMock = vi.fn().mockResolvedValue({ + deliveries, + records: [], + fileDeliveries: [], + }); + const { mod } = await armPipeline(runMock); + + const result = await mod.readMediaViaOmniDelivery({ + filePath: await realFile('vid.mp4'), + config: policyConfig({ maxUploadFileBytes: 500 }), + displayName: 'vid.mp4', + relativePathForDisplay: 'vid.mp4', + expectedModality: 'video', + }); + + const parts = result.llmContent as Array>; + expect(parts).toHaveLength(3); + expect(parts[0]!['text']).toContain('【媒体省略】vid.mp4'); + expect(parts[1]!['text']).toBe('【媒体降质】vid.mp4：帧 2/3'); + expect(parts[2]).toEqual({ + fileData: { + fileUri: 'oss://bucket/degraded', + mimeType: 'image/jpeg', + displayName: 'vid.mp4', + }, + }); + }); + + it('readMediaViaOmniDelivery places the disclosure immediately before the fileData part', async () => { + const runMock = vi.fn().mockResolvedValue({ + deliveries: [ + { + filePath: path.join(tmpDir, 'objects', 'deadbeef.jpg'), + recognized: DEGRADED_RECOGNIZED, + sha256: 'b'.repeat(64), + disclosure: 'downsampled to 1568px', + degraded: true, + }, + ], + records: [], + fileDeliveries: [], + }); + const { mod } = await armPipeline(runMock); + const result = await mod.readMediaViaOmniDelivery({ + filePath: await realFile('pic.png'), + config: policyConfig(), + displayName: 'pic.png', + relativePathForDisplay: 'pic.png', + expectedModality: 'image', + }); + const parts = result.llmContent as Array>; + expect(parts).toHaveLength(3); + // Zoom hint shows the DELIVERED image's resolution (the derivative) — + // and must not call it "full resolution", which would contradict the + // degradation disclosure right below and steer the model away from + // zoom_image (the remedy that reads the original from disk). + expect(parts[0]!['text']).toContain('delivered at 1568x1176 px'); + expect(parts[0]!['text']).toContain('after degradation'); + expect(parts[0]!['text']).not.toContain('full resolution'); + expect(parts[1]!['text']).toBe( + '【媒体降质】pic.png：downsampled to 1568px', + ); + expect(parts[2]).toEqual({ + fileData: { + fileUri: 'oss://bucket/degraded', + mimeType: 'image/jpeg', + displayName: 'pic.png', + }, + }); + }); + + // ── Transcript delivery (§6.2) ──────────────────────────────────────── + const TRANSCRIPT_FILE_DELIVERY = { + filePath: '/tmp/objects/t.txt', + role: 'transcript', + mimeType: 'text/plain', + text: '你好，世界', + sha256: 'c'.repeat(64), + sizeBytes: 15, + disclosure: '原 63s 音频 → 转写文本 5 字', + }; + + it('returns a pure-transcript delivery without storing or uploading anything', async () => { + const runMock = vi.fn().mockResolvedValue({ + deliveries: [], + records: [], + fileDeliveries: [TRANSCRIPT_FILE_DELIVERY], + }); + const { putFileMock, uploadFileMock, mod } = await armPipeline(runMock); + const result = await mod.processMediaForOmniDelivery( + await realFile('pic.png'), + policyConfig(), + ); + + // No media deliverable → nothing enters objects/ or the upload channel. + expect(putFileMock).not.toHaveBeenCalled(); + expect(uploadFileMock).not.toHaveBeenCalled(); + expect(result.fileUri).toBe(''); + expect(result.sha256).toBe(''); + expect(result.mimeType).toBe('image/png'); + expect(result.degraded).toBe(true); + expect(result.transcripts).toEqual([ + { text: '你好，世界', disclosure: '原 63s 音频 → 转写文本 5 字' }, + ]); + }); + + it('threads transcripts alongside a media deliverable into the upload result', async () => { + const runMock = vi.fn().mockResolvedValue({ + deliveries: [ + { + filePath: path.join(tmpDir, 'objects', 'deadbeef.jpg'), + recognized: DEGRADED_RECOGNIZED, + sha256: 'b'.repeat(64), + disclosure: 'downsampled to 1568px', + degraded: true, + }, + ], + records: [], + fileDeliveries: [TRANSCRIPT_FILE_DELIVERY], + }); + const { mod } = await armPipeline(runMock); + const result = await mod.processMediaForOmniDelivery( + await realFile('pic.png'), + policyConfig(), + ); + expect(result.fileUri).toBe('oss://bucket/degraded'); + expect(result.transcripts).toEqual([ + { text: '你好，世界', disclosure: '原 63s 音频 → 转写文本 5 字' }, + ]); + }); + + it('readMediaViaOmniDelivery renders a pure-transcript delivery as text parts only', async () => { + const runMock = vi.fn().mockResolvedValue({ + deliveries: [], + records: [], + fileDeliveries: [TRANSCRIPT_FILE_DELIVERY], + }); + const { mod } = await armPipeline(runMock); + const result = await mod.readMediaViaOmniDelivery({ + filePath: await realFile('pic.png'), + config: policyConfig(), + displayName: 'pic.png', + relativePathForDisplay: 'pic.png', + expectedModality: 'image', + }); + // Disclosure precedes its transcript (D8 adjacency); no fileData part. + expect(result.llmContent).toEqual([ + { text: '【媒体降质】pic.png：原 63s 音频 → 转写文本 5 字' }, + { text: '【媒体转写】pic.png：你好，世界' }, + ]); + expect(result.returnDisplay).toBe( + 'Read image as transcript (omni policy): pic.png', + ); + expect(result.error).toBeUndefined(); + }); + + it('readMediaViaOmniDelivery appends transcript parts after the media part', async () => { + const runMock = vi.fn().mockResolvedValue({ + deliveries: [ + { + filePath: path.join(tmpDir, 'objects', 'deadbeef.jpg'), + recognized: DEGRADED_RECOGNIZED, + sha256: 'b'.repeat(64), + disclosure: 'downsampled to 1568px', + degraded: true, + }, + ], + records: [], + fileDeliveries: [TRANSCRIPT_FILE_DELIVERY], + }); + const { mod } = await armPipeline(runMock); + const result = await mod.readMediaViaOmniDelivery({ + filePath: await realFile('pic.png'), + config: policyConfig(), + displayName: 'pic.png', + relativePathForDisplay: 'pic.png', + expectedModality: 'image', + }); + const parts = result.llmContent as Array>; + expect(parts).toHaveLength(5); + expect(parts[0]!['text']).toContain('1568x1176'); // zoom hint + expect(parts[1]!['text']).toBe( + '【媒体降质】pic.png：downsampled to 1568px', + ); + expect(parts[2]!['fileData']).toBeDefined(); + expect(parts[3]!['text']).toBe( + '【媒体降质】pic.png：原 63s 音频 → 转写文本 5 字', + ); + expect(parts[4]!['text']).toBe('【媒体转写】pic.png：你好，世界'); + }); + + it('readMediaViaOmniDelivery keeps transcripts when the media itself is omitted', async () => { + const runMock = vi.fn().mockResolvedValue({ + deliveries: [ + { + filePath: path.join(tmpDir, 'objects', 'deadbeef.jpg'), + recognized: { ...DEGRADED_RECOGNIZED, sizeBytes: 900 }, + sha256: 'b'.repeat(64), + degraded: true, + }, + ], + records: [], + fileDeliveries: [TRANSCRIPT_FILE_DELIVERY], + }); + const { mod } = await armPipeline(runMock); + const result = await mod.readMediaViaOmniDelivery({ + filePath: await realFile('pic.png'), + config: policyConfig({ maxUploadFileBytes: 500 }), + displayName: 'pic.png', + relativePathForDisplay: 'pic.png', + expectedModality: 'image', + }); + const parts = result.llmContent as Array>; + expect(parts).toHaveLength(3); + expect(parts[0]!['text']).toMatch(/^【媒体省略】pic\.png：/); + expect(parts[1]!['text']).toBe( + '【媒体降质】pic.png：原 63s 音频 → 转写文本 5 字', + ); + expect(parts[2]!['text']).toBe('【媒体转写】pic.png：你好，世界'); + expect(result.error).toBeUndefined(); + }); + + // ── Stage B transport-guard pass loop ──────────────────────────────── + // With `policies: []` the fixed-policy stage is skipped entirely, so + // every runFixedPolicies call in these tests is a GUARD pass on the + // 5000-byte source (cap 500 → violation). + const IMG_GUARD = { id: 'img-guard', mediaTypes: ['image'] }; + + it('runs a matching guard policy on a violation and delivers the compliant result', async () => { + const guardedPath = path.join(tmpDir, 'objects', 'guarded.jpg'); + const runMock = vi.fn().mockResolvedValue({ + deliveries: [ + { + filePath: guardedPath, + recognized: DEGRADED_RECOGNIZED, + sha256: 'b'.repeat(64), + disclosure: 'downsampled to 1568px', + degraded: true, + }, + ], + records: [], + fileDeliveries: [], + }); + const { mod } = await armPipeline(runMock); + const filePath = await realFile('pic.png'); + const config = policyConfig({ + policies: [], + maxUploadFileBytes: 500, + transportGuardPolicies: [IMG_GUARD], + }); + + const result = await mod.processMediaForOmniDelivery(filePath, config); + + // One guard pass over the SOURCE, restricted to the matching policies. + expect(runMock).toHaveBeenCalledTimes(1); + expect(runMock).toHaveBeenCalledWith( + config, + { + filePath, + recognized: SOURCE_RECOGNIZED, + displayName: 'pic.png', + origin: 'user', + }, + expect.objectContaining({ + policies: [IMG_GUARD], + limits: expect.objectContaining({ maxTransportPasses: 3 }), + }), + ); + expect(result.fileUri).toBe('oss://bucket/degraded'); + expect(result.omission).toBeUndefined(); + expect(result.degraded).toBe(true); + expect(result.disclosure).toBe('downsampled to 1568px'); + }); + + it('chains preprocessing and guard disclosures instead of replacing (D8)', async () => { + // Preprocessing degrades once (disclosure A) but the derivative is + // still over the byte cap; the guard degrades AGAIN (disclosure B). + // Both lossy steps must reach the model — a replaced disclosure would + // silently hide the first degradation. + const preprocessedPath = path.join(tmpDir, 'objects', 'pre.jpg'); + const guardedPath = path.join(tmpDir, 'objects', 'guarded.jpg'); + const runMock = vi + .fn() + .mockResolvedValueOnce({ + deliveries: [ + { + filePath: preprocessedPath, + recognized: { ...DEGRADED_RECOGNIZED, sizeBytes: 900 }, + sha256: 'b'.repeat(64), + disclosure: 'downsampled to 1568px', + degraded: true, + }, + ], + records: [], + fileDeliveries: [], + }) + .mockResolvedValueOnce({ + deliveries: [ + { + filePath: guardedPath, + recognized: DEGRADED_RECOGNIZED, + sha256: 'c'.repeat(64), + disclosure: 're-encoded at quality 60', + degraded: true, + }, + ], + records: [], + fileDeliveries: [], + }); + const { mod } = await armPipeline(runMock); + const result = await mod.processMediaForOmniDelivery( + await realFile('pic.png'), + policyConfig({ + maxUploadFileBytes: 500, + transportGuardPolicies: [IMG_GUARD], + }), + ); + + expect(runMock).toHaveBeenCalledTimes(2); + // Guard pass ran on the PREPROCESSED derivative, not the source. + expect(runMock.mock.calls[1][1]).toMatchObject({ + filePath: preprocessedPath, + }); + expect(result.omission).toBeUndefined(); + expect(result.disclosure).toBe( + 'downsampled to 1568px；re-encoded at quality 60', + ); + expect(result.degraded).toBe(true); + expect(result.sha256).toBe('c'.repeat(64)); + }); + + it('stops after maxTransportPasses passes and omits when still violating', async () => { + let call = 0; + const runMock = vi.fn().mockImplementation(async () => { + call += 1; + return { + deliveries: [ + { + // A NEW path every pass: progress is being made, so only the + // pass counter can end the loop. + filePath: path.join(tmpDir, 'objects', `pass-${call}.jpg`), + recognized: { ...DEGRADED_RECOGNIZED, sizeBytes: 900 }, + sha256: String(call).repeat(64).slice(0, 64), + degraded: true, + }, + ], + records: [], + fileDeliveries: [], + }; + }); + const { mod } = await armPipeline(runMock); + const result = await mod.processMediaForOmniDelivery( + await realFile('pic.png'), + policyConfig({ + policies: [], + maxUploadFileBytes: 500, + maxTransportPasses: 2, + transportGuardPolicies: [IMG_GUARD], + }), + ); + expect(runMock).toHaveBeenCalledTimes(2); + expect(result.omission?.reason).toContain('900 bytes > 500 bytes'); + expect(result.fileUri).toBe(''); + }); + + it('breaks out of the guard loop when a pass makes no progress', async () => { + // Every guard policy no_op'd: the delivery IS the input resource. A + // second pass would repeat identical work forever. + const runMock = vi.fn().mockImplementation(async (_config, resource) => ({ + deliveries: [ + { filePath: resource.filePath, recognized: resource.recognized }, + ], + records: [], + fileDeliveries: [], + })); + const { mod } = await armPipeline(runMock); + const result = await mod.processMediaForOmniDelivery( + await realFile('pic.png'), + policyConfig({ + policies: [], + maxUploadFileBytes: 500, + transportGuardPolicies: [IMG_GUARD], + }), + ); + expect(runMock).toHaveBeenCalledTimes(1); + expect(result.omission?.reason).toContain('5000 bytes > 500 bytes'); + }); + + it('fails closed when a guard pass itself fails', async () => { + // A guard configuration error must never degrade into sending + // over-limit media (policy design §10.2). The error class matters: + // OmniTransportGuardError is what tells consumers with an inline + // fallback (the tool-result funnel) to WITHHOLD the bytes — a generic + // delivery error would fall back to delivering exactly what the guard + // rejected. + const runMock = vi.fn().mockRejectedValue(new Error('guard blew up')); + const { mod } = await armPipeline(runMock); + await expect( + mod.processMediaForOmniDelivery( + await realFile('pic.png'), + policyConfig({ + policies: [], + maxUploadFileBytes: 500, + transportGuardPolicies: [IMG_GUARD], + }), + ), + ).rejects.toMatchObject({ + name: 'OmniTransportGuardError', + message: 'Transport-guard processing failed for pic.png: guard blew up', + }); + }); + + it('re-filters guard policies by modality after a pass changes it', async () => { + // Pass 1 transforms the over-limit image into an over-limit AUDIO + // derivative (modality change); pass 2 must run the AUDIO guard policy + // against it. A pre-loop filter (image only) would find no matching + // policy and omit a resource the audio policy can still fix. + const imageGuard = { id: 'img-guard', mediaTypes: ['image'] }; + const audioGuard = { id: 'audio-guard', mediaTypes: ['audio'] }; + const bigAudioPath = path.join(tmpDir, 'objects', 'audio-big.mp3'); + await fs.mkdir(path.dirname(bigAudioPath), { recursive: true }); + await fs.writeFile(bigAudioPath, Buffer.alloc(900)); + const smallAudioPath = path.join(tmpDir, 'objects', 'audio-small.mp3'); + await fs.writeFile(smallAudioPath, Buffer.alloc(400)); + const audioRecognized = (sizeBytes: number) => ({ + modality: 'audio', + detectedMimeType: 'audio/mpeg', + sizeBytes, + metadata: { durationMs: 60000 }, + }); + const runMock = vi + .fn() + .mockResolvedValueOnce({ + deliveries: [ + { + filePath: bigAudioPath, + recognized: audioRecognized(900), + sha256: 'c'.repeat(64), + degraded: true, + }, + ], + records: [], + fileDeliveries: [], + }) + .mockResolvedValueOnce({ + deliveries: [ + { + filePath: smallAudioPath, + recognized: audioRecognized(400), + sha256: 'd'.repeat(64), + degraded: true, + }, + ], + records: [], + fileDeliveries: [], + }); + const { mod } = await armPipeline(runMock); + const result = await mod.processMediaForOmniDelivery( + await realFile('pic.png'), + policyConfig({ + policies: [], + maxUploadFileBytes: 500, + transportGuardPolicies: [imageGuard, audioGuard], + maxTransportPasses: 3, + }), + ); + expect(runMock).toHaveBeenCalledTimes(2); + // Pass 1 ran the image policy set; pass 2 must have run the AUDIO set. + expect(runMock.mock.calls[0][2].policies).toEqual([imageGuard]); + expect(runMock.mock.calls[1][2].policies).toEqual([audioGuard]); + expect(result.omission).toBeUndefined(); + expect(result.fileUri).not.toBe(''); + }); + + it('readMediaViaOmniDelivery renders an omission as the notice text, not an error', async () => { + const runMock = vi.fn().mockResolvedValue({ + deliveries: [ + { + filePath: path.join(tmpDir, 'objects', 'deadbeef.jpg'), + recognized: { ...DEGRADED_RECOGNIZED, sizeBytes: 900 }, + sha256: 'b'.repeat(64), + degraded: true, + }, + ], + records: [], + fileDeliveries: [], + }); + const { mod } = await armPipeline(runMock); + const result = await mod.readMediaViaOmniDelivery({ + filePath: await realFile('pic.png'), + config: policyConfig({ maxUploadFileBytes: 500 }), + displayName: 'pic.png', + relativePathForDisplay: 'pic.png', + expectedModality: 'image', + }); + expect(typeof result.llmContent).toBe('string'); + expect(result.llmContent).toMatch(/^【媒体省略】pic\.png：/); + expect(result.llmContent).toContain('900 bytes > 500 bytes'); + expect(result.returnDisplay).toBe( + 'Media omitted by the omni transport guard: pic.png', + ); + expect(result.error).toBeUndefined(); + expect(result.errorType).toBeUndefined(); + }); + + it('threads the quarantine retention settings into startup recovery', async () => { + const recoveryMock = vi.fn().mockResolvedValue(undefined); + vi.doMock('./recovery.js', () => ({ + runStartupRecoveryOnce: recoveryMock, + resetRecoveryLatchForTests: vi.fn(), + })); + const runMock = vi.fn().mockResolvedValue({ + deliveries: [ + { + filePath: path.join(tmpDir, 'objects', 'deadbeef.jpg'), + recognized: DEGRADED_RECOGNIZED, + sha256: 'b'.repeat(64), + }, + ], + records: [], + fileDeliveries: [], + }); + const { mod } = await armPipeline(runMock); + const config = { + ...policyConfig(), + getOmniQuarantineRetentionDays: () => 3, + getOmniQuarantineMaxBytes: () => 1024, + } as unknown as Config; + + await mod.processMediaForOmniDelivery(await realFile('pic.png'), config); + + expect(recoveryMock).toHaveBeenCalledWith( + expect.anything(), + expect.anything(), + { + quarantineRetentionDays: 3, + quarantineMaxBytes: 1024, + // Corrupt-object deletion cascades into the degradation cache. + // (Structural match: armPipeline's fresh module graph makes the + // class identity differ from this file's static import.) + degradationCache: expect.objectContaining({ + removeByOriginalSha256: expect.any(Function), + removeByDegradedSha256: expect.any(Function), + }), + }, + ); + }); +}); diff --git a/packages/core/src/omni/index.ts b/packages/core/src/omni/index.ts index b7fbca799df..dcc3055c8a6 100644 --- a/packages/core/src/omni/index.ts +++ b/packages/core/src/omni/index.ts @@ -8,17 +8,19 @@ import { createHash } from 'node:crypto'; import path from 'node:path'; import fs from 'node:fs/promises'; import type { Config } from '../config/config.js'; -import { AuthType } from '../core/contentGenerator.js'; -import { DashScopeOpenAICompatibleProvider } from '../core/openaiContentGenerator/provider/dashscope.js'; import { ToolErrorType } from '../tools/tool-error.js'; import { createDebugLogger } from '../utils/debugLogger.js'; import { isAbortError } from '../utils/errors.js'; import { isFfmpegAvailable, isFfprobeAvailable } from './ffmpeg.js'; -import type { OmniTokenEstimate } from './estimation.js'; +import { + estimateRawResourceTokens, + type OmniTokenEstimate, +} from './estimation.js'; import { assertWithinByteLimit, assertWithinTokenLimit, effectiveMaxUploadFileBytes, + OmniTransportGuardError, } from './guard.js'; import { extensionForMime, @@ -34,6 +36,19 @@ import { DEFAULT_UPLOAD_CACHE_TTL_HOURS, } from './upload-cache.js'; import { runStartupRecoveryOnce } from './recovery.js'; +import { OmniDegradationCache } from './policy/degradation-cache.js'; +import { + formatDisclosureText, + formatOmissionText, + formatTranscriptText, +} from './disclosure.js'; +import { + runFixedPolicies, + type PolicyDeliveryResource, + type PolicyFileDelivery, +} from './policy/orchestrator.js'; +import { buildSessionConditionNamespace } from './policy/session-context.js'; +import type { OmniProcessingConfigView } from './policy/types.js'; export { assertOmniRuntimeDependencies, @@ -78,6 +93,27 @@ export { resetRecoveryLatchForTests, } from './recovery.js'; export { resetCredentialCacheForTests } from './upload.js'; +export { + OMNI_DISCLOSURE_TEXT_PREFIX, + OMNI_OMISSION_TEXT_PREFIX, + OMNI_TRANSCRIPT_TEXT_PREFIX, + formatDisclosureText, + formatOmissionText, + formatTranscriptText, + isDisclosureText, +} from './disclosure.js'; +export { + runFixedPolicies, + OmniPolicyExecutionError, + type PolicyDeliveryResource, + type PolicyFileDelivery, + type PolicyRunRecord, +} from './policy/orchestrator.js'; +export type { + FixedPolicyOrigin, + NormalizedFixedPolicy, + NormalizedOmniProcessingConfig, +} from './policy/types.js'; const debugLogger = createDebugLogger('omni'); @@ -114,14 +150,6 @@ export function sanitizeErrorMessage( ); } -/** - * Placeholder the model-config resolver assigns under Qwen OAuth; the real - * token is swapped in per-request by QwenContentGenerator and never lands - * in the ContentGeneratorConfig, so it cannot authenticate the uploads - * endpoint. See modelConfigResolver.ts. - */ -const QWEN_OAUTH_PLACEHOLDER_API_KEY = 'QWEN_OAUTH_DYNAMIC_TOKEN'; - /** Result of the omni media delivery pipeline. */ export interface OmniMediaDelivery { /** `oss://…` URL to place in fileData.fileUri. */ @@ -140,6 +168,106 @@ export interface OmniMediaDelivery { /** True when the oss URL came from the persistent upload cache (no * network transfer happened for this delivery). */ uploadCacheHit: boolean; + /** Disclosure text that must accompany the media Part (present iff the + * delivered content is a lossy policy derivative). */ + disclosure?: string; + /** True when a fixed policy replaced the source with a lossy + * derivative. */ + degraded?: boolean; + /** Present when the transport guard could not bring the resource within + * limits even after the transport-guard policies ran: the media was NOT + * uploaded (`fileUri` is empty) and callers must materialize an + * explicit-omission text Part in its place (policy design §10.2). */ + omission?: { reason: string }; + /** Transcript-protocol text deliverables (upstream P §6.2): file + * artifacts (`kind:'file'`, `metadata.omniRole:'transcript'`) produced + * by fixed policies and selected for delivery. They travel as text Parts + * after the media Part — or stand alone when the policies omitted the + * media entirely (`fileUri` is empty and `omission` is absent). */ + transcripts?: Array<{ text: string; disclosure?: string }>; + /** Media deliverables beyond the primary one — present when a + * multi-output fixed policy (e.g. `omni_extract_keyframes`) produced + * more than one media derivative for this source (#8187 多产物投递). + * Each entry was uploaded through the same store/upload pipeline as + * the primary; an entry that violated the transport limits carries + * `omission` instead of a usable `fileUri` (policy design §10.2 — + * additional derivatives are already policy products, so a violating + * one is withheld rather than re-derived). Callers materialize each + * entry as [disclosure?, fileData] (or the omission notice) after the + * primary media Part and before any transcripts. */ + additionalMedia?: OmniAdditionalMediaDelivery[]; +} + +/** One extra media deliverable of a multi-output fixed policy. */ +export interface OmniAdditionalMediaDelivery { + /** `oss://…` URL; empty iff `omission` is present. */ + fileUri: string; + mimeType: string; + sha256: string; + /** Disclosure text that must immediately precede this media Part. */ + disclosure?: string; + /** Present when the transport limits rejected this deliverable: it was + * NOT uploaded and an explicit-omission text Part stands in its place. */ + omission?: { reason: string }; +} + +/** A Part materialized from an additional media deliverable. */ +export type OmniAdditionalMediaPart = + | { text: string } + | { fileData: { fileUri: string; mimeType: string; displayName: string } }; + +/** + * Materialize `additionalMedia` as Parts — shared by every delivery + * consumer (fileUtils read results, tool-result funnels, the @url funnel) + * so the multi-output contract has a single shape: per extra + * [disclosure?, fileData] (or [disclosure?, omission text]), placed after + * the primary media Part (or its omission/transcript stand-in) and before + * any transcript Parts. D8 adjacency applies to each pair independently. + */ +export function buildAdditionalMediaParts( + displayName: string, + additionalMedia: OmniAdditionalMediaDelivery[] | undefined, +): OmniAdditionalMediaPart[] { + const parts: OmniAdditionalMediaPart[] = []; + for (const extra of additionalMedia ?? []) { + if (extra.disclosure) { + parts.push({ text: formatDisclosureText(displayName, extra.disclosure) }); + } + if (extra.omission) { + parts.push({ + text: formatOmissionText(displayName, extra.omission.reason), + }); + } else { + parts.push({ + fileData: { + fileUri: extra.fileUri, + mimeType: extra.mimeType, + displayName, + }, + }); + } + } + return parts; +} + +/** + * Materialize transcript deliverables (§6.2) as text Parts — shared by + * every delivery consumer: each transcript follows its media Part (or the + * omission notice), preceded by its own disclosure (same D8 adjacency + * contract as media disclosures). + */ +export function buildTranscriptParts( + displayName: string, + transcripts: Array<{ text: string; disclosure?: string }> | undefined, +): Array<{ text: string }> { + const parts: Array<{ text: string }> = []; + for (const t of transcripts ?? []) { + if (t.disclosure) { + parts.push({ text: formatDisclosureText(displayName, t.disclosure) }); + } + parts.push({ text: formatTranscriptText(displayName, t.text) }); + } + return parts; } /** Thrown for omni pipeline failures. The pipeline fails closed: callers @@ -153,65 +281,52 @@ export class OmniDeliveryError extends Error { } } -/** - * Gate for the omni delivery path. All conditions must hold: - * - * 1. omni enabled (settings or QWEN_CODE_ENABLE_OMNI=1); - * 2. trusted workspace (the pipeline writes .qwen/omni/ and uploads - * workspace bytes off-machine); - * 3. a usable API key for the uploads endpoint — Qwen OAuth is excluded: - * its ContentGeneratorConfig carries a placeholder, and the OAuth token - * is not accepted by the uploads channel; - * 4. an explicit baseUrl (the uploads origin is derived from it — never - * send the configured credential to an origin the user didn't set); - * 5. a DashScope-compatible provider. - * - * Any failed condition falls back to the pre-omni inline behavior. - * Modality support is checked by the caller (fileUtils) alongside the - * existing modality gate. - */ -export function isOmniDeliveryActive(config: Config): boolean { - // Optional calls so stub Configs in tests (and embedders constructing - // partial configs) don't need the omni accessors to process files. - if (!config.isOmniEnabled?.()) return false; - if (config.isTrustedFolder?.() === false) { - debugLogger.debug('omni delivery inactive: untrusted workspace'); - return false; - } - const cgc = config.getContentGeneratorConfig?.(); - if (!cgc) return false; - if ( - cgc.authType === AuthType.QWEN_OAUTH || - !cgc.apiKey || - cgc.apiKey === QWEN_OAUTH_PLACEHOLDER_API_KEY - ) { - debugLogger.debug( - 'omni delivery inactive: no static API key usable for the uploads endpoint (Qwen OAuth is not supported)', - ); - return false; - } - if (!cgc.baseUrl) { - debugLogger.debug( - 'omni delivery inactive: no explicit baseUrl to derive the uploads origin from', - ); - return false; +export { isOmniDeliveryActive } from './delivery-gate.js'; + +/** Non-throwing transport-limit check: runs both guard dimensions and + * reports the first violation as a message instead of an exception, so + * the Stage B guard loop can react (run guard policies / omit) while + * configs without a processing config keep the fail-closed throw. */ +function evaluateTransportLimits( + config: Config, + recognized: RecognizedMedia, + displayName: string, +): { estimate: OmniTokenEstimate; violation?: string } { + try { + assertWithinByteLimit(config, recognized.sizeBytes, displayName); + return { + estimate: assertWithinTokenLimit(config, recognized, displayName), + }; + } catch (err) { + if (err instanceof OmniTransportGuardError) { + return { + estimate: estimateRawResourceTokens(recognized), + violation: err.message, + }; + } + throw err; } - return DashScopeOpenAICompatibleProvider.isDashScopeProvider(cgc); } /** - * Omni pipeline: recognize → transport guard → hash → promote into the - * content-addressed store → upload via the DashScope temporary channel → - * return the oss:// URL plus the token estimate. + * Omni pipeline: recognize → fixed policies (degradation) → transport + * guard → hash → promote into the content-addressed store → upload via the + * DashScope temporary channel → return the oss:// URL plus the token + * estimate. * - * All modalities are uploaded AS-IS — no resizing, no transcoding. - * Degradation is the job of S4 policies (which must disclose); the default - * path never silently alters content. Successful uploads are remembered in - * the persistent upload cache (`.qwen/omni/upload-cache.json`, keyed by - * sha256 + model + endpoint scope) for the oss URL validity window, so a - * re-read of unchanged content skips both the store copy and the network - * transfer. Throws OmniDeliveryError / OmniTransportGuardError on failure; - * user aborts propagate untouched. + * The default pipeline never SILENTLY alters content: any degradation is + * performed by configured fixed policies through real media-policy tools, + * and every lossy derivative carries a mandatory disclosure (decision D8) + * that reaches the model next to the media Part. The transport guard runs + * AFTER the policies (decision D1) so it judges what is actually delivered + * — an oversized original that a policy shrank must pass, and a policy + * failure leaves the guard as the backstop. Successful uploads are + * remembered in the persistent upload cache + * (`.qwen/omni/upload-cache.json`, keyed by sha256 + model + endpoint + * scope) for the oss URL validity window, so a re-read of unchanged + * content skips both the store copy and the network transfer. Throws + * OmniDeliveryError / OmniTransportGuardError on failure; user aborts + * propagate untouched. */ export async function processMediaForOmniDelivery( filePath: string, @@ -226,6 +341,9 @@ export async function processMediaForOmniDelivery( * user-recognizable name instead. */ displayName?: string; + /** Provenance for fixed-policy origin matching. Defaults to 'user'; + * the tool-result funnel passes 'tool'. */ + origin?: 'user' | 'tool'; }, ): Promise { const { expectedModality, signal } = options ?? {}; @@ -243,15 +361,15 @@ export async function processMediaForOmniDelivery( ); } - // Byte guard from a cheap stat BEFORE hashing/probing — a 60GB capture - // must not stream through SHA-256 only to be rejected. - const stat = await fs.stat(filePath).catch((err) => { + // Existence pre-check with a clean caller-facing error (recognition + // failures on a missing file read worse). The byte guard is NOT applied + // here anymore — it judges the post-policy delivery set below. + await fs.stat(filePath).catch((err) => { throw new OmniDeliveryError( `Cannot stat media file ${displayName}: ${sanitizeErrorMessage(err, [filePath])}`, { cause: err }, ); }); - assertWithinByteLimit(config, stat.size, displayName); let recognized: RecognizedMedia; try { @@ -267,23 +385,6 @@ export async function processMediaForOmniDelivery( ); } - // Token guard AFTER probe (needs metadata), BEFORE hash/copy/upload — a - // token-oversized input must not pay a full-file SHA-256 to be rejected. - const tokenEstimate = assertWithinTokenLimit(config, recognized, displayName); - - // Content hash: identity of the stored object. Computed only once all - // guards have passed, immediately before promotion into the store. - let sha256: string; - try { - sha256 = await hashFileSha256(filePath, signal); - } catch (err) { - if (signal?.aborted) throw err; - throw new OmniDeliveryError( - `Failed to hash media file ${displayName}: ${sanitizeErrorMessage(err, [filePath])}`, - { cause: err }, - ); - } - const store = new OmniObjectStore(config.storage.getQwenDir()); const cgc = config.getContentGeneratorConfig(); // Scope the cache to the endpoint credential: an oss:// URL minted for one @@ -294,7 +395,7 @@ export async function processMediaForOmniDelivery( .update(`${cgc.baseUrl ?? ''}|${cgc.apiKey ?? ''}`) .digest('hex') .slice(0, 16); - const configuredTtl = config.getOmniUploadCacheTtlHours?.(); + const configuredTtl = config.getOmniUploadUrlTtlHours?.(); const uploadCache = new OmniUploadCache( store.getOmniRootDir(), configuredTtl === undefined @@ -303,86 +404,407 @@ export async function processMediaForOmniDelivery( cacheScope, ); // Lazy one-time hygiene scan (expired .part files, promotion orphans, - // sampled object verification). Never throws. - await runStartupRecoveryOnce(store, uploadCache); + // quarantine retention/size sweeps, sampled object verification). MUST + // run before the orchestrator: the scan deletes stale staging entries, + // which would race this process's own live invocations. (Other + // processes' live entries are protected by the staging grace window.) + await runStartupRecoveryOnce(store, uploadCache, { + quarantineRetentionDays: config.getOmniQuarantineRetentionDays?.(), + quarantineMaxBytes: config.getOmniQuarantineMaxBytes?.(), + // Corrupt-object deletion must also invalidate degradation-cache + // entries (as source or derivative) — otherwise policy-cache.json + // accumulates orphans that can never be served again. + degradationCache: new OmniDegradationCache(store.getOmniRootDir()), + }); - // Cache lookup BEFORE store promotion: a hit means the server already - // holds these bytes for this model+endpoint, so neither the local copy - // nor the upload is needed (zoom_image reads the original path, not the - // store). Checking after putFile would pay a full-file copy per hit. + // Fixed-policy preprocessing (decision D5: this single site covers + // @-commands, tool results, the URL funnel and ACP). Structural view — + // a config without the accessor (stub configs, embedders skipping + // initialize) or with no policies changes nothing. + const processingConfig = ( + config as OmniProcessingConfigView + ).getOmniProcessingConfig?.(); + // Session-namespace snapshot for `when` conditions (policy design §8.3): + // taken ONCE before any policy executes and reused across the + // preprocessing run and every transport-guard pass of this delivery. + // The request namespace is computed inside the orchestrator from the + // pending delivery set. + const conditionContext = processingConfig + ? { + session: buildSessionConditionNamespace( + config, + processingConfig.limits.reservedOutputTokens, + ), + } + : undefined; + let final: PolicyDeliveryResource = { filePath, recognized }; + /** Media deliverables beyond the primary (multi-output fixed policies). */ + let extraDeliveries: PolicyDeliveryResource[] = []; + // Transcript-protocol text deliverables (upstream P §6.2) accumulated + // across preprocessing and guard passes; threaded into every return. + const transcripts: Array<{ text: string; disclosure?: string }> = []; + const collectTranscripts = (files: PolicyFileDelivery[]) => { + for (const file of files) { + transcripts.push({ text: file.text, disclosure: file.disclosure }); + } + }; + // Pure-transcript delivery result (§6.2): the policies replaced the + // media with text-only deliverables — no media Part is emitted + // (`fileUri: ''`), nothing to guard or upload for the primary, and the + // collected transcripts ride along. Shared by the preprocessing and + // transport-guard resolutions of this shape. + const textOnlyDelivery = ( + recognizedFinal: RecognizedMedia, + tokenEstimate: OmniTokenEstimate, + extras?: { + disclosure?: string; + additionalMedia?: OmniAdditionalMediaDelivery[]; + }, + ): OmniMediaDelivery => ({ + fileUri: '', + mimeType: recognizedFinal.detectedMimeType, + sha256: '', + recognized: recognizedFinal, + tokenEstimate, + deduped: false, + uploadCacheHit: false, + degraded: true, + transcripts, + ...extras, + }); + const policies = processingConfig?.fixedPolicies ?? []; + if (policies.length > 0) { + let deliveries: PolicyDeliveryResource[]; + let fileDeliveries: PolicyFileDelivery[]; + try { + ({ deliveries, fileDeliveries } = await runFixedPolicies( + config, + { + filePath, + recognized, + displayName, + origin: options?.origin ?? 'user', + }, + { + store, + policies, + signal, + limits: processingConfig?.limits, + conditionContext, + }, + )); + } catch (err) { + if (signal?.aborted) throw err; + throw new OmniDeliveryError( + `Fixed-policy processing failed for ${displayName}: ` + + `${sanitizeErrorMessage(err, [filePath, store.getOmniRootDir()])}`, + { cause: err }, + ); + } + collectTranscripts(fileDeliveries); + // Pure-transcript delivery (§6.2): the token estimate reports the RAW + // resource for logs/telemetry; no media Part is emitted, so the guard + // verdict is irrelevant. + if (deliveries.length === 0 && transcripts.length > 0) { + return textOnlyDelivery( + recognized, + evaluateTransportLimits(config, recognized, displayName).estimate, + ); + } + // The S4 delivery contract keeps ONE primary media Part per source + // (plus any transcript text Parts); a multi-output fixed policy + // (e.g. omni_extract_keyframes) additionally yields extra media + // deliverables, carried in `additionalMedia` and materialized by the + // callers as [disclosure?, fileData] pairs after the primary Part + // (#8187 多产物投递). Zero media deliverables without transcripts + // remains a configuration error. + if (deliveries.length === 0) { + throw new OmniDeliveryError( + `Fixed policies produced 0 media deliverables for ${displayName}; exactly one is supported.`, + ); + } + final = deliveries[0]; + extraDeliveries = deliveries.slice(1); + } + + // Hash → upload-cache lookup → store promotion → upload. Shared by the + // primary deliverable and every additional media deliverable of a + // multi-output policy. Derivatives arrive with their promotion hash; + // sources are hashed at call time, after all guards. const model = config.getModel(); - const cachedUrl = await uploadCache.get(sha256, model); - if (cachedUrl) { - debugLogger.debug( - `omni upload cache hit: sha256=${sha256.slice(0, 12)}… model=${model}`, - ); - return { - fileUri: cachedUrl, - mimeType: recognized.detectedMimeType, - sha256, - recognized, - tokenEstimate, + const uploadResource = async ( + item: PolicyDeliveryResource, + estimate: OmniTokenEstimate, + ): Promise<{ + fileUri: string; + sha256: string; + deduped: boolean; + uploadCacheHit: boolean; + }> => { + let sha256: string; + try { + sha256 = item.sha256 ?? (await hashFileSha256(item.filePath, signal)); + } catch (err) { + if (signal?.aborted) throw err; + throw new OmniDeliveryError( + `Failed to hash media file ${displayName}: ${sanitizeErrorMessage(err, [item.filePath])}`, + { cause: err }, + ); + } + + // Cache lookup BEFORE store promotion: a hit means the server already + // holds these bytes for this model+endpoint, so neither the local copy + // nor the upload is needed (zoom_image reads the original path, not the + // store). Checking after putFile would pay a full-file copy per hit. + const cachedUrl = await uploadCache.get(sha256, model); + if (cachedUrl) { + debugLogger.debug( + `omni upload cache hit: sha256=${sha256.slice(0, 12)}… model=${model}`, + ); // No new copy was made: the content is already known to the system // (a prior delivery both stored and uploaded it). - deduped: true, - uploadCacheHit: true, - }; - } + return { + fileUri: cachedUrl, + sha256, + deduped: true, + uploadCacheHit: true, + }; + } - const extension = extensionForMime(recognized.detectedMimeType); - let objectPath: string; - let deduped: boolean; - try { - const put = await store.putFile(filePath, sha256, extension, signal); - objectPath = put.objectPath; - deduped = put.deduped; - } catch (err) { - if (signal?.aborted) throw err; - throw new OmniDeliveryError( - `Failed to store media in the omni object store: ` + - `${sanitizeErrorMessage(err, [filePath, store.getOmniRootDir()])}`, - { cause: err }, - ); - } + const extension = extensionForMime(item.recognized.detectedMimeType); + let objectPath: string; + let deduped: boolean; + try { + const put = await store.putFile(item.filePath, sha256, extension, signal); + objectPath = put.objectPath; + deduped = put.deduped; + } catch (err) { + if (signal?.aborted) throw err; + throw new OmniDeliveryError( + `Failed to store media in the omni object store: ` + + `${sanitizeErrorMessage(err, [item.filePath, store.getOmniRootDir()])}`, + { cause: err }, + ); + } - const uploader = new DashScopeUploader({ - apiKey: cgc.apiKey ?? '', - baseUrl: cgc.baseUrl, - }); - let fileUri: string; - try { - fileUri = await uploader.uploadFile({ - filePath: objectPath, - model, - mimeType: recognized.detectedMimeType, - signal, + const uploader = new DashScopeUploader({ + apiKey: cgc.apiKey ?? '', + baseUrl: cgc.baseUrl, }); - } catch (err) { - if (signal?.aborted) throw err; - // Upload errors can embed the object-store path (spawn/fs failures) — - // sanitize with the concrete path AND the store root, since a path with - // a space in a segment defeats the pattern pass (segment classes break - // at whitespace) and only exact replacement is immune. - throw new OmniDeliveryError( - sanitizeErrorMessage(err, [objectPath, store.getOmniRootDir()]), - { cause: err }, + let fileUri: string; + try { + fileUri = await uploader.uploadFile({ + filePath: objectPath, + model, + mimeType: item.recognized.detectedMimeType, + signal, + }); + } catch (err) { + if (signal?.aborted) throw err; + // Upload errors can embed the object-store path (spawn/fs failures) — + // sanitize with the concrete path AND the store root, since a path with + // a space in a segment defeats the pattern pass (segment classes break + // at whitespace) and only exact replacement is immune. + throw new OmniDeliveryError( + sanitizeErrorMessage(err, [objectPath, store.getOmniRootDir()]), + { cause: err }, + ); + } + + debugLogger.debug( + `omni ${item.recognized.modality} delivered: sha256=${sha256.slice(0, 12)}… ` + + `size=${item.recognized.sizeBytes} est=${estimate.estimatedTokenCount}(${estimate.status}) ` + + `deduped=${deduped} degraded=${item.degraded === true} uri=${fileUri}`, ); - } + await uploadCache.put(sha256, model, fileUri); + return { fileUri, sha256, deduped, uploadCacheHit: false }; + }; - debugLogger.debug( - `omni ${recognized.modality} delivered: sha256=${sha256.slice(0, 12)}… ` + - `size=${recognized.sizeBytes} est=${tokenEstimate.estimatedTokenCount}(${tokenEstimate.status}) ` + - `deduped=${deduped} uri=${fileUri}`, - ); - await uploadCache.put(sha256, model, fileUri); + // Additional media deliverables (multi-output fixed policies, #8187 + // 多产物投递): each is judged against the transport limits and uploaded + // through the same pipeline as the primary. A violating extra is + // explicitly omitted (policy design §10.2) rather than re-derived — it + // is already a policy product, and a second derivation pass over + // derivatives is out of scope for this stage. Deferred until the + // primary's own fate is decided (each return site below calls this + // exactly once): the primary uploads first, and a fail-closed throw on + // the primary never wastes extra uploads. + const processAdditionalMedia = async (): Promise< + OmniAdditionalMediaDelivery[] | undefined + > => { + if (extraDeliveries.length === 0) return undefined; + // Extras are independent (content-addressed store + per-file serialized + // upload cache), so they upload concurrently; map keeps output order + // aligned with the policy's deliverable order. + return Promise.all( + extraDeliveries.map( + async (extra): Promise => { + const extraGuard = evaluateTransportLimits( + config, + extra.recognized, + displayName, + ); + if (extraGuard.violation) { + debugLogger.debug( + `omni additional ${extra.recognized.modality} explicitly omitted (transport guard): ${extraGuard.violation}`, + ); + return { + fileUri: '', + mimeType: extra.recognized.detectedMimeType, + sha256: extra.sha256 ?? '', + disclosure: extra.disclosure, + omission: { reason: extraGuard.violation }, + }; + } + const uploaded = await uploadResource(extra, extraGuard.estimate); + return { + fileUri: uploaded.fileUri, + mimeType: extra.recognized.detectedMimeType, + sha256: uploaded.sha256, + disclosure: extra.disclosure, + }; + }, + ), + ); + }; + + // Transport guard on the FINAL delivery set (decision D1): the bytes + // and token estimate judged are the ones actually delivered. Stage B: + // a violation first runs the transport-guard policies (matched by + // modality only — no `when`, coverage of all three modalities is + // enforced at config normalization) for up to + // `limits.maxTransportPasses` passes; a still-over-limit resource is + // explicitly OMITTED (policy design §10.2) rather than delivered + // oversized. Without a normalized processing config (stub configs, + // embedders skipping initialize) the guard keeps its fail-closed throw. + let guard = evaluateTransportLimits(config, final.recognized, displayName); + if (guard.violation && processingConfig) { + const maxPasses = processingConfig.limits.maxTransportPasses; + for (let pass = 0; guard.violation && pass < maxPasses; pass++) { + // Re-filter per pass: a guard policy may transform the resource into + // another modality (e.g. video → extracted audio), and the NEXT pass + // must run that modality's guard policies — the pre-loop set would + // silently no-op and omit a resource the right policy could still + // bring under the limit. Coverage of all three modalities is + // enforced at config normalization, so the filter never strands a + // modality without a policy. + const guardPolicies = processingConfig.transportGuardPolicies.filter( + (p) => p.mediaTypes.includes(final.recognized.modality), + ); + if (guardPolicies.length === 0) break; + let deliveries: PolicyDeliveryResource[]; + let fileDeliveries: PolicyFileDelivery[]; + try { + ({ deliveries, fileDeliveries } = await runFixedPolicies( + config, + { + filePath: final.filePath, + recognized: final.recognized, + displayName, + origin: options?.origin ?? 'user', + }, + { + store, + policies: guardPolicies, + signal, + limits: processingConfig.limits, + conditionContext, + }, + )); + } catch (err) { + if (signal?.aborted) throw err; + // Guard-policy failure with no compliant alternative: fail closed + // — a guard configuration error must never degrade into sending + // over-limit media (policy design §10.2). Thrown as a GUARD error + // (not a generic delivery error): the verdict "this resource is + // over the limit" already stands, so consumers with an inline + // fallback (the tool-result funnel) must withhold the bytes, not + // fall back to delivering exactly what the guard rejected. + throw new OmniTransportGuardError( + `Transport-guard processing failed for ${displayName}: ` + + `${sanitizeErrorMessage(err, [final.filePath, store.getOmniRootDir()])}`, + { cause: err }, + ); + } + collectTranscripts(fileDeliveries); + // Pure-transcript guard resolution (§6.2): the guard policy + // replaced the over-limit media with text-only deliverables — the + // violation is resolved by not sending media at all. Keyed on THIS + // pass's fileDeliveries, not the cumulative transcripts: a guard + // pass that omitted the source without producing any deliverable + // must fall through to the zero-deliverable throw below, even when + // an earlier pass already collected a transcript. + if (deliveries.length === 0 && fileDeliveries.length > 0) { + return textOnlyDelivery(final.recognized, guard.estimate, { + disclosure: final.disclosure, + additionalMedia: await processAdditionalMedia(), + }); + } + if (deliveries.length !== 1) { + // Same guard-error class as the pass failure above: the violation + // verdict stands, so inline fallbacks must withhold. + throw new OmniTransportGuardError( + `Transport-guard policies produced ${deliveries.length} media deliverables for ${displayName}; exactly one is supported.`, + ); + } + if (deliveries[0].filePath === final.filePath) { + // No progress (every guard policy was a no_op for this input) — + // further passes would repeat the same work. + break; + } + // Chain the disclosures instead of replacing: when preprocessing + // already degraded the resource and the guard degrades it AGAIN, + // the model must be told about both steps (decision D8 — every + // lossy step is disclosed, not just the last one). + const priorDisclosure = final.disclosure; + final = deliveries[0]; + if (priorDisclosure && final.disclosure) { + final = { + ...final, + disclosure: `${priorDisclosure}；${final.disclosure}`, + }; + } else if (priorDisclosure) { + final = { ...final, disclosure: priorDisclosure }; + } + guard = evaluateTransportLimits(config, final.recognized, displayName); + } + } + if (guard.violation) { + if (!processingConfig) { + throw new OmniTransportGuardError(guard.violation); + } + debugLogger.debug( + `omni ${final.recognized.modality} explicitly omitted (transport guard): ${guard.violation}`, + ); + return { + fileUri: '', + mimeType: final.recognized.detectedMimeType, + sha256: final.sha256 ?? '', + recognized: final.recognized, + tokenEstimate: guard.estimate, + deduped: false, + uploadCacheHit: false, + disclosure: final.disclosure, + degraded: final.degraded, + omission: { reason: guard.violation }, + transcripts: transcripts.length > 0 ? transcripts : undefined, + additionalMedia: await processAdditionalMedia(), + }; + } + const tokenEstimate = guard.estimate; + const uploaded = await uploadResource(final, tokenEstimate); return { - fileUri, - mimeType: recognized.detectedMimeType, - sha256, - recognized, + fileUri: uploaded.fileUri, + mimeType: final.recognized.detectedMimeType, + sha256: uploaded.sha256, + recognized: final.recognized, tokenEstimate, - deduped, - uploadCacheHit: false, + deduped: uploaded.deduped, + uploadCacheHit: uploaded.uploadCacheHit, + disclosure: final.disclosure, + degraded: final.degraded, + transcripts: transcripts.length > 0 ? transcripts : undefined, + additionalMedia: await processAdditionalMedia(), }; } @@ -438,6 +860,52 @@ export async function readMediaViaOmniDelivery(params: { expectedModality, signal, }); + // §6.2/D8 ordering contract documented on buildTranscriptParts. + const transcriptParts = buildTranscriptParts( + displayName, + delivery.transcripts, + ); + // Additional media Parts (multi-output fixed policies): materialized + // right after the primary media slot in every branch below. + const additionalParts = buildAdditionalMediaParts( + displayName, + delivery.additionalMedia, + ); + if (delivery.omission) { + // Explicit omission (policy design §10.2): the media is withheld and + // the omission notice text stands in its place. Not an error — the + // read succeeded; the transport guard's verdict is the content. + const omissionPart = { + text: formatOmissionText(displayName, delivery.omission.reason), + }; + return { + llmContent: + transcriptParts.length > 0 || additionalParts.length > 0 + ? [omissionPart, ...additionalParts, ...transcriptParts] + : omissionPart.text, + returnDisplay: `Media omitted by the omni transport guard: ${relativePathForDisplay}`, + tokenEstimate: delivery.tokenEstimate, + }; + } + if (!delivery.fileUri && transcriptParts.length > 0) { + // Pure-transcript delivery (§6.2): the policies replaced the media + // with text-only deliverables — no media Part is emitted for the + // primary (additional media deliverables, if any, still are). The + // primary disclosure (chained prior lossy steps, decision D8) still + // renders: the transcript was derived through those steps. + const disclosureParts = delivery.disclosure + ? [{ text: formatDisclosureText(displayName, delivery.disclosure) }] + : []; + return { + llmContent: [ + ...disclosureParts, + ...additionalParts, + ...transcriptParts, + ], + returnDisplay: `Read ${delivery.recognized.modality} as transcript (omni policy): ${relativePathForDisplay}`, + tokenEstimate: delivery.tokenEstimate, + }; + } const fileDataPart = { fileData: { fileUri: delivery.fileUri, @@ -445,20 +913,39 @@ export async function readMediaViaOmniDelivery(params: { displayName, }, }; + const parts: Array<{ text: string } | typeof fileDataPart> = []; const { width, height } = delivery.recognized.metadata; - const llmContent = + if ( delivery.recognized.modality === 'image' && width !== undefined && height !== undefined - ? [ - { - text: - `Image ${displayName}: full resolution ${width}x${height} px. ` + - `Use zoom_image for a closer look at details.`, - }, - fileDataPart, - ] - : fileDataPart; + ) { + // On the degradation path `delivery.recognized` re-recognizes the + // DERIVATIVE, so width/height are the downsampled dimensions — + // calling them "full resolution" would contradict the disclosure + // pushed right below and steer the model away from zoom_image, the + // exact remedy for degradation-stripped detail (it reads the + // original from disk). + parts.push({ + text: delivery.disclosure + ? `Image ${displayName}: delivered at ${width}x${height} px ` + + `after degradation. Use zoom_image to inspect details — it ` + + `reads the original file.` + : `Image ${displayName}: full resolution ${width}x${height} px. ` + + `Use zoom_image for a closer look at details.`, + }); + } + // Disclosure IMMEDIATELY before its media part (decision D8): provider + // converters that relocate media move the adjacent pair together. + if (delivery.disclosure) { + parts.push({ + text: formatDisclosureText(displayName, delivery.disclosure), + }); + } + parts.push(fileDataPart); + parts.push(...additionalParts); + parts.push(...transcriptParts); + const llmContent = parts.length === 1 ? fileDataPart : parts; return { llmContent, returnDisplay: `Read ${delivery.recognized.modality} file (omni upload): ${relativePathForDisplay}`, @@ -487,10 +974,11 @@ export async function readMediaViaOmniDelivery(params: { /** Effective download byte ceiling — never above the upload channel cap * (downloading more than can be delivered is pointless), including when - * `omni.download.maxFileBytes` is explicitly configured higher. */ + * `omni.ingestion.localization.url.maxFileBytes` is explicitly configured + * higher. */ export function effectiveMaxDownloadFileBytes(config: Config): number { const uploadCap = effectiveMaxUploadFileBytes(config); - const configured = config.getOmniDownloadMaxFileBytes?.(); + const configured = config.getOmniUrlDownloadMaxFileBytes?.(); if (configured !== undefined && configured > 0) { return Math.min(configured, uploadCap); } diff --git a/packages/core/src/omni/json-cache-file.ts b/packages/core/src/omni/json-cache-file.ts new file mode 100644 index 00000000000..4cfed9375a9 --- /dev/null +++ b/packages/core/src/omni/json-cache-file.ts @@ -0,0 +1,204 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { atomicWriteFile } from '../utils/atomicFileWrite.js'; +import { createDebugLogger, type DebugLogger } from '../utils/debugLogger.js'; + +/** Per-cache-file operation serializer: cache instances are constructed + * per use site, and safe-tool batches run deliveries concurrently in one + * process — unserialized load-modify-save would drop entries. Module + * scope is deliberate: two instances on the same file must share the + * chain. Cross-process writes remain last-writer-wins (documented). */ +const fileOps = new Map>(); + +function serialize(key: string, fn: () => Promise): Promise { + const prev = fileOps.get(key) ?? Promise.resolve(); + const run = prev.then(fn, fn); + const settled = run.then( + () => {}, + () => {}, + ); + fileOps.set(key, settled); + void settled.then(() => { + // Drop the tail once it settles — otherwise the map grows with every + // distinct cache file touched over the process lifetime. Only delete + // when OUR promise is still the tail: a later op may have chained on. + if (fileOps.get(key) === settled) fileOps.delete(key); + }); + return run; +} + +/** Keep at most this many `.corrupt-*` backups (newest wins): a crash + * loop over a corrupt file must not litter the directory without bound. */ +const MAX_CORRUPT_BACKUPS = 2; + +interface CacheFileShape { + version: 1; + entries: Record; +} + +/** + * Shared mechanics for omni's persistent JSON entry caches + * (`upload-cache.json`, `policy-cache.json`): one flat + * `{ version: 1, entries: {} }` file with + * + * - per-file serialized load-modify-save (in-process), + * - atomic writes via `atomicWriteFile` (tmp + rename, `noFollow`; 0600 + * forced on every save, 0700 dir), + * - corrupt files backed up as `.corrupt-` (newest + * {@link MAX_CORRUPT_BACKUPS} kept) and rebuilt empty — never fatal, + * - unreadable-but-existing files (EACCES, EMFILE, …) making the current + * operation a no-op instead of an empty rebuild: a transient read + * failure must never lead to a save that wipes every persisted entry. + * + * Entry semantics (keys, TTLs, invalidation) stay in the owning cache. + */ +export class OmniJsonCacheFile { + private readonly debugLogger: DebugLogger; + + constructor( + readonly filePath: string, + debugChannel: string, + ) { + this.debugLogger = createDebugLogger(debugChannel); + } + + /** + * Run one serialized operation against the entry map. `fn` returns the + * operation result plus whether it changed the map (triggering an + * atomic save). When the file exists but cannot be read, + * `unreadableResult` is returned and nothing is saved. + */ + async access( + unreadableResult: R, + fn: ( + entries: Record, + ) => + | { result: R; changed?: boolean } + | Promise<{ result: R; changed?: boolean }>, + ): Promise { + return serialize(this.filePath, async () => { + const data = await this.load(); + if (!data) return unreadableResult; + const { result, changed } = await fn(data.entries); + if (changed) await this.save(data); + return result; + }); + } + + /** + * Load the cache file. Returns null when the file exists but could not + * be read (EACCES, EMFILE, …): the caller must skip its operation for + * this call — proceeding with an empty snapshot and later saving it + * would overwrite N valid entries with one (self-inflicted cache wipe). + * Only a genuinely missing file means empty-and-writable. + */ + private async load(): Promise | null> { + let raw: string; + try { + raw = await fs.readFile(this.filePath, 'utf8'); + } catch (err) { + const code = (err as NodeJS.ErrnoException).code; + // ENOENT: no cache file yet (POSIX and Windows). ENOTDIR: a parent + // path component is a plain file — POSIX raises ENOTDIR where + // Windows reports ENOENT for the same condition. + if (code === 'ENOENT' || code === 'ENOTDIR') { + return { version: 1, entries: {} }; + } + this.debugLogger.debug( + `cache read failed, operation skipped: ${err instanceof Error ? err.message : err}`, + ); + return null; + } + try { + const parsed = JSON.parse(raw) as CacheFileShape; + // `entries` must be a plain non-null object: `typeof null` and + // `typeof []` are both 'object', and either shape would throw raw + // TypeErrors from every accessor (escaping the never-fatal contract + // and skipping backup+rebuild). + if ( + parsed?.version === 1 && + typeof parsed.entries === 'object' && + parsed.entries !== null && + !Array.isArray(parsed.entries) + ) { + // Entry VALUES must be plain non-null non-array objects too: the + // file is edited/shipped by hand (workspace caches), and a value + // like `null` or `"x"` would surface as raw TypeErrors from the + // owning cache's field accessors (`v.expiresAt`, + // `v.degradedSha256`, …) — escaping the never-fatal contract. + // Malformed values are pruned individually (cheap re-work for + // just those keys) instead of condemning the whole file. + let pruned = 0; + for (const [k, v] of Object.entries(parsed.entries)) { + if (typeof v !== 'object' || v === null || Array.isArray(v)) { + delete parsed.entries[k]; + pruned++; + } + } + if (pruned > 0) { + this.debugLogger.debug( + `dropped ${pruned} malformed cache entr${pruned === 1 ? 'y' : 'ies'} from ${this.filePath}`, + ); + } + return parsed; + } + throw new Error('unexpected shape'); + } catch { + // Corrupt cache: preserve for inspection, start fresh. Losing a + // cache only costs re-work — never fail the pipeline over it. + const backup = `${this.filePath}.corrupt-${Date.now()}`; + await fs.rename(this.filePath, backup).catch(() => {}); + await this.pruneCorruptBackups(); + this.debugLogger.debug(`corrupt cache backed up to ${backup}`); + return { version: 1, entries: {} }; + } + } + + /** Best-effort: keep only the newest {@link MAX_CORRUPT_BACKUPS}. */ + private async pruneCorruptBackups(): Promise { + const dir = path.dirname(this.filePath); + const prefix = `${path.basename(this.filePath)}.corrupt-`; + try { + const backups = (await fs.readdir(dir)) + .filter((n) => n.startsWith(prefix)) + // Millisecond timestamps are fixed-width for centuries, so the + // lexicographic sort is chronological; newest first. + .sort() + .reverse(); + for (const name of backups.slice(MAX_CORRUPT_BACKUPS)) { + await fs.rm(path.join(dir, name), { force: true }).catch(() => {}); + } + } catch { + // Pruning is hygiene; never let it affect the read path. + } + } + + private async save(data: CacheFileShape): Promise { + try { + await fs.mkdir(path.dirname(this.filePath), { + recursive: true, + mode: 0o700, + }); + await atomicWriteFile(this.filePath, JSON.stringify(data, null, 1), { + mode: 0o600, + forceMode: true, + // Rename-replacement semantics: a symlink planted at the cache + // path is REPLACED by the rename, never written through — without + // this the default symlink resolution would redirect the write + // (and the 0600 chmod) onto the link's target. + noFollow: true, + }); + } catch (err) { + // Cache persistence is best-effort by design. + this.debugLogger.debug( + `cache write failed: ${err instanceof Error ? err.message : err}`, + ); + } + } +} diff --git a/packages/core/src/omni/media-guidance.test.ts b/packages/core/src/omni/media-guidance.test.ts new file mode 100644 index 00000000000..b655584fed8 --- /dev/null +++ b/packages/core/src/omni/media-guidance.test.ts @@ -0,0 +1,109 @@ +/** + * @license + * Copyright 2026 Qwen Team + * SPDX-License-Identifier: Apache-2.0 + */ + +import { describe, expect, it, vi } from 'vitest'; +import type { Config } from '../config/config.js'; +import { AuthType } from '../core/contentGenerator.js'; +import { ToolNames } from '../tools/tool-names.js'; +import { buildOmniMediaGuidanceSection } from './media-guidance.js'; +import { + OMNI_DISCLOSURE_TEXT_PREFIX, + OMNI_OMISSION_TEXT_PREFIX, + OMNI_TRANSCRIPT_TEXT_PREFIX, +} from './disclosure.js'; + +const DASHSCOPE_CGC = { + authType: AuthType.USE_OPENAI, + apiKey: 'sk-real-key', + baseUrl: 'https://dashscope.aliyuncs.com/compatible-mode/v1', +}; + +function stubConfig(overrides?: { + omniEnabled?: boolean; + cgc?: Record; + policyTools?: Record; +}): Config { + return { + isOmniEnabled: vi.fn().mockReturnValue(overrides?.omniEnabled ?? true), + isTrustedFolder: vi.fn().mockReturnValue(true), + getContentGeneratorConfig: vi + .fn() + .mockReturnValue(overrides?.cgc ?? DASHSCOPE_CGC), + getOmniPolicyToolsSettings: vi.fn().mockReturnValue(overrides?.policyTools), + } as unknown as Config; +} + +describe('buildOmniMediaGuidanceSection', () => { + it('returns null when omni is disabled', () => { + expect( + buildOmniMediaGuidanceSection(stubConfig({ omniEnabled: false })), + ).toBeNull(); + }); + + it('returns null when the delivery gate rejects the provider', () => { + expect( + buildOmniMediaGuidanceSection( + stubConfig({ + cgc: { ...DASHSCOPE_CGC, baseUrl: 'https://api.openai.com/v1' }, + }), + ), + ).toBeNull(); + }); + + it('explains all three disclosure markers and the progressive contract', () => { + const section = buildOmniMediaGuidanceSection(stubConfig()); + expect(section).toContain(OMNI_DISCLOSURE_TEXT_PREFIX); + expect(section).toContain(OMNI_OMISSION_TEXT_PREFIX); + expect(section).toContain(OMNI_TRANSCRIPT_TEXT_PREFIX); + expect(section).toContain('progressive-understanding'); + // The two behavioral pillars: overview-not-complete + no extrapolation. + expect(section).toContain('not the complete content'); + expect(section).toContain('Never conclude'); + }); + + it('with no tools enabled, instructs stating missing evidence instead of listing tools', () => { + const section = buildOmniMediaGuidanceSection(stubConfig())!; + expect(section).toContain('No media tools are enabled'); + expect(section).not.toContain('Available media tools'); + expect(section).not.toContain(ToolNames.OMNI_CLIP_VIDEO); + }); + + it('lists exactly the modelAccess-enabled tools', () => { + const section = buildOmniMediaGuidanceSection( + stubConfig({ + policyTools: { + [ToolNames.OMNI_CLIP_VIDEO]: { modelAccess: { enabled: true } }, + [ToolNames.OMNI_EXTRACT_KEYFRAMES]: { + modelAccess: { enabled: true }, + }, + // Present but not enabled — must not be listed. + [ToolNames.OMNI_TRANSCRIBE_AUDIO]: { + modelAccess: { enabled: false }, + }, + }, + }), + )!; + expect(section).toContain('Available media tools'); + expect(section).toContain(ToolNames.OMNI_CLIP_VIDEO); + expect(section).toContain(ToolNames.OMNI_EXTRACT_KEYFRAMES); + expect(section).not.toContain(ToolNames.OMNI_TRANSCRIBE_AUDIO); + expect(section).not.toContain('No media tools are enabled'); + // Tool-usage direction present when tools are available. + expect(section).toContain('fetch the evidence yourself'); + }); + + it('tolerates malformed policyTools settings (fail-closed per tool)', () => { + const section = buildOmniMediaGuidanceSection( + stubConfig({ + policyTools: { + [ToolNames.OMNI_CLIP_VIDEO]: 'not-an-object', + [ToolNames.OMNI_EXTRACT_KEYFRAMES]: { modelAccess: 'nope' }, + }, + }), + )!; + expect(section).toContain('No media tools are enabled'); + }); +}); diff --git a/packages/core/src/omni/media-guidance.ts b/packages/core/src/omni/media-guidance.ts new file mode 100644 index 00000000000..e9ccccf5d58 --- /dev/null +++ b/packages/core/src/omni/media-guidance.ts @@ -0,0 +1,109 @@ +/** + * @license + * Copyright 2026 Qwen Team + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Progressive media understanding guidance (system-prompt layer). + * + * The delivery pipeline degrades media to fit transport/context limits and + * discloses each transformation next to the media Part (decision D8). The + * disclosures state WHAT was done; this module supplies the missing WHY + * and the follow-up contract: degradation is a context-budget-driven + * progressive-understanding strategy, the degraded delivery is an + * overview/entry point rather than the complete content, and the model + * should proactively fetch higher-fidelity / more complete evidence with + * the media policy tools when the task needs it. + * + * Injected once per session as a STABLE system-prompt layer (see + * client.ts getMainSessionSystemInstruction): the disclosure texts arrive + * mid-conversation at unpredictable points, so per-delivery preambles + * would repeat across three assembly sites (atCommandProcessor, + * fileUtils, tool-result-media) and would still miss the reactive + * server-reject swaps; one durable contract in the prompt covers all of + * them. Kept as a leaf module (no pipeline imports) so prompt assembly + * stays lightweight. + */ + +import type { Config } from '../config/config.js'; +import { ToolNames } from '../tools/tool-names.js'; +import { isOmniDeliveryActive } from './delivery-gate.js'; +import { resolveMediaPolicyModelAccess } from './policy/model-access.js'; +import { + OMNI_DISCLOSURE_TEXT_PREFIX, + OMNI_OMISSION_TEXT_PREFIX, + OMNI_TRANSCRIPT_TEXT_PREFIX, +} from './disclosure.js'; + +/** One-line capability summaries for the model-callable media tools. + * Deliberately terse: the full parameter schema ships with the tool + * declaration; this list only tells the model WHEN to reach for each. */ +const MEDIA_TOOL_CAPABILITIES: ReadonlyArray<[string, string]> = [ + [ + ToolNames.OMNI_CLIP_VIDEO, + 'cut a specific time range out of a video (startSec/durationSec) — the primary way to inspect parts of a long video that were not delivered', + ], + [ + ToolNames.OMNI_EXTRACT_KEYFRAMES, + 'extract still frames from a video (clip a range first to sample frames from a specific segment)', + ], + [ + ToolNames.OMNI_DOWNSCALE_VIDEO, + 're-encode a video at lower resolution / frame rate to fit transport limits', + ], + [ + ToolNames.OMNI_EXTRACT_AUDIO, + 'extract the audio track from a video for listening or transcription', + ], + [ + ToolNames.OMNI_TRANSCRIBE_AUDIO, + 'transcribe speech in an audio file to text', + ], + [ + ToolNames.OMNI_DOWNSAMPLE_AUDIO, + 're-encode audio at a lower bitrate/sample rate', + ], + [ToolNames.OMNI_DOWNSAMPLE_IMAGE, 'shrink an image to a smaller resolution'], + [ToolNames.OMNI_CONVERT_IMAGE, 'convert an image to another format'], +]; + +/** + * Build the progressive media understanding section for the system + * prompt, or `null` when omni delivery is inactive (no disclosures will + * ever reach the model, so the contract would be noise). + * + * The tool list only names tools whose + * `omni.processing.policyTools..modelAccess.enabled` is true — + * directing the model at tools the scheduler would reject teaches it a + * dead end. With no tools enabled, the section instead instructs the + * model to state what evidence is missing rather than extrapolate. + */ +export function buildOmniMediaGuidanceSection(config: Config): string | null { + if (!isOmniDeliveryActive(config)) return null; + + const enabledTools = MEDIA_TOOL_CAPABILITIES.filter( + ([name]) => resolveMediaPolicyModelAccess(config, name).enabled, + ); + + const toolGuidance = + enabledTools.length > 0 + ? `- When the task needs evidence beyond what was delivered — later time ranges, finer visual detail, more frames, a fuller transcript — do not stop at the delivered subset: fetch the evidence yourself with the media tools below, then read the produced file(s) to bring them into context. Work in targeted excerpts (a specific time range or region at a time) so each request stays within limits, and iterate until you have seen enough to complete the task. +- Available media tools: +${enabledTools.map(([name, capability]) => ` - ${name}: ${capability}`).join('\n')}` + : `- No media tools are enabled in this session. When the delivered evidence does not cover what the task needs, say explicitly which part of the media you could not observe instead of extrapolating from the delivered subset.`; + + return `# Media Delivery (Progressive Understanding) + +Media files in this session reach you through a preprocessing pipeline that must fit them into transport and context-window limits. Large or long media is therefore delivered in reduced form on purpose: this is a progressive-understanding strategy — you first get an affordable overview, then fetch the specific higher-fidelity evidence the task needs. Every transformation is disclosed in a text part placed immediately BEFORE the media it describes: + +- ${OMNI_DISCLOSURE_TEXT_PREFIX}: the adjacent media is a degraded derivative (clipped, downscaled, resampled, or keyframes); the marker states exactly what was reduced. +- ${OMNI_OMISSION_TEXT_PREFIX}: the media could not be delivered at all; the notice stands in its place. +- ${OMNI_TRANSCRIPT_TEXT_PREFIX}: text derived from the media (e.g. a speech transcript), possibly delivered instead of the media itself. + +Interpret delivered media under this contract: + +- A degraded delivery is an OVERVIEW or entry point, not the complete content. The original file on disk is untouched and remains fully available for further processing. +- Never conclude that content outside the delivered portion does not exist, and never present conclusions drawn from a partial delivery as covering the whole file. Read each disclosure quantitatively: a clip marker covering [0s–600s] of a 4882s video means 4282s exist that you have NOT seen; a keyframe marker tells you which timestamps you actually saw. +${toolGuidance}`; +} diff --git a/packages/core/src/omni/policy/conditions.test.ts b/packages/core/src/omni/policy/conditions.test.ts new file mode 100644 index 00000000000..bfe7b2eabc7 --- /dev/null +++ b/packages/core/src/omni/policy/conditions.test.ts @@ -0,0 +1,337 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { describe, expect, it } from 'vitest'; +import type { + FixedPolicyCondition, + FixedPolicyConditionContext, +} from './conditions.js'; +import { + evaluateFixedPolicyCondition, + validateFixedPolicyCondition, +} from './conditions.js'; + +const CONTEXT: FixedPolicyConditionContext = { + resource: { + sizeBytes: 8_200_000, + width: 4096, + height: 3072, + estimatedTokenCount: 150_000, + }, + request: { totalEstimatedMediaTokens: 180_000 }, + session: { + contextWindowTokens: 131_072, + promptTokenCount: 20_000, + reservedOutputTokens: 8_192, + availableContextTokens: 102_880, + }, +}; + +const expr = (...parts: unknown[]): FixedPolicyCondition => + parts as unknown as FixedPolicyCondition; + +describe('evaluateFixedPolicyCondition — comparisons', () => { + it.each([ + // [operator, right literal, expected outcome] against width=4096 + ['>', 4095, 'match'], + ['>', 4096, 'no_match'], + ['>=', 4096, 'match'], + ['>=', 4097, 'no_match'], + ['<', 4097, 'match'], + ['<', 4096, 'no_match'], + ['<=', 4096, 'match'], + ['<=', 4095, 'no_match'], + ['==', 4096, 'match'], + ['==', 4095, 'no_match'], + ['!=', 4095, 'match'], + ['!=', 4096, 'no_match'], + ] as const)('width %s %d → %s', (operator, right, outcome) => { + const result = evaluateFixedPolicyCondition( + expr(operator, ['field', 'resource.width'], right), + CONTEXT, + ); + expect(result.outcome).toBe(outcome); + }); + + it('compares field to field (the §8.3 keyframe-extraction example)', () => { + const result = evaluateFixedPolicyCondition( + expr( + '>', + ['field', 'resource.estimatedTokenCount'], + ['field', 'session.availableContextTokens'], + ), + CONTEXT, + ); + expect(result).toEqual({ outcome: 'match' }); + }); + + it('compares literal to literal', () => { + expect(evaluateFixedPolicyCondition(expr('<', 2, 3), CONTEXT).outcome).toBe( + 'match', + ); + }); + + it('== supports strict string/boolean equality; type mismatch is a determinate no_match', () => { + expect( + evaluateFixedPolicyCondition(expr('==', 'aac', 'aac'), CONTEXT).outcome, + ).toBe('match'); + expect( + evaluateFixedPolicyCondition(expr('==', '3', 3), CONTEXT).outcome, + ).toBe('no_match'); + // ...and != is its exact complement, including across types. + expect( + evaluateFixedPolicyCondition(expr('!=', '3', 3), CONTEXT).outcome, + ).toBe('match'); + }); + + it('an absent field is unavailable, never false', () => { + const result = evaluateFixedPolicyCondition( + expr('>', ['field', 'resource.durationMs'], 0), + CONTEXT, + ); + expect(result).toEqual({ + outcome: 'unavailable', + missingFields: ['resource.durationMs'], + }); + }); + + it('an unknown field name is unavailable and named', () => { + const result = evaluateFixedPolicyCondition( + expr('>', ['field', 'resource.doesNotExist'], 0), + CONTEXT, + ); + expect(result).toMatchObject({ + outcome: 'unavailable', + missingFields: ['resource.doesNotExist'], + }); + }); + + it('both operands missing → both fields recorded', () => { + const result = evaluateFixedPolicyCondition( + expr( + '>', + ['field', 'resource.bitRate'], + ['field', 'resource.sampleRateHz'], + ), + CONTEXT, + ); + expect(result).toEqual({ + outcome: 'unavailable', + missingFields: ['resource.bitRate', 'resource.sampleRateHz'], + }); + }); + + it('ordering over a non-numeric literal is unavailable, not false', () => { + const result = evaluateFixedPolicyCondition( + expr('>', ['field', 'resource.width'], 'wide'), + CONTEXT, + ); + expect(result).toMatchObject({ outcome: 'unavailable' }); + }); +}); + +describe('evaluateFixedPolicyCondition — combinators (strong Kleene)', () => { + const TRUE = expr('==', 1, 1); + const FALSE = expr('==', 1, 2); + const UNAVAILABLE = expr('>', ['field', 'resource.durationMs'], 0); + + it('all: every branch true → match', () => { + expect( + evaluateFixedPolicyCondition(expr('all', TRUE, TRUE), CONTEXT).outcome, + ).toBe('match'); + }); + + it('all: a false branch dominates an unavailable sibling', () => { + expect( + evaluateFixedPolicyCondition(expr('all', UNAVAILABLE, FALSE), CONTEXT) + .outcome, + ).toBe('no_match'); + }); + + it('all: true + unavailable → unavailable with the missing field', () => { + expect( + evaluateFixedPolicyCondition(expr('all', TRUE, UNAVAILABLE), CONTEXT), + ).toEqual({ + outcome: 'unavailable', + missingFields: ['resource.durationMs'], + }); + }); + + it('any: a true branch dominates an unavailable sibling', () => { + expect( + evaluateFixedPolicyCondition(expr('any', UNAVAILABLE, TRUE), CONTEXT) + .outcome, + ).toBe('match'); + }); + + it('any: every branch false → no_match', () => { + expect( + evaluateFixedPolicyCondition(expr('any', FALSE, FALSE), CONTEXT).outcome, + ).toBe('no_match'); + }); + + it('any: false + unavailable → unavailable', () => { + expect( + evaluateFixedPolicyCondition(expr('any', FALSE, UNAVAILABLE), CONTEXT), + ).toEqual({ + outcome: 'unavailable', + missingFields: ['resource.durationMs'], + }); + }); + + it('!: flips determinate outcomes', () => { + expect(evaluateFixedPolicyCondition(expr('!', TRUE), CONTEXT).outcome).toBe( + 'no_match', + ); + expect( + evaluateFixedPolicyCondition(expr('!', FALSE), CONTEXT).outcome, + ).toBe('match'); + }); + + it('!: unavailable passes through — negation must not launder unknowns', () => { + expect( + evaluateFixedPolicyCondition(expr('!', UNAVAILABLE), CONTEXT), + ).toEqual({ + outcome: 'unavailable', + missingFields: ['resource.durationMs'], + }); + }); + + it('nests recursively and dedups missing fields', () => { + const result = evaluateFixedPolicyCondition( + expr( + 'any', + expr('all', UNAVAILABLE, TRUE), + expr('<', ['field', 'resource.durationMs'], 100), + ), + CONTEXT, + ); + expect(result).toEqual({ + outcome: 'unavailable', + missingFields: ['resource.durationMs'], + }); + }); + + it('vacuous combinators: ["all"] → match, ["any"] → no_match', () => { + expect(evaluateFixedPolicyCondition(expr('all'), CONTEXT).outcome).toBe( + 'match', + ); + expect(evaluateFixedPolicyCondition(expr('any'), CONTEXT).outcome).toBe( + 'no_match', + ); + }); + + it('never throws on malformed nodes — degrades to unavailable', () => { + for (const bad of [ + null, + 42, + 'gt', + {}, + [], + ['between', 1, 2], + ['>', 1], // wrong arity + ['>', 1, 2, 3], // wrong arity + ['!'], // missing operand + ['!', TRUE, FALSE], // extra operand + ['>', ['field'], 1], // malformed field reference + // The retired object form must degrade, not silently match. + { + left: { field: 'resource.width' }, + operator: 'gt', + right: { value: 1 }, + }, + ]) { + const result = evaluateFixedPolicyCondition( + bad as unknown as FixedPolicyCondition, + CONTEXT, + ); + expect(result.outcome).toBe('unavailable'); + } + }); +}); + +describe('validateFixedPolicyCondition', () => { + it('accepts the §8.3 documentation example', () => { + expect( + validateFixedPolicyCondition([ + 'all', + [ + '>', + ['field', 'resource.estimatedTokenCount'], + ['field', 'session.availableContextTokens'], + ], + ['>=', ['field', 'session.contextWindowTokens'], 131072], + ]), + ).toEqual([]); + }); + + it('accepts negation and != comparisons', () => { + expect( + validateFixedPolicyCondition([ + '!', + ['!=', ['field', 'resource.channels'], 2], + ]), + ).toEqual([]); + }); + + it.each([ + ['non-array root', 7, /must be an expression array/], + ['empty array', [], /must be an expression array/], + ['non-string head', [42, 1, 2], /must be an expression array/], + ['bare all', ['all'], /"all" requires at least one operand/], + ['bare any', ['any'], /"any" requires at least one operand/], + ['! with two operands', ['!', ['==', 1, 1], ['==', 2, 2]], /exactly one/], + ['unknown operator', ['between', 1, 2], /unknown operator "between"/], + ['comparison arity', ['>', 1], /takes exactly two operands/], + ['unknown field', ['>', ['field', 'resource.nope'], 1], /unknown field/], + [ + 'malformed field reference', + ['>', ['field'], 1], + /field reference must be/, + ], + [ + 'array that is not a field reference', + ['>', ['resource.width'], 1], + /field reference must be/, + ], + [ + 'ordering operator with a string literal', + ['>', ['field', 'resource.width'], 'wide'], + /requires a finite numeric literal/, + ], + [ + 'non-primitive literal', + ['==', { nested: true }, 1], + /number, string, or boolean/, + ], + [ + 'legacy object form gets a migration hint', + { + left: { field: 'resource.width' }, + operator: 'gt', + right: { value: 3000 }, + }, + /no longer supported/, + ], + ])('rejects %s', (_label, raw, pattern) => { + const errors = validateFixedPolicyCondition(raw); + expect(errors.length).toBeGreaterThan(0); + expect(errors.join('\n')).toMatch(pattern); + }); + + it('== and != allow string and boolean literals', () => { + expect(validateFixedPolicyCondition(['==', true, 'x'])).toEqual([]); + expect(validateFixedPolicyCondition(['!=', 'aac', 'opus'])).toEqual([]); + }); + + it('reports nested positional paths for errors inside combinators', () => { + const errors = validateFixedPolicyCondition([ + 'any', + ['all', ['nope', 1, 2]], + ]); + expect(errors.join('\n')).toContain('when[1][1][0]'); + }); +}); diff --git a/packages/core/src/omni/policy/conditions.ts b/packages/core/src/omni/policy/conditions.ts new file mode 100644 index 00000000000..1f1b2f34a17 --- /dev/null +++ b/packages/core/src/omni/policy/conditions.ts @@ -0,0 +1,377 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Restricted `when` condition DSL for fixed policies (policy design §8.3). + * + * Conditions are Mapbox-style expression arrays: `[operator, ...operands]`. + * A comparison takes exactly two operands, each either a + * `["field", ""]` reference or a bare literal — + * `[">", ["field", "resource.width"], 3000]`. Combinators nest recursively: + * `["all", , ...]`, `["any", , ...]`, `["!", ]`. No + * arbitrary code, no JSONPath, no value-producing sub-expressions inside + * operands. Evaluation is three-valued: a comparison over a field that + * cannot be resolved yields `unavailable`, which must NEVER be silently + * treated as false — the caller applies the policy's + * `onConditionUnavailable` behavior (default: skip) and the run record + * names the missing fields. + */ + +/** Readable fields, grouped in their three natural namespaces. */ +export const RESOURCE_CONDITION_FIELDS = [ + 'sizeBytes', + 'durationMs', + 'width', + 'height', + 'maxWidth', + 'maxHeight', + 'frameRate', + 'frameCount', + 'bitRate', + 'sampleRateHz', + 'channels', + 'estimatedTokenCount', +] as const; +export const REQUEST_CONDITION_FIELDS = ['totalEstimatedMediaTokens'] as const; +export const SESSION_CONDITION_FIELDS = [ + 'contextWindowTokens', + 'promptTokenCount', + 'reservedOutputTokens', + 'availableContextTokens', +] as const; + +export type ResourceConditionField = (typeof RESOURCE_CONDITION_FIELDS)[number]; +export type RequestConditionField = (typeof REQUEST_CONDITION_FIELDS)[number]; +export type SessionConditionField = (typeof SESSION_CONDITION_FIELDS)[number]; + +/** Fully-qualified field name, e.g. `resource.sizeBytes`. */ +export type FixedPolicyField = + | `resource.${ResourceConditionField}` + | `request.${RequestConditionField}` + | `session.${SessionConditionField}`; + +/** Operand of a comparison: a field reference or a bare literal. */ +export type ConditionOperand = + | ['field', FixedPolicyField] + | number + | string + | boolean; + +export type ComparisonOperator = '>' | '>=' | '<' | '<=' | '==' | '!='; + +export type ComparisonCondition = [ + ComparisonOperator, + ConditionOperand, + ConditionOperand, +]; + +export type FixedPolicyCondition = + | ComparisonCondition + | ['all' | 'any', ...FixedPolicyCondition[]] + | ['!', FixedPolicyCondition]; + +/** Values feeding field resolution. All defined fields are numeric; an + * absent entry means the field could not be obtained for this resource + * (e.g. `durationMs` for an image, or a probe that returned nothing). */ +export interface FixedPolicyConditionContext { + resource?: Partial>; + request?: Partial>; + session?: Partial>; +} + +export type ConditionEvaluation = + | { outcome: 'match' } + | { outcome: 'no_match' } + | { + outcome: 'unavailable'; + /** Field names (or operand descriptions) that made the result + * undecidable — surfaced in the policy run record. */ + missingFields: string[]; + }; + +const MATCH: ConditionEvaluation = { outcome: 'match' }; +const NO_MATCH: ConditionEvaluation = { outcome: 'no_match' }; + +function unavailable(missingFields: string[]): ConditionEvaluation { + return { outcome: 'unavailable', missingFields: [...new Set(missingFields)] }; +} + +function isPlainObject(v: unknown): v is Record { + return typeof v === 'object' && v !== null && !Array.isArray(v); +} + +const COMPARISON_OPERATORS: readonly ComparisonOperator[] = [ + '>', + '>=', + '<', + '<=', + '==', + '!=', +]; + +function isComparisonOperator(v: unknown): v is ComparisonOperator { + return (COMPARISON_OPERATORS as readonly unknown[]).includes(v); +} + +const KNOWN_FIELDS: ReadonlySet = new Set([ + ...RESOURCE_CONDITION_FIELDS.map((f) => `resource.${f}`), + ...REQUEST_CONDITION_FIELDS.map((f) => `request.${f}`), + ...SESSION_CONDITION_FIELDS.map((f) => `session.${f}`), +]); + +type ResolvedOperand = + | { ok: true; value: number | string | boolean; describe: string } + | { ok: false; missing: string }; + +function resolveOperand( + operand: unknown, + context: FixedPolicyConditionContext, +): ResolvedOperand { + if (Array.isArray(operand)) { + if ( + operand.length !== 2 || + operand[0] !== 'field' || + typeof operand[1] !== 'string' + ) { + return { ok: false, missing: '' }; + } + const field = operand[1]; + if (!KNOWN_FIELDS.has(field)) { + return { ok: false, missing: field }; + } + const [namespace, name] = field.split('.') as [ + 'resource' | 'request' | 'session', + string, + ]; + const value = ( + context[namespace] as Record | undefined + )?.[name]; + if (typeof value !== 'number' || Number.isNaN(value)) { + return { ok: false, missing: field }; + } + return { ok: true, value, describe: field }; + } + if ( + typeof operand === 'number' || + typeof operand === 'string' || + typeof operand === 'boolean' + ) { + return { ok: true, value: operand, describe: 'value' }; + } + return { ok: false, missing: '' }; +} + +function evaluateComparison( + operator: ComparisonOperator, + leftRaw: unknown, + rightRaw: unknown, + context: FixedPolicyConditionContext, +): ConditionEvaluation { + const left = resolveOperand(leftRaw, context); + const right = resolveOperand(rightRaw, context); + if (!left.ok || !right.ok) { + const missing: string[] = []; + if (!left.ok) missing.push(left.missing); + if (!right.ok) missing.push(right.missing); + return unavailable(missing); + } + if (operator === '==' || operator === '!=') { + // Strict (in)equality: a type mismatch between two AVAILABLE values is + // a determinate not-equal, not an unavailability. + const equal = left.value === right.value; + return equal === (operator === '==') ? MATCH : NO_MATCH; + } + // Ordering requires two finite numbers; anything else cannot be ordered + // and must not silently collapse to false. + if ( + typeof left.value !== 'number' || + typeof right.value !== 'number' || + !Number.isFinite(left.value) || + !Number.isFinite(right.value) + ) { + const missing: string[] = []; + if (typeof left.value !== 'number' || !Number.isFinite(left.value)) { + missing.push(`${left.describe} (not orderable)`); + } + if (typeof right.value !== 'number' || !Number.isFinite(right.value)) { + missing.push(`${right.describe} (not orderable)`); + } + return unavailable(missing); + } + switch (operator) { + case '>': + return left.value > right.value ? MATCH : NO_MATCH; + case '>=': + return left.value >= right.value ? MATCH : NO_MATCH; + case '<': + return left.value < right.value ? MATCH : NO_MATCH; + case '<=': + return left.value <= right.value ? MATCH : NO_MATCH; + default: { + const exhaustive: never = operator; + return unavailable([`unknown operator ${String(exhaustive)}`]); + } + } +} + +/** + * Evaluate a `when` condition against a context snapshot. Total function — + * never throws, even on structurally malformed input (which startup + * validation rejects; anything that slips through degrades to + * `unavailable`, the fail-safe outcome). + * + * Combinators use strong Kleene logic so `unavailable` propagates only + * when it is actually decisive: `all` with a false branch is false + * regardless of an unavailable sibling; `any` with a true branch is true; + * `!` flips determinate outcomes and passes `unavailable` through. + */ +export function evaluateFixedPolicyCondition( + condition: FixedPolicyCondition, + context: FixedPolicyConditionContext, +): ConditionEvaluation { + // Deliberately treated as unknown: this total function must handle + // malformed nodes anyway, and narrowing the tuple union member-by-member + // buys nothing here. + const node: unknown = condition; + if (!Array.isArray(node) || node.length === 0) { + return unavailable(['']); + } + const head = node[0]; + if (head === 'all' || head === 'any') { + const isAll = head === 'all'; + const missing: string[] = []; + let sawUnavailable = false; + for (let i = 1; i < node.length; i++) { + const result = evaluateFixedPolicyCondition( + node[i] as FixedPolicyCondition, + context, + ); + if (result.outcome === 'unavailable') { + sawUnavailable = true; + missing.push(...result.missingFields); + continue; + } + // Dominant outcomes short-circuit: false for `all`, true for `any`. + if (isAll && result.outcome === 'no_match') return NO_MATCH; + if (!isAll && result.outcome === 'match') return MATCH; + } + if (sawUnavailable) return unavailable(missing); + return isAll ? MATCH : NO_MATCH; + } + if (head === '!') { + if (node.length !== 2) { + return unavailable(['']); + } + const result = evaluateFixedPolicyCondition( + node[1] as FixedPolicyCondition, + context, + ); + if (result.outcome === 'unavailable') return result; + return result.outcome === 'match' ? NO_MATCH : MATCH; + } + if (isComparisonOperator(head) && node.length === 3) { + return evaluateComparison(head, node[1], node[2], context); + } + return unavailable(['']); +} + +function validateOperand( + raw: unknown, + operator: ComparisonOperator, + where: string, + errors: string[], +): void { + if (Array.isArray(raw)) { + if (raw.length !== 2 || raw[0] !== 'field') { + errors.push( + `${where}: field reference must be ["field", ""]`, + ); + return; + } + if (typeof raw[1] !== 'string' || !KNOWN_FIELDS.has(raw[1])) { + errors.push(`${where}: unknown field ${JSON.stringify(raw[1])}`); + } + return; + } + if ( + typeof raw !== 'number' && + typeof raw !== 'string' && + typeof raw !== 'boolean' + ) { + errors.push(`${where}: literal must be a number, string, or boolean`); + return; + } + if (operator !== '==' && operator !== '!=') { + if (typeof raw !== 'number' || !Number.isFinite(raw)) { + errors.push( + `${where}: operator "${operator}" requires a finite numeric literal`, + ); + } + } +} + +/** + * Structural validation for a raw `when` condition (policy design §13 #5), + * run at config-normalization time. Returns a list of human-readable + * errors; empty means valid. + */ +export function validateFixedPolicyCondition( + raw: unknown, + where = 'when', +): string[] { + const errors: string[] = []; + if (isPlainObject(raw)) { + // Catch the pre-expression object form with a pointed migration hint + // instead of a generic type error. + errors.push( + `${where}: condition must be an expression array like ` + + `[">", ["field", "resource.width"], 3000] or ["all", , ...] ` + + `(the {left, operator, right} object form is no longer supported)`, + ); + return errors; + } + if (!Array.isArray(raw) || raw.length === 0 || typeof raw[0] !== 'string') { + errors.push( + `${where}: condition must be an expression array [operator, ...operands]`, + ); + return errors; + } + const head = raw[0]; + if (head === 'all' || head === 'any') { + if (raw.length < 2) { + errors.push( + `${where}: "${head}" requires at least one operand condition`, + ); + return errors; + } + for (let i = 1; i < raw.length; i++) { + errors.push(...validateFixedPolicyCondition(raw[i], `${where}[${i}]`)); + } + return errors; + } + if (head === '!') { + if (raw.length !== 2) { + errors.push(`${where}: "!" takes exactly one operand condition`); + return errors; + } + errors.push(...validateFixedPolicyCondition(raw[1], `${where}[1]`)); + return errors; + } + if (!isComparisonOperator(head)) { + errors.push( + `${where}[0]: unknown operator ${JSON.stringify(head)} (expected one ` + + `of ${COMPARISON_OPERATORS.join(', ')}, all, any, !)`, + ); + return errors; + } + if (raw.length !== 3) { + errors.push(`${where}: comparison "${head}" takes exactly two operands`); + return errors; + } + validateOperand(raw[1], head, `${where}[1]`, errors); + validateOperand(raw[2], head, `${where}[2]`, errors); + return errors; +} diff --git a/packages/core/src/omni/policy/config.test.ts b/packages/core/src/omni/policy/config.test.ts new file mode 100644 index 00000000000..05c57d04200 --- /dev/null +++ b/packages/core/src/omni/policy/config.test.ts @@ -0,0 +1,1340 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { describe, expect, it } from 'vitest'; +import { + DEFAULT_OMNI_PROCESSING_LIMITS, + OmniPolicyConfigError, + normalizeOmniProcessingConfig, +} from './config.js'; +import type { + OmniPolicyToolLookup, + RawOmniProcessingSettings, +} from './config.js'; +import type { MediaPolicyToolDescriptor } from '../../tools/tools.js'; +import type { OmniModality } from '../recognition.js'; +import { STAGING_GRACE_MS } from '../recovery.js'; + +const TUNABLE_SCHEMA = { + type: 'object', + properties: { + maxDimension: { type: 'number', minimum: 1 }, + quality: { type: 'number', minimum: 1, maximum: 100 }, + }, + additionalProperties: false, +}; + +interface ToolStub { + mediaPolicyDescriptor?: MediaPolicyToolDescriptor; + /** Native schema, mirroring DeclarativeTool's public field — the lookup + * contract deliberately avoids the projected `schema` getter. */ + parameterSchema?: unknown; +} + +function makeTool( + inputMediaTypes: OmniModality[], + overrides: Partial = {}, +): ToolStub { + return { + mediaPolicyDescriptor: { + kind: 'media_policy', + inputMediaTypes, + outputs: [ + { kind: 'media', required: true, lossy: true }, + { kind: 'text', role: 'disclosure', required: true }, + ], + settingsSchema: TUNABLE_SCHEMA, + ...overrides, + }, + parameterSchema: { + type: 'object', + properties: { + inputPath: { type: 'string' }, + outputDir: { type: 'string' }, + maxDimension: { type: 'number' }, + quality: { type: 'number' }, + }, + }, + }; +} + +function defaultTools(): Record { + return { + omni_downsample_image: makeTool(['image']), + omni_downscale_video: makeTool(['video']), + omni_downsample_audio: makeTool(['audio']), + }; +} + +function lookup(tools: Record): OmniPolicyToolLookup { + return { getTool: (name) => tools[name] }; +} + +function normalize( + raw: RawOmniProcessingSettings = {}, + tools: Record = defaultTools(), +) { + return normalizeOmniProcessingConfig(raw, lookup(tools)); +} + +describe('normalizeOmniProcessingConfig', () => { + describe('system defaults', () => { + it('normalizes against the REAL degradation tools, not just stubs', async () => { + // The stub lookup above can drift from the shipped tool descriptors; + // this is the startup path every real CLI run takes, so a descriptor + // that fails §13 validation (e.g. a lossy output without a declared + // disclosure) must fail HERE, not at first launch. + const [image, video, audio] = await Promise.all([ + import('./tools/downsample-image.js'), + import('./tools/downscale-video.js'), + import('./tools/downsample-audio.js'), + ]); + const real: Record = { + omni_downsample_image: new image.OmniDownsampleImageTool({}), + omni_downscale_video: new video.OmniDownscaleVideoTool({}), + omni_downsample_audio: new audio.OmniDownsampleAudioTool({}), + }; + const config = normalize({}, real); + expect(config.fixedPolicies).toHaveLength(0); + expect(config.transportGuardPolicies).toHaveLength(3); + }); + + it('registers no default fixed policies: zero config → zero preprocessing (D7)', () => { + // The upstream design gives fixedPolicies pure user-experiment + // semantics: with no configuration, NOTHING may trigger below + // transport limits. Only the transport guard is always-on. + const config = normalize(); + expect(config.fixedPolicies).toEqual([]); + }); + + it('produces the three default guard policies without when, stage transport_guard', () => { + const config = normalize(); + expect(config.transportGuardPolicies.map((p) => p.id).sort()).toEqual([ + 'audio-downsample', + 'image-downsample', + 'video-downscale', + ]); + for (const policy of config.transportGuardPolicies) { + expect(policy.when).toBeUndefined(); + expect(policy.stage).toBe('transport_guard'); + expect(policy.output.source).toBe('omit'); + } + }); + + it('defaults limits per policy design §12.2', () => { + expect(normalize().limits).toEqual({ + maxConcurrentResources: 1, + reservedOutputTokens: 8192, + maxLineageDepth: 8, + maxPolicyRunsPerRoot: 64, + maxArtifactsPerRoot: 256, + maxDerivedBytesPerRoot: 1073741824, + maxTransportPasses: 3, + }); + expect(normalize().limits).toEqual(DEFAULT_OMNI_PROCESSING_LIMITS); + }); + }); + + describe('id-merge semantics', () => { + it('rejects a "__proto__" policy id instead of silently dropping it', () => { + // JSON.parse produces "__proto__" as an ordinary own key; a plain + // object-spread merge would route it through the prototype setter and + // the entry would vanish without a diagnostic. The null-prototype + // merge map keeps it as a real key so the id pattern rejects it. + expect(() => + normalize({ + fixedPolicies: JSON.parse( + '{"__proto__": {"mediaTypes": ["image"], "toolName": "omni_downsample_image"}}', + ), + }), + ).toThrow(/__proto__: policy id must match/); + }); + + it('accepts a null tombstone with no matching entry (no fixed defaults exist)', () => { + const config = normalize({ + fixedPolicies: { 'image-downsample': null }, + }); + expect(config.fixedPolicies).toEqual([]); + }); + + it('normalizes a user fixed policy with full defaults applied', () => { + const config = normalize({ + fixedPolicies: { + 'image-downsample': { + mediaTypes: ['image'], + toolName: 'omni_downsample_image', + arguments: { maxDimension: 1024 }, + }, + }, + }); + const image = config.fixedPolicies.find( + (p) => p.id === 'image-downsample', + ); + expect(image).toEqual({ + id: 'image-downsample', + priority: 0, + mediaTypes: ['image'], + origins: ['user', 'tool'], + when: undefined, + onConditionUnavailable: 'skip', + toolName: 'omni_downsample_image', + arguments: { maxDimension: 1024 }, + maxRunsPerLineage: 1, + onFailure: 'continue', + output: { + reprocessMedia: false, + source: 'omit', + artifacts: { '*': 'include' }, + }, + stage: 'preprocessing', + }); + }); + + it('replaces a default guard entry wholesale (no field-level merge)', () => { + // Whole-entry replacement: the override does NOT inherit the + // default's toolName, so omitting it must be a validation error — + // a field-level merge would inherit it and pass. + expect(() => + normalize({ + transportGuardPolicies: { + 'image-downsample': { mediaTypes: ['image'] }, + }, + }), + ).toThrow( + 'omni.processing.transportGuard.policies.image-downsample.toolName: ' + + 'must be a non-empty string', + ); + const config = normalize({ + transportGuardPolicies: { + 'image-downsample': { + mediaTypes: ['image'], + toolName: 'omni_downsample_image', + arguments: { maxDimension: 1024 }, + }, + }, + }); + const image = config.transportGuardPolicies.find( + (p) => p.id === 'image-downsample', + ); + expect(image?.arguments).toEqual({ maxDimension: 1024 }); + }); + + it('accepts user fixed policies (the only preprocessing source)', () => { + const config = normalize({ + fixedPolicies: { + 'my-policy': { + priority: 5, + mediaTypes: ['image'], + toolName: 'omni_downsample_image', + }, + }, + }); + expect(config.fixedPolicies).toHaveLength(1); + const mine = config.fixedPolicies.find((p) => p.id === 'my-policy'); + expect(mine?.priority).toBe(5); + expect(mine?.stage).toBe('preprocessing'); + }); + + it('rejects transport-guard tombstones (the guard is mandatory)', () => { + expect(() => + normalize({ transportGuardPolicies: { 'image-downsample': null } }), + ).toThrow( + 'omni.processing.transportGuard.policies.image-downsample: ' + + 'transport guard policies cannot be removed (the guard is ' + + 'mandatory); override the entry instead', + ); + }); + + it('rejects non-object policy maps', () => { + expect(() => normalize({ fixedPolicies: ['nope'] })).toThrow( + 'omni.processing.fixedPolicies: must be an object map of policy id → policy', + ); + expect(() => + normalize({ fixedPolicies: { bad: 'string' as never } }), + ).toThrow( + 'omni.processing.fixedPolicies.bad: must be an object (or null to remove a default)', + ); + }); + }); + + describe('policy entry validation', () => { + it('rejects unknown keys (§13 #1)', () => { + expect(() => + normalize({ + fixedPolicies: { + p: { + mediaTypes: ['image'], + toolName: 'omni_downsample_image', + retries: 3, + }, + }, + }), + ).toThrow('omni.processing.fixedPolicies.p: unknown key "retries"'); + }); + + it('rejects malformed policy ids', () => { + expect(() => + normalize({ + fixedPolicies: { + 'has space': { + mediaTypes: ['image'], + toolName: 'omni_downsample_image', + }, + }, + }), + ).toThrow(OmniPolicyConfigError); + }); + + it('rejects empty or unknown mediaTypes (§13 #3)', () => { + expect(() => + normalize({ + fixedPolicies: { + p: { mediaTypes: [], toolName: 'omni_downsample_image' }, + }, + }), + ).toThrow( + 'omni.processing.fixedPolicies.p.mediaTypes: must be a non-empty array', + ); + expect(() => + normalize({ + fixedPolicies: { + p: { mediaTypes: ['text'], toolName: 'omni_downsample_image' }, + }, + }), + ).toThrow( + 'omni.processing.fixedPolicies.p.mediaTypes: unknown modality "text" ' + + '(expected image, video, audio)', + ); + }); + + it('rejects unknown origins (§13 #4)', () => { + expect(() => + normalize({ + fixedPolicies: { + p: { + mediaTypes: ['image'], + origins: ['model'], + toolName: 'omni_downsample_image', + }, + }, + }), + ).toThrow( + 'omni.processing.fixedPolicies.p.origins: unknown origin "model" ' + + '(expected user, tool, policy)', + ); + }); + + it('rejects onConditionUnavailable "abortTurn" with an explicit not-yet-supported error', () => { + expect(() => + normalize({ + fixedPolicies: { + p: { + mediaTypes: ['image'], + toolName: 'omni_downsample_image', + onConditionUnavailable: 'abortTurn', + }, + }, + }), + ).toThrow(/"abortTurn" is not yet supported/); + }); + + it('rejects invalid onFailure', () => { + expect(() => + normalize({ + fixedPolicies: { + p: { + mediaTypes: ['image'], + toolName: 'omni_downsample_image', + onFailure: 'retry', + }, + }, + }), + ).toThrow( + 'omni.processing.fixedPolicies.p.onFailure: must be "continue" or "abort" (got "retry")', + ); + }); + + it('rejects non-positive maxRunsPerLineage', () => { + expect(() => + normalize({ + fixedPolicies: { + p: { + mediaTypes: ['image'], + toolName: 'omni_downsample_image', + maxRunsPerLineage: 0, + }, + }, + }), + ).toThrow( + 'omni.processing.fixedPolicies.p.maxRunsPerLineage: must be a positive integer (got 0)', + ); + }); + + it('rejects unknown output keys and illegal output.source (§13 #23)', () => { + expect(() => + normalize({ + fixedPolicies: { + p: { + mediaTypes: ['image'], + toolName: 'omni_downsample_image', + output: { keepBoth: true }, + }, + }, + }), + ).toThrow( + 'omni.processing.fixedPolicies.p.output: unknown key "keepBoth"', + ); + expect(() => + normalize({ + fixedPolicies: { + p: { + mediaTypes: ['image'], + toolName: 'omni_downsample_image', + output: { source: 'drop' }, + }, + }, + }), + ).toThrow( + 'omni.processing.fixedPolicies.p.output.source: must be "keep" or "omit" (got "drop")', + ); + }); + + it('allows output.source "keep" for preprocessing policies', () => { + const config = normalize({ + fixedPolicies: { + p: { + mediaTypes: ['image'], + toolName: 'omni_downsample_image', + origins: ['user', 'tool', 'policy'], + output: { source: 'keep', reprocessMedia: true }, + }, + }, + }); + const p = config.fixedPolicies.find((x) => x.id === 'p'); + expect(p?.output).toEqual({ + reprocessMedia: true, + source: 'keep', + artifacts: { '*': 'include' }, + }); + }); + + it('rejects reprocessMedia when no policy in the set accepts origin "policy"', () => { + // Derivatives re-enter matching with origin 'policy'; with no policy + // accepting that origin, reprocessMedia can never take effect. + expect(() => + normalize({ + fixedPolicies: { + p: { + mediaTypes: ['image'], + toolName: 'omni_downsample_image', + output: { source: 'keep', reprocessMedia: true }, + }, + }, + }), + ).toThrow( + 'omni.processing.fixedPolicies: "p" sets output.reprocessMedia, ' + + 'but no policy in this set accepts origin "policy"', + ); + }); + + it('accepts reprocessMedia when ANOTHER policy in the set accepts origin "policy"', () => { + expect(() => + normalize({ + fixedPolicies: { + p: { + mediaTypes: ['image'], + toolName: 'omni_downsample_image', + output: { source: 'keep', reprocessMedia: true }, + }, + q: { + mediaTypes: ['image'], + toolName: 'omni_downsample_image', + origins: ['policy'], + }, + }, + }), + ).not.toThrow(); + }); + + it('applies the inert-reprocessMedia check to the transport-guard set independently', () => { + expect(() => + normalize({ + transportGuardPolicies: { + g: { + mediaTypes: ['image'], + toolName: 'omni_downsample_image', + output: { reprocessMedia: true }, + }, + }, + }), + ).toThrow( + 'omni.processing.transportGuard.policies: "g" sets ' + + 'output.reprocessMedia, but no policy in this set accepts ' + + 'origin "policy"', + ); + }); + + it('rejects invalid when-conditions via the shared validator (§13 #5)', () => { + expect(() => + normalize({ + fixedPolicies: { + p: { + mediaTypes: ['image'], + toolName: 'omni_downsample_image', + when: ['>', ['field', 'resource.nonexistent'], 1], + }, + }, + }), + ).toThrow(/omni\.processing\.fixedPolicies\.p\.when/); + }); + + it('preserves a valid when-condition verbatim through normalization (D7)', () => { + // `when` is preprocessing's ONLY trigger mechanism: a normalization + // regression that drops or rewrites it would silently widen every + // user condition to ALL matching resources. Pin the round-trip. + const when = [ + 'all', + ['>', ['field', 'resource.sizeBytes'], 10_000_000], + ['>=', ['field', 'session.availableContextTokens'], 4096], + ]; + const config = normalize({ + fixedPolicies: { + p: { + mediaTypes: ['image'], + toolName: 'omni_downsample_image', + when, + }, + }, + }); + const p = config.fixedPolicies.find((x) => x.id === 'p'); + expect(p?.when).toEqual(when); + }); + }); + + describe('output.artifacts selectors (§13 #22/#24)', () => { + /** Media tool whose descriptor declares producible mime types, so + * `kind:` selectors have something to match. */ + const mediaToolWithMimes = () => + makeTool(['image'], { + outputs: [ + { + kind: 'media', + role: 'preview', + mimeTypes: ['image/jpeg'], + required: true, + lossy: true, + }, + { kind: 'text', role: 'disclosure', required: true }, + ], + }); + + /** Transcript-protocol tool (§6.2): bounded UTF-8 text/plain file. */ + const transcribeLikeTool = () => + makeTool(['audio'], { + outputs: [ + { + kind: 'file', + role: 'transcript', + mimeTypes: ['text/plain'], + required: true, + lossy: true, + }, + { kind: 'text', role: 'disclosure', required: true }, + ], + }); + + const withTool = (tool: ToolStub) => ({ + ...defaultTools(), + tool_under_test: tool, + }); + + const policyWith = ( + artifacts: Record, + tool: ToolStub, + mediaTypes: OmniModality[] = ['image'], + ) => + normalize( + { + fixedPolicies: { + p: { + mediaTypes, + toolName: 'tool_under_test', + output: { artifacts }, + }, + }, + }, + withTool(tool), + ); + + it('defaults an unconfigured artifacts map to include-all', () => { + const config = normalize({ + fixedPolicies: { + p: { mediaTypes: ['image'], toolName: 'omni_downsample_image' }, + }, + }); + expect(config.fixedPolicies[0].output.artifacts).toEqual({ + '*': 'include', + }); + }); + + it('preserves an explicit selector map verbatim', () => { + const config = policyWith( + { 'role:preview': 'include', 'kind:image': 'retain', '*': 'retain' }, + mediaToolWithMimes(), + ); + expect(config.fixedPolicies[0].output.artifacts).toEqual({ + 'role:preview': 'include', + 'kind:image': 'retain', + '*': 'retain', + }); + }); + + it('rejects actions other than include/retain', () => { + expect(() => policyWith({ '*': 'drop' }, mediaToolWithMimes())).toThrow( + 'omni.processing.fixedPolicies.p.output.artifacts["*"]: must be "include" or "retain" (got "drop")', + ); + }); + + it('rejects unknown selector shapes', () => { + expect(() => + policyWith({ preview: 'include' }, mediaToolWithMimes()), + ).toThrow( + 'omni.processing.fixedPolicies.p.output.artifacts["preview"]: unknown selector (expected "*", "kind:", or "role:")', + ); + }); + + it('rejects unknown kind targets', () => { + expect(() => + policyWith({ 'kind:text': 'include' }, mediaToolWithMimes()), + ).toThrow(/unknown artifact kind "text"/); + }); + + it('rejects malformed role tokens', () => { + expect(() => + policyWith({ 'role:no spaces!': 'include' }, mediaToolWithMimes()), + ).toThrow(/invalid role token "no spaces!"/); + }); + + it('rejects a kind selector the descriptor cannot produce (§13 #22)', () => { + expect(() => + policyWith({ 'kind:video': 'retain' }, mediaToolWithMimes()), + ).toThrow( + 'omni.processing.fixedPolicies.p.output.artifacts["kind:video"]: tool "tool_under_test" declares no output of kind "video"', + ); + }); + + it('rejects a role selector no artifact output declares (§13 #22)', () => { + expect(() => + policyWith({ 'role:thumbnail': 'include' }, mediaToolWithMimes()), + ).toThrow( + 'omni.processing.fixedPolicies.p.output.artifacts["role:thumbnail"]: tool "tool_under_test" declares no artifact output with role "thumbnail"', + ); + }); + + it('accepts role:transcript and kind:file against a transcript-protocol descriptor (§13 #24)', () => { + const config = policyWith( + { 'role:transcript': 'include', 'kind:file': 'include' }, + transcribeLikeTool(), + ['audio'], + ); + expect(config.fixedPolicies[0].output.artifacts).toEqual({ + 'role:transcript': 'include', + 'kind:file': 'include', + }); + }); + + it('rejects role:transcript when the declared output is not bounded text/plain file (§13 #24)', () => { + const wrongMime = makeTool(['audio'], { + outputs: [ + { + kind: 'file', + role: 'transcript', + mimeTypes: ['text/markdown'], + required: true, + lossy: true, + }, + { kind: 'text', role: 'disclosure', required: true }, + ], + }); + expect(() => + policyWith({ 'role:transcript': 'include' }, wrongMime, ['audio']), + ).toThrow( + 'omni.processing.fixedPolicies.p.output.artifacts["role:transcript"]: a transcript selector must point at a bounded UTF-8 text/plain file output, but tool "tool_under_test" declares role "transcript" differently', + ); + + const mediaTranscript = makeTool(['audio'], { + outputs: [ + { + kind: 'media', + role: 'transcript', + mimeTypes: ['audio/wav'], + required: true, + lossy: true, + }, + { kind: 'text', role: 'disclosure', required: true }, + ], + }); + expect(() => + policyWith({ 'role:transcript': 'include' }, mediaTranscript, [ + 'audio', + ]), + ).toThrow(/a transcript selector must point at a bounded UTF-8/); + }); + + it('accepts the REAL transcribe tool as a fixed-policy target with role:transcript', async () => { + const { OmniTranscribeAudioTool } = await import( + './tools/transcribe-audio.js' + ); + const tools: Record = { + ...defaultTools(), + omni_transcribe_audio: new OmniTranscribeAudioTool({}), + }; + const config = normalize( + { + fixedPolicies: { + 'audio-transcribe': { + mediaTypes: ['audio'], + toolName: 'omni_transcribe_audio', + output: { + source: 'omit', + artifacts: { 'role:transcript': 'include' }, + }, + }, + }, + }, + tools, + ); + expect(config.fixedPolicies[0].output).toEqual({ + reprocessMedia: false, + source: 'omit', + artifacts: { 'role:transcript': 'include' }, + }); + }); + }); + + describe('tool reference validation (§13 #6/#8/#14)', () => { + it('rejects a missing toolName', () => { + expect(() => + normalize({ fixedPolicies: { p: { mediaTypes: ['image'] } } }), + ).toThrow( + 'omni.processing.fixedPolicies.p.toolName: must be a non-empty string', + ); + }); + + it('rejects an unregistered tool (covers excluded tools too)', () => { + expect(() => + normalize({ + fixedPolicies: { + p: { mediaTypes: ['image'], toolName: 'no_such_tool' }, + }, + }), + ).toThrow( + 'omni.processing.fixedPolicies.p.toolName: tool "no_such_tool" is ' + + 'not registered (unknown name, or excluded by tool filtering)', + ); + }); + + it('rejects a registered tool without a media_policy descriptor', () => { + const tools = defaultTools(); + tools['read_file'] = { parameterSchema: {} }; + expect(() => + normalize( + { + fixedPolicies: { + p: { mediaTypes: ['image'], toolName: 'read_file' }, + }, + }, + tools, + ), + ).toThrow( + 'omni.processing.fixedPolicies.p.toolName: tool "read_file" is not ' + + 'a media policy tool (no media_policy descriptor)', + ); + }); + + it('rejects a tool declaring no required output', () => { + const tools = defaultTools(); + tools['weak_tool'] = makeTool(['image'], { + outputs: [{ kind: 'media', required: false, lossy: false }], + }); + expect(() => + normalize( + { + fixedPolicies: { + p: { mediaTypes: ['image'], toolName: 'weak_tool' }, + }, + }, + tools, + ), + ).toThrow(/declares no required output/); + }); + + it('rejects a lossy tool without a disclosure output (§13 #8)', () => { + const tools = defaultTools(); + tools['sneaky_tool'] = makeTool(['image'], { + outputs: [{ kind: 'media', required: true, lossy: true }], + }); + expect(() => + normalize( + { + fixedPolicies: { + p: { mediaTypes: ['image'], toolName: 'sneaky_tool' }, + }, + }, + tools, + ), + ).toThrow( + 'omni.processing.fixedPolicies.p.toolName: tool "sneaky_tool" ' + + 'declares a lossy media output but no disclosure text output', + ); + }); + + it('rejects mediaTypes the tool does not accept', () => { + expect(() => + normalize({ + fixedPolicies: { + p: { + mediaTypes: ['image', 'video'], + toolName: 'omni_downsample_image', + }, + }, + }), + ).toThrow( + 'omni.processing.fixedPolicies.p.mediaTypes: tool ' + + '"omni_downsample_image" does not accept "video" input (accepts image)', + ); + }); + }); + + describe('fixed arguments validation (§13 #11)', () => { + it('rejects reserved io keys in arguments', () => { + expect(() => + normalize({ + fixedPolicies: { + p: { + mediaTypes: ['image'], + toolName: 'omni_downsample_image', + arguments: { inputPath: '/tmp/x.png' }, + }, + }, + }), + ).toThrow( + 'omni.processing.fixedPolicies.p.arguments: "inputPath" is injected ' + + 'by the orchestrator per invocation and must not be configured', + ); + }); + + it('validates arguments against the settingsSchema (io-stripped)', () => { + expect(() => + normalize({ + fixedPolicies: { + p: { + mediaTypes: ['image'], + toolName: 'omni_downsample_image', + arguments: { bogus: true }, + }, + }, + }), + ).toThrow(/omni\.processing\.fixedPolicies\.p\.arguments/); + // Valid tunables pass through untouched. + const config = normalize({ + fixedPolicies: { + p: { + mediaTypes: ['image'], + toolName: 'omni_downsample_image', + arguments: { maxDimension: 800, quality: 70 }, + }, + }, + }); + expect(config.fixedPolicies.find((x) => x.id === 'p')?.arguments).toEqual( + { maxDimension: 800, quality: 70 }, + ); + }); + }); + + describe('transport guard rules (§13 #15-#17)', () => { + it('rejects guard policies declaring when', () => { + expect(() => + normalize({ + transportGuardPolicies: { + 'image-downsample': { + mediaTypes: ['image'], + toolName: 'omni_downsample_image', + when: ['>', ['field', 'resource.width'], 1], + }, + }, + }), + ).toThrow( + 'omni.processing.transportGuard.policies.image-downsample.when: ' + + 'transport guard policies must not declare "when" (they run ' + + 'exactly when transport limits are exceeded)', + ); + }); + + it('rejects guard policies with output.source "keep"', () => { + expect(() => + normalize({ + transportGuardPolicies: { + 'image-downsample': { + mediaTypes: ['image'], + toolName: 'omni_downsample_image', + output: { source: 'keep' }, + }, + }, + }), + ).toThrow( + 'omni.processing.transportGuard.policies.image-downsample.output.source: ' + + 'transport guard policies must use "omit" (the over-limit source ' + + 'cannot stay in the delivery set)', + ); + }); + + it('rejects a merged guard set that does not cover all three modalities', () => { + const tools = defaultTools(); + // Point every guard entry at image only → video+audio uncovered. + expect(() => + normalize( + { + transportGuardPolicies: { + 'video-downscale': { + mediaTypes: ['image'], + toolName: 'omni_downsample_image', + }, + 'audio-downsample': { + mediaTypes: ['image'], + toolName: 'omni_downsample_image', + }, + }, + }, + tools, + ), + ).toThrow( + 'omni.processing.transportGuard.policies: no guard policy covers ' + + 'video, audio — the merged set must cover image, video, and audio', + ); + }); + }); + + describe('limits (§12.2)', () => { + it('merges overrides over defaults', () => { + const config = normalize({ limits: { maxLineageDepth: 3 } }); + expect(config.limits).toEqual({ + ...DEFAULT_OMNI_PROCESSING_LIMITS, + maxLineageDepth: 3, + }); + }); + + it('rejects unknown limit keys', () => { + expect(() => normalize({ limits: { maxFoo: 1 } })).toThrow( + 'omni.processing.limits: unknown key "maxFoo"', + ); + }); + + it('rejects non-positive-integer values', () => { + expect(() => normalize({ limits: { maxLineageDepth: 0 } })).toThrow( + 'omni.processing.limits.maxLineageDepth: must be a positive integer (got 0)', + ); + expect(() => normalize({ limits: { maxLineageDepth: 2.5 } })).toThrow( + 'omni.processing.limits.maxLineageDepth: must be a positive integer (got 2.5)', + ); + }); + + it('allows reservedOutputTokens of zero', () => { + const config = normalize({ limits: { reservedOutputTokens: 0 } }); + expect(config.limits.reservedOutputTokens).toBe(0); + }); + }); + + describe('channel caps (§13 #18/#19)', () => { + it('rejects maxUploadFileBytes above the 1 GiB channel cap', () => { + expect(() => normalize({ maxUploadFileBytes: 1073741824 + 1 })).toThrow( + 'omni.processing.transportGuard.maxUploadFileBytes: 1073741825 ' + + 'exceeds the DashScope per-file upload cap (1073741824)', + ); + expect(() => normalize({ maxUploadFileBytes: 1073741824 })).not.toThrow(); + }); + + it('rejects urlTtlHours outside 0..48', () => { + expect(() => normalize({ urlTtlHours: 49 })).toThrow( + 'omni.delivery.upload.urlTtlHours: must be a number between 0 and 48 (got 49)', + ); + expect(() => normalize({ urlTtlHours: -1 })).toThrow( + OmniPolicyConfigError, + ); + expect(() => normalize({ urlTtlHours: 48 })).not.toThrow(); + expect(() => normalize({ urlTtlHours: 0 })).not.toThrow(); + }); + + it('rejects a non-numeric or negative maxEstimatedTokens (fail-open guard)', () => { + // guard.ts compares with `<=`/`>`: a string would make both false and + // silently disable the token guard — must abort startup instead. + expect(() => + normalize({ maxEstimatedTokens: 'abc' as unknown as number }), + ).toThrow( + 'omni.processing.transportGuard.maxEstimatedTokens: must be a ' + + 'finite number >= 0, where 0 disables the token guard (got "abc")', + ); + expect(() => + normalize({ maxEstimatedTokens: true as unknown as number }), + ).toThrow(OmniPolicyConfigError); + expect(() => normalize({ maxEstimatedTokens: -1 })).toThrow( + OmniPolicyConfigError, + ); + expect(() => + normalize({ maxEstimatedTokens: Number.POSITIVE_INFINITY }), + ).toThrow(OmniPolicyConfigError); + expect(() => normalize({ maxEstimatedTokens: 0 })).not.toThrow(); + expect(() => normalize({ maxEstimatedTokens: 262144 })).not.toThrow(); + }); + }); + + describe('policyTools validation (§13 #7/#20/#21)', () => { + it('accepts null tombstones and valid entries', () => { + expect(() => + normalize({ + policyTools: { + omni_downsample_image: null, + omni_downscale_video: { + settings: { maxDimension: 640 }, + runtime: { timeoutMs: 30000 }, + }, + }, + }), + ).not.toThrow(); + }); + + it('rejects entries naming a non-media-policy tool', () => { + expect(() => + normalize({ policyTools: { no_such_tool: { settings: {} } } }), + ).toThrow( + 'omni.processing.policyTools.no_such_tool: "no_such_tool" is not a ' + + 'registered media policy tool', + ); + }); + + it('rejects unknown keys at every level of an entry (§13 #1)', () => { + // Typos like "settigns" would otherwise read as absent downstream + // and the intended configuration would silently never take effect. + expect(() => + normalize({ + policyTools: { omni_downsample_image: { settigns: {} } as never }, + }), + ).toThrow( + 'omni.processing.policyTools.omni_downsample_image: unknown key "settigns"', + ); + expect(() => + normalize({ + policyTools: { + omni_downsample_image: { runtime: { timeout: 30000 } }, + }, + }), + ).toThrow( + 'omni.processing.policyTools.omni_downsample_image.runtime: ' + + 'unknown key "timeout"', + ); + expect(() => + normalize({ + policyTools: { + omni_downsample_image: { modelAccess: { lockedArgs: {} } as never }, + }, + }), + ).toThrow( + 'omni.processing.policyTools.omni_downsample_image.modelAccess: ' + + 'unknown key "lockedArgs"', + ); + }); + + it('validates settings against the settingsSchema (§13 #7)', () => { + expect(() => + normalize({ + policyTools: { + omni_downsample_image: { settings: { bogus: 1 } }, + }, + }), + ).toThrow( + /omni\.processing\.policyTools\.omni_downsample_image\.settings/, + ); + }); + + it('rejects non-positive runtime.timeoutMs', () => { + expect(() => + normalize({ + policyTools: { + omni_downsample_image: { runtime: { timeoutMs: -5 } }, + }, + }), + ).toThrow( + 'omni.processing.policyTools.omni_downsample_image.runtime.timeoutMs: ' + + 'must be a positive integer (got -5)', + ); + }); + + it('caps runtime.timeoutMs below the staging sweep grace window (cross-file invariant with recovery §5)', () => { + // A tool allowed to run for >= STAGING_GRACE_MS could have its live + // staging directory classified as crash leftovers and deleted + // mid-run by another process's startup sweep. Pin BOTH sides of the + // boundary so removing, inverting (`<=`), or relocating the cap + // fails a test instead of shipping green. + expect(() => + normalize({ + policyTools: { + omni_downsample_image: { + runtime: { timeoutMs: STAGING_GRACE_MS }, + }, + }, + }), + ).toThrow( + 'omni.processing.policyTools.omni_downsample_image.runtime.timeoutMs: ' + + `must be below the staging sweep grace window (${STAGING_GRACE_MS}ms) ` + + `so a live invocation's staging directory is never reclaimed mid-run`, + ); + expect(() => + normalize({ + policyTools: { + omni_downsample_image: { + runtime: { timeoutMs: STAGING_GRACE_MS - 1 }, + }, + }, + }), + ).not.toThrow(); + }); + + it('rejects overlapping defaultArguments and lockedArguments (§13 #21)', () => { + expect(() => + normalize({ + policyTools: { + omni_downsample_image: { + modelAccess: { + defaultArguments: { quality: 80 }, + lockedArguments: { quality: 60 }, + }, + }, + }, + }), + ).toThrow( + 'omni.processing.policyTools.omni_downsample_image.modelAccess: ' + + '"quality" present in both defaultArguments and lockedArguments', + ); + }); + + it('rejects a defaultArguments key the native schema does not declare', () => { + expect(() => + normalize({ + policyTools: { + omni_downsample_image: { + modelAccess: { + defaultArguments: { sharpen: 2 }, + }, + }, + }, + }), + ).toThrow( + /omni\.processing\.policyTools\.omni_downsample_image\.modelAccess\.defaultArguments/, + ); + }); + + it('rejects a lockedArguments value the native sub-schema refuses', () => { + expect(() => + normalize({ + policyTools: { + omni_downsample_image: { + modelAccess: { + lockedArguments: { quality: 'very high' }, + }, + }, + }, + }), + ).toThrow( + /omni\.processing\.policyTools\.omni_downsample_image\.modelAccess\.lockedArguments/, + ); + }); + + it('accepts schema-valid partial defaultArguments and lockedArguments', () => { + expect(() => + normalize({ + policyTools: { + omni_downsample_image: { + modelAccess: { + defaultArguments: { quality: 80 }, + lockedArguments: { maxDimension: 1024 }, + }, + }, + }, + }), + ).not.toThrow(); + }); + + it('validates locked/operator-only arguments against a REAL tool (whose `schema` getter hides them)', async () => { + // Regression: validation must read the tool's NATIVE parameterSchema. + // A real BaseMediaPolicyTool's `schema` getter is the model-visible + // projection, which strips lockedArguments and operatorOnlyParams + // keys — validated against THAT, every legitimate locked/operator + // config would abort startup with "must NOT have additional + // properties". The stub lookup can't catch this (its shape is + // static), so this test wires real tool instances whose config view + // serves the very settings under validation. + const raw: RawOmniProcessingSettings = { + policyTools: { + omni_downsample_image: { + modelAccess: { + enabled: true, + lockedArguments: { quality: 80 }, + }, + }, + omni_transcribe_audio: { + modelAccess: { + enabled: true, + defaultArguments: { baseUrl: 'https://asr.example/v1' }, + }, + }, + }, + }; + const view = { + getOmniPolicyToolsSettings: () => raw.policyTools, + }; + const [image, transcribe] = await Promise.all([ + import('./tools/downsample-image.js'), + import('./tools/transcribe-audio.js'), + ]); + const real: Record = { + ...defaultTools(), + omni_downsample_image: new image.OmniDownsampleImageTool(view), + omni_transcribe_audio: new transcribe.OmniTranscribeAudioTool(view), + }; + expect(() => normalize(raw, real)).not.toThrow(); + }); + + it('rejects parameterSchema properties absent from the native schema (§13 #20)', () => { + expect(() => + normalize({ + policyTools: { + omni_downsample_image: { + modelAccess: { + parameterSchema: { + properties: { quality: {}, sharpen: {} }, + }, + }, + }, + }, + }), + ).toThrow( + 'omni.processing.policyTools.omni_downsample_image.modelAccess.parameterSchema: ' + + '"sharpen" not present in the tool\'s native schema (projection may only narrow)', + ); + }); + + it('accepts a narrowing-only parameterSchema', () => { + expect(() => + normalize({ + policyTools: { + omni_downsample_image: { + modelAccess: { + parameterSchema: { properties: { quality: {} } }, + }, + }, + }, + }), + ).not.toThrow(); + }); + + describe('constraint-value narrowing (§11.2: 不能扩大类型、枚举、范围)', () => { + /** Tool whose native schema carries real constraints to loosen. */ + const constrainedTools = (): Record => { + const tools = defaultTools(); + tools['omni_downsample_image'] = { + ...makeTool(['image']), + parameterSchema: { + type: 'object', + properties: { + inputPath: { type: 'string' }, + outputDir: { type: 'string' }, + quality: { type: 'number', minimum: 1, maximum: 100 }, + format: { type: 'string', enum: ['jpeg', 'webp'] }, + tags: { type: 'array', minItems: 1, maxItems: 4 }, + }, + }, + }; + return tools; + }; + const withProjection = ( + prop: string, + override: Record, + ) => + normalize( + { + policyTools: { + omni_downsample_image: { + modelAccess: { + parameterSchema: { properties: { [prop]: override } }, + }, + }, + }, + }, + constrainedTools(), + ); + const at = + 'omni.processing.policyTools.omni_downsample_image.modelAccess.' + + 'parameterSchema.properties.'; + + it('rejects an override raising the native maximum (probe case)', () => { + expect(() => withProjection('quality', { maximum: 200 })).toThrow( + `${at}quality: the upper bound loosens the native one ` + + '(200 vs native 100) (projection may only narrow)', + ); + }); + + it('rejects an override lowering the native minimum', () => { + expect(() => withProjection('quality', { minimum: 0 })).toThrow( + `${at}quality: the lower bound loosens the native one ` + + '(0 vs native 1) (projection may only narrow)', + ); + }); + + it('rejects an enum override adding values outside the native enum', () => { + expect(() => + withProjection('format', { enum: ['jpeg', 'png'] }), + ).toThrow( + `${at}format: "enum" adds values the native enum does not allow ` + + '("png") (projection may only narrow)', + ); + }); + + it('rejects an override changing the native type', () => { + expect(() => withProjection('quality', { type: 'string' })).toThrow( + `${at}quality: "type" changes the native type ` + + '("string" vs native "number") (projection may only narrow)', + ); + }); + + it('rejects maxItems above the native cap', () => { + expect(() => withProjection('tags', { maxItems: 10 })).toThrow( + `${at}tags: "maxItems" loosens the native constraint ` + + '(10 vs native 4) (projection may only narrow)', + ); + }); + + it('accepts genuinely narrowing overrides', () => { + expect(() => + withProjection('quality', { + type: 'integer', // integer narrows number + minimum: 10, + maximum: 80, + }), + ).not.toThrow(); + expect(() => + withProjection('format', { enum: ['jpeg'] }), + ).not.toThrow(); + expect(() => + withProjection('tags', { minItems: 2, maxItems: 3 }), + ).not.toThrow(); + // Adding a bound where the native schema has none narrows too. + expect(() => + withProjection('inputPath', { minLength: 1 }), + ).not.toThrow(); + }); + }); + }); +}); diff --git a/packages/core/src/omni/policy/config.ts b/packages/core/src/omni/policy/config.ts new file mode 100644 index 00000000000..180afae71e8 --- /dev/null +++ b/packages/core/src/omni/policy/config.ts @@ -0,0 +1,1034 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import type { MediaPolicyToolDescriptor } from '../../tools/tools.js'; +import { ToolNames } from '../../tools/tool-names.js'; +import { SchemaValidator } from '../../utils/schemaValidator.js'; +import type { OmniModality } from '../recognition.js'; +import { STAGING_GRACE_MS } from '../recovery.js'; +import { validateFixedPolicyCondition } from './conditions.js'; +import type { FixedPolicyCondition } from './conditions.js'; +import { isPlainRecord } from './types.js'; +import type { + FixedPolicyOrigin, + NormalizedFixedPolicy, + NormalizedOmniProcessingConfig, + NormalizedOmniProcessingLimits, + OmniPolicyToolsSettings, +} from './types.js'; + +/** + * Startup normalization of `omni.processing` (policy design §13 applicable + * subset — see the S4 mapping doc §7). Raw settings enter, a fully + * defaulted and validated {@link NormalizedOmniProcessingConfig} leaves; + * any violation throws {@link OmniPolicyConfigError} and MUST abort + * startup — a mis-configured guard must never degrade into delivering + * over-limit media. + */ + +/** A configuration error in `omni.processing.*`. Startup-fatal. */ +export class OmniPolicyConfigError extends Error { + constructor(message: string) { + super(message); + this.name = 'OmniPolicyConfigError'; + } +} + +/** Per-root derivation budget defaults (policy design §12.2). */ +export const DEFAULT_OMNI_PROCESSING_LIMITS: NormalizedOmniProcessingLimits = { + maxConcurrentResources: 1, + reservedOutputTokens: 8192, + maxLineageDepth: 8, + maxPolicyRunsPerRoot: 64, + maxArtifactsPerRoot: 256, + maxDerivedBytesPerRoot: 1024 * 1024 * 1024, + maxTransportPasses: 3, +}; + +const GIB = 1024 * 1024 * 1024; +/** DashScope temporary-upload per-file cap (§13 #18). */ +const MAX_UPLOAD_FILE_BYTES_CEILING = GIB; +/** DashScope temporary uploads live 48h (§13 #19). */ +const MAX_URL_TTL_HOURS = 48; + +/** Raw fixed-policy entry shape accepted from settings. Everything + * optional except `mediaTypes` and `toolName`; unknown keys rejected. */ +const POLICY_ENTRY_KEYS = new Set([ + 'priority', + 'mediaTypes', + 'origins', + 'when', + 'onConditionUnavailable', + 'toolName', + 'arguments', + 'maxRunsPerLineage', + 'onFailure', + 'output', +]); +const OUTPUT_KEYS = new Set(['reprocessMedia', 'source', 'artifacts']); +/** Valid `kind:<…>` selector targets in `output.artifacts` — the media + * modalities plus the non-media `file` artifact kind (transcripts). */ +const ARTIFACT_SELECTOR_KINDS = new Set(['image', 'video', 'audio', 'file']); +const MODALITIES: readonly OmniModality[] = ['image', 'video', 'audio']; +const ORIGINS: readonly FixedPolicyOrigin[] = ['user', 'tool', 'policy']; +/** io params are harness-injected per invocation — fixed `arguments` + * naming them would be overwritten silently, so they are rejected. */ +const RESERVED_ARGUMENT_KEYS = ['inputPath', 'outputDir', 'resourceId']; + +/** Policy ids feed run records and execution origins; keep them to a + * conservative token charset. */ +const POLICY_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]*$/; + +/** + * System default policies, registered ONLY as `transportGuard.policies` + * (decision D7, revised): no `when`, so the guard stage runs them exactly + * when the final delivery set still exceeds transport limits. This + * satisfies the mandatory non-empty + three-modality-coverage validation + * (policy design §10.1). There are NO system-default `fixedPolicies` — + * preprocessing is pure user-experiment semantics, and a zero-config + * setup must never trigger a policy below transport limits. Returns a + * fresh object per call: the entries flow into normalization as raw + * settings input, which must never share mutable state across calls. + */ +export function systemDefaultTransportGuardPolicies(): Record< + string, + Record +> { + return { + 'image-downsample': { + mediaTypes: ['image'], + toolName: ToolNames.OMNI_DOWNSAMPLE_IMAGE, + }, + 'video-downscale': { + mediaTypes: ['video'], + toolName: ToolNames.OMNI_DOWNSCALE_VIDEO, + }, + 'audio-downsample': { + mediaTypes: ['audio'], + toolName: ToolNames.OMNI_DOWNSAMPLE_AUDIO, + }, + }; +} + +/** Raw inputs to normalization, as threaded from settings. */ +export interface RawOmniProcessingSettings { + /** `omni.processing.fixedPolicies` (id → entry | null tombstone). */ + fixedPolicies?: unknown; + /** `omni.processing.transportGuard.policies` (id → entry; tombstones + * are a configuration error — the guard cannot be disabled). */ + transportGuardPolicies?: unknown; + /** `omni.processing.limits`. */ + limits?: unknown; + /** `omni.processing.policyTools`. */ + policyTools?: OmniPolicyToolsSettings; + /** `omni.processing.transportGuard.maxUploadFileBytes`. */ + maxUploadFileBytes?: number; + /** `omni.processing.transportGuard.maxEstimatedTokens` + * (0/unset = token guard disabled). */ + maxEstimatedTokens?: number; + /** `omni.delivery.upload.urlTtlHours`. */ + urlTtlHours?: number; +} + +/** Tool lookup surface normalization validates against (§13 #6/#14: a + * tool that is unregistered — including excluded via tool filtering — or + * not a media-policy tool fails normalization). */ +export interface OmniPolicyToolLookup { + getTool(name: string): + | { + mediaPolicyDescriptor?: MediaPolicyToolDescriptor; + /** The tool's NATIVE parameter schema (DeclarativeTool's public + * field) — never the `schema` getter: for media-policy tools that + * getter is the model-visible projection, which strips + * lockedArguments and operatorOnly keys, and validating operator + * config against the projection would reject every legitimate + * locked/operator-only argument at startup. */ + parameterSchema?: unknown; + } + | undefined; +} + +function fail(message: string): never { + throw new OmniPolicyConfigError(message); +} + +/** §13 #1: unknown keys are configuration errors, never silently ignored. */ +function rejectUnknownKeys( + record: Record, + where: string, + known: readonly string[], +): void { + for (const key of Object.keys(record)) { + if (!known.includes(key)) { + fail(`${where}: unknown key "${key}"`); + } + } +} + +function requirePositiveInteger( + value: unknown, + where: string, + { allowZero = false }: { allowZero?: boolean } = {}, +): number { + if ( + typeof value !== 'number' || + !Number.isInteger(value) || + (allowZero ? value < 0 : value <= 0) + ) { + fail( + `${where}: must be ${allowZero ? 'a non-negative' : 'a positive'} integer (got ${JSON.stringify(value)})`, + ); + } + return value; +} + +/** ID-merge two policy maps: user entry replaces the whole default entry; + * `null` tombstones (where allowed) remove it. */ +function mergePolicyMaps( + defaults: Record>, + raw: unknown, + where: string, + { allowTombstones }: { allowTombstones: boolean }, +): Record> { + // Null prototype: a raw "__proto__" key must become an ordinary own key + // (handed to normalizePolicy, whose id pattern rejects it) instead of + // silently re-prototyping the map through the __proto__ setter. + const merged: Record | null> = Object.assign( + Object.create(null), + defaults, + ); + if (raw === undefined) { + return merged as Record>; + } + if (!isPlainRecord(raw)) { + fail(`${where}: must be an object map of policy id → policy`); + } + for (const [id, entry] of Object.entries(raw)) { + if (entry === null) { + if (!allowTombstones) { + fail( + `${where}.${id}: transport guard policies cannot be removed ` + + `(the guard is mandatory); override the entry instead`, + ); + } + delete merged[id]; + continue; + } + if (!isPlainRecord(entry)) { + fail(`${where}.${id}: must be an object (or null to remove a default)`); + } + merged[id] = entry; + } + return merged as Record>; +} + +function normalizePolicy( + id: string, + entry: Record, + stage: 'preprocessing' | 'transport_guard', + where: string, + tools: OmniPolicyToolLookup, +): NormalizedFixedPolicy { + // §13 #2: unique ids come free with the map shape; the charset check + // keeps ids safe for run records and log lines. + if (!POLICY_ID_PATTERN.test(id)) { + fail( + `${where}.${id}: policy id must match ${POLICY_ID_PATTERN} ` + + `(letters, digits, ".", "_", "-")`, + ); + } + // §13 #1: strict structure — unknown keys are errors, not warnings. + for (const key of Object.keys(entry)) { + if (!POLICY_ENTRY_KEYS.has(key)) { + fail(`${where}.${id}: unknown key "${key}"`); + } + } + + // §13 #3/#4: legal values and enums. + const priority = + entry['priority'] === undefined + ? 0 + : typeof entry['priority'] === 'number' && + Number.isFinite(entry['priority']) + ? entry['priority'] + : fail(`${where}.${id}.priority: must be a finite number`); + + const rawMediaTypes = entry['mediaTypes']; + if (!Array.isArray(rawMediaTypes) || rawMediaTypes.length === 0) { + fail(`${where}.${id}.mediaTypes: must be a non-empty array`); + } + for (const m of rawMediaTypes) { + if (!MODALITIES.includes(m as OmniModality)) { + fail( + `${where}.${id}.mediaTypes: unknown modality ${JSON.stringify(m)} ` + + `(expected ${MODALITIES.join(', ')})`, + ); + } + } + const mediaTypes = [...new Set(rawMediaTypes as OmniModality[])]; + + const rawOrigins = entry['origins'] ?? ['user', 'tool']; + if (!Array.isArray(rawOrigins) || rawOrigins.length === 0) { + fail(`${where}.${id}.origins: must be a non-empty array`); + } + for (const o of rawOrigins) { + if (!ORIGINS.includes(o as FixedPolicyOrigin)) { + fail( + `${where}.${id}.origins: unknown origin ${JSON.stringify(o)} ` + + `(expected ${ORIGINS.join(', ')})`, + ); + } + } + const origins = [...new Set(rawOrigins as FixedPolicyOrigin[])]; + + const onConditionUnavailable = entry['onConditionUnavailable'] ?? 'skip'; + if (onConditionUnavailable === 'abortTurn') { + fail( + `${where}.${id}.onConditionUnavailable: "abortTurn" is not yet ` + + `supported (design reserves it for a later stage); use "skip" or "run"`, + ); + } + if (onConditionUnavailable !== 'skip' && onConditionUnavailable !== 'run') { + fail( + `${where}.${id}.onConditionUnavailable: must be "skip" or "run" ` + + `(got ${JSON.stringify(onConditionUnavailable)})`, + ); + } + + const onFailure = entry['onFailure'] ?? 'continue'; + if (onFailure !== 'continue' && onFailure !== 'abort') { + fail( + `${where}.${id}.onFailure: must be "continue" or "abort" ` + + `(got ${JSON.stringify(onFailure)})`, + ); + } + + const maxRunsPerLineage = + entry['maxRunsPerLineage'] === undefined + ? 1 + : requirePositiveInteger( + entry['maxRunsPerLineage'], + `${where}.${id}.maxRunsPerLineage`, + ); + + const rawOutput = entry['output'] ?? {}; + if (!isPlainRecord(rawOutput)) { + fail(`${where}.${id}.output: must be an object`); + } + // §13 #23: output fields are a closed set with concrete defaults. + for (const key of Object.keys(rawOutput)) { + if (!OUTPUT_KEYS.has(key)) { + fail(`${where}.${id}.output: unknown key "${key}"`); + } + } + const reprocessMedia = rawOutput['reprocessMedia'] ?? false; + if (typeof reprocessMedia !== 'boolean') { + fail(`${where}.${id}.output.reprocessMedia: must be a boolean`); + } + const source = rawOutput['source'] ?? 'omit'; + if (source !== 'keep' && source !== 'omit') { + fail( + `${where}.${id}.output.source: must be "keep" or "omit" ` + + `(got ${JSON.stringify(source)})`, + ); + } + // §13 #17: transport-guard outputs must replace the offending source — + // keeping it would re-deliver the very media the guard rejected. + if (stage === 'transport_guard' && source !== 'omit') { + fail( + `${where}.${id}.output.source: transport guard policies must use ` + + `"omit" (the over-limit source cannot stay in the delivery set)`, + ); + } + // `output.artifacts` selector map (upstream P): selector → action. + // Unconfigured defaults to include-all — the historical "every + // derivative delivers" behavior of this stage. Selector producibility + // (§13 #22/#24) is checked below, once the descriptor is known. + const rawArtifacts = rawOutput['artifacts']; + let artifacts: Record; + if (rawArtifacts === undefined) { + artifacts = { '*': 'include' }; + } else { + if (!isPlainRecord(rawArtifacts)) { + fail(`${where}.${id}.output.artifacts: must be an object`); + } + artifacts = {}; + for (const [selector, action] of Object.entries(rawArtifacts)) { + const at = `${where}.${id}.output.artifacts["${selector}"]`; + if (action !== 'include' && action !== 'retain') { + fail( + `${at}: must be "include" or "retain" (got ${JSON.stringify(action)})`, + ); + } + if (selector === '*') { + // Default action for artifacts no other selector matches. + } else if (selector.startsWith('kind:')) { + const kind = selector.slice('kind:'.length); + if (!ARTIFACT_SELECTOR_KINDS.has(kind)) { + fail( + `${at}: unknown artifact kind "${kind}" ` + + `(expected one of ${[...ARTIFACT_SELECTOR_KINDS].join(', ')})`, + ); + } + } else if (selector.startsWith('role:')) { + const role = selector.slice('role:'.length); + if (!POLICY_ID_PATTERN.test(role)) { + fail(`${at}: invalid role token ${JSON.stringify(role)}`); + } + } else { + fail( + `${at}: unknown selector (expected "*", "kind:", or "role:")`, + ); + } + artifacts[selector] = action; + } + } + + // §13 #5: when-condition structure and field names. + let when: FixedPolicyCondition | undefined; + if (entry['when'] !== undefined) { + if (stage === 'transport_guard') { + // Guard policies are triggered by the limit breach itself, never by + // conditions; a `when` here would silently punch a hole in coverage. + fail( + `${where}.${id}.when: transport guard policies must not declare ` + + `"when" (they run exactly when transport limits are exceeded)`, + ); + } + const errors = validateFixedPolicyCondition( + entry['when'], + `${where}.${id}.when`, + ); + if (errors.length > 0) { + fail(errors.join('; ')); + } + when = entry['when'] as FixedPolicyCondition; + } + + // §13 #6 (+#14): the referenced tool must be registered — an excluded + // (tools.disabled) tool is absent from the registry — and be a + // media-policy tool. + const toolName = entry['toolName']; + if (typeof toolName !== 'string' || toolName.length === 0) { + fail(`${where}.${id}.toolName: must be a non-empty string`); + } + const tool = tools.getTool(toolName); + if (!tool) { + fail( + `${where}.${id}.toolName: tool "${toolName}" is not registered ` + + `(unknown name, or excluded by tool filtering)`, + ); + } + const descriptor = tool.mediaPolicyDescriptor; + if (!descriptor || descriptor.kind !== 'media_policy') { + fail( + `${where}.${id}.toolName: tool "${toolName}" is not a media policy ` + + `tool (no media_policy descriptor)`, + ); + } + // §13 #8: the descriptor must declare a deliverable output at all, and + // a lossy media output obligates a disclosure text output — otherwise + // the pipeline could degrade media with no user-visible disclosure. + if (!descriptor.outputs.some((o) => o.required)) { + fail( + `${where}.${id}.toolName: tool "${toolName}" declares no required ` + + `output; a fixed policy cannot rely on it producing anything`, + ); + } + const hasLossyMedia = descriptor.outputs.some( + (o) => o.kind === 'media' && o.lossy, + ); + const hasDisclosure = descriptor.outputs.some( + (o) => o.kind === 'text' && o.role === 'disclosure', + ); + if (hasLossyMedia && !hasDisclosure) { + fail( + `${where}.${id}.toolName: tool "${toolName}" declares a lossy media ` + + `output but no disclosure text output`, + ); + } + // The policy's modalities must be servable by the tool. + for (const m of mediaTypes) { + if (!descriptor.inputMediaTypes.includes(m)) { + fail( + `${where}.${id}.mediaTypes: tool "${toolName}" does not accept ` + + `"${m}" input (accepts ${descriptor.inputMediaTypes.join(', ')})`, + ); + } + } + + // §13 #22/#24: every configured artifact selector must correspond to an + // output the tool's descriptor can actually produce — a selector that + // can never match is a configuration error, not a silent no-op. #24 in + // full: a `role:transcript` selector must point at a bounded, managed + // UTF-8 text/plain file output. + { + const producibleKinds = new Set(); + // role → declaring output spec (first declaration wins, matching the + // outputs-order semantics of a find): one structure serves BOTH the + // role-selector existence check and the transcript shape check below. + const producibleRoleSpecs = new Map< + string, + (typeof descriptor.outputs)[number] + >(); + for (const o of descriptor.outputs) { + if (o.kind === 'media') { + for (const mimeType of o.mimeTypes ?? []) { + producibleKinds.add(mimeType.split('/')[0]); + } + if (o.role && !producibleRoleSpecs.has(o.role)) { + producibleRoleSpecs.set(o.role, o); + } + } else if (o.kind === 'file') { + producibleKinds.add('file'); + if (o.role && !producibleRoleSpecs.has(o.role)) { + producibleRoleSpecs.set(o.role, o); + } + } + } + for (const selector of Object.keys(artifacts)) { + const at = `${where}.${id}.output.artifacts["${selector}"]`; + if (selector === '*') continue; + if (selector.startsWith('kind:')) { + const kind = selector.slice('kind:'.length); + if (!producibleKinds.has(kind)) { + fail( + `${at}: tool "${toolName}" declares no output of kind "${kind}"`, + ); + } + continue; + } + const role = selector.slice('role:'.length); + const spec = producibleRoleSpecs.get(role); + if (!spec) { + fail( + `${at}: tool "${toolName}" declares no artifact output with ` + + `role "${role}"`, + ); + } + if ( + role === 'transcript' && + (spec.kind !== 'file' || + spec.mimeTypes?.length !== 1 || + spec.mimeTypes[0] !== 'text/plain') + ) { + fail( + `${at}: a transcript selector must point at a bounded UTF-8 ` + + `text/plain file output, but tool "${toolName}" declares ` + + `role "transcript" differently`, + ); + } + } + } + + // §13 #11: fixed arguments validate against the tool's io-stripped + // tunable schema; the harness-injected io keys are reserved. + const args = entry['arguments'] ?? {}; + if (!isPlainRecord(args)) { + fail(`${where}.${id}.arguments: must be an object`); + } + for (const reserved of RESERVED_ARGUMENT_KEYS) { + if (Object.prototype.hasOwnProperty.call(args, reserved)) { + fail( + `${where}.${id}.arguments: "${reserved}" is injected by the ` + + `orchestrator per invocation and must not be configured`, + ); + } + } + if (descriptor.settingsSchema) { + const schemaError = SchemaValidator.validate( + descriptor.settingsSchema, + args, + ); + if (schemaError) { + fail(`${where}.${id}.arguments: ${schemaError}`); + } + } + + return { + id, + priority, + mediaTypes, + origins, + when, + onConditionUnavailable, + toolName, + arguments: args, + maxRunsPerLineage, + onFailure, + output: { reprocessMedia, source, artifacts }, + stage, + }; +} + +function normalizeLimits(raw: unknown): NormalizedOmniProcessingLimits { + if (raw === undefined) { + return { ...DEFAULT_OMNI_PROCESSING_LIMITS }; + } + if (!isPlainRecord(raw)) { + fail('omni.processing.limits: must be an object'); + } + const limits = { ...DEFAULT_OMNI_PROCESSING_LIMITS }; + for (const [key, value] of Object.entries(raw)) { + if (!Object.prototype.hasOwnProperty.call(limits, key)) { + fail(`omni.processing.limits: unknown key "${key}"`); + } + limits[key as keyof NormalizedOmniProcessingLimits] = + requirePositiveInteger(value, `omni.processing.limits.${key}`, { + // Reserving zero output tokens is odd but not incoherent. + allowZero: key === 'reservedOutputTokens', + }); + } + return limits; +} + +/** A numeric bound with its exclusivity, for the §11.2 range checks. */ +interface NumericBound { + value: number; + exclusive: boolean; +} + +/** Effective lower bound of a property schema: the tighter of `minimum` + * and (numeric draft-2020) `exclusiveMinimum` — higher value wins, an + * exclusive bound beats an inclusive one at the same value. */ +function lowerBoundOf(s: Record): NumericBound | undefined { + let bound: NumericBound | undefined; + if (typeof s['minimum'] === 'number') { + bound = { value: s['minimum'], exclusive: false }; + } + if (typeof s['exclusiveMinimum'] === 'number') { + const b = { value: s['exclusiveMinimum'], exclusive: true }; + if (!bound || b.value >= bound.value) bound = b; + } + return bound; +} + +/** Effective upper bound: the tighter of `maximum` and + * `exclusiveMaximum` — lower value wins, exclusive beats inclusive. */ +function upperBoundOf(s: Record): NumericBound | undefined { + let bound: NumericBound | undefined; + if (typeof s['maximum'] === 'number') { + bound = { value: s['maximum'], exclusive: false }; + } + if (typeof s['exclusiveMaximum'] === 'number') { + const b = { value: s['exclusiveMaximum'], exclusive: true }; + if (!bound || b.value <= bound.value) bound = b; + } + return bound; +} + +/** + * §11.2 constraint-VALUE narrowing: given one native property schema and + * the projection override that will be merged over it + * (`{...native, ...override}` in model-access.ts), return a description + * of the first constraint the merge would LOOSEN, or undefined when the + * merged schema is at least as tight as the native one. Covers the + * design's "类型、枚举、范围" beyond the property-set check: `type`, + * `enum` subsets, numeric ranges (minimum/maximum and their exclusive + * forms considered together), and the minLength/maxLength/ + * minItems/maxItems scalar families. + */ +function findProjectionLoosening( + native: Record, + override: Record, +): string | undefined { + const merged: Record = { ...native, ...override }; + + // Type may not change — a different type is a different surface, not a + // narrowing. The one true narrowing is integer over number. + if (typeof native['type'] === 'string' && merged['type'] !== native['type']) { + if (!(native['type'] === 'number' && merged['type'] === 'integer')) { + return ( + `"type" changes the native type ` + + `(${JSON.stringify(merged['type'])} vs native "${native['type']}")` + ); + } + } + + // enum: the projected value set must be a subset of the native one. + if (Array.isArray(native['enum'])) { + if (!Array.isArray(merged['enum'])) { + return '"enum" replaces the native enum with a non-array'; + } + const allowed = new Set(native['enum'].map((v) => JSON.stringify(v))); + const added = merged['enum'].filter((v) => !allowed.has(JSON.stringify(v))); + if (added.length > 0) { + return ( + `"enum" adds values the native enum does not allow ` + + `(${added.map((v) => JSON.stringify(v)).join(', ')})` + ); + } + } + + // Numeric range: the merged effective bounds may not extend past the + // native effective bounds. + const nativeLower = lowerBoundOf(native); + if (nativeLower) { + const mergedLower = lowerBoundOf(merged); + if ( + !mergedLower || + mergedLower.value < nativeLower.value || + (mergedLower.value === nativeLower.value && + nativeLower.exclusive && + !mergedLower.exclusive) + ) { + return ( + `the lower bound loosens the native one ` + + `(${mergedLower?.value ?? 'none'} vs native ${nativeLower.value})` + ); + } + } + const nativeUpper = upperBoundOf(native); + if (nativeUpper) { + const mergedUpper = upperBoundOf(merged); + if ( + !mergedUpper || + mergedUpper.value > nativeUpper.value || + (mergedUpper.value === nativeUpper.value && + nativeUpper.exclusive && + !mergedUpper.exclusive) + ) { + return ( + `the upper bound loosens the native one ` + + `(${mergedUpper?.value ?? 'none'} vs native ${nativeUpper.value})` + ); + } + } + + // Scalar tightness families: min* may not shrink, max* may not grow. + for (const key of ['minLength', 'minItems'] as const) { + const n = native[key]; + if (typeof n !== 'number') continue; + const m = merged[key]; + if (typeof m !== 'number' || m < n) { + return `"${key}" loosens the native constraint (${JSON.stringify(m)} vs native ${n})`; + } + } + for (const key of ['maxLength', 'maxItems'] as const) { + const n = native[key]; + if (typeof n !== 'number') continue; + const m = merged[key]; + if (typeof m !== 'number' || m > n) { + return `"${key}" loosens the native constraint (${JSON.stringify(m)} vs native ${n})`; + } + } + return undefined; +} + +function validatePolicyTools( + policyTools: OmniPolicyToolsSettings | undefined, + tools: OmniPolicyToolLookup, +): void { + if (policyTools === undefined) return; + if (!isPlainRecord(policyTools)) { + fail('omni.processing.policyTools: must be an object map'); + } + for (const [toolName, entry] of Object.entries(policyTools)) { + const where = `omni.processing.policyTools.${toolName}`; + if (entry === null) continue; // scope-merge tombstone + if (!isPlainRecord(entry)) { + fail(`${where}: must be an object`); + } + const tool = tools.getTool(toolName); + if (!tool?.mediaPolicyDescriptor) { + fail(`${where}: "${toolName}" is not a registered media policy tool`); + } + const descriptor = tool.mediaPolicyDescriptor; + + // §13 #1: unknown keys are errors — a typo like "settigns" or + // "modelaccess" would otherwise read as absent downstream and the + // intended configuration would silently never take effect. + rejectUnknownKeys(entry, where, ['settings', 'runtime', 'modelAccess']); + + // §13 #7: tool-level settings validate against the settingsSchema. + if (entry['settings'] !== undefined) { + if (!isPlainRecord(entry['settings'])) { + fail(`${where}.settings: must be an object`); + } + if (descriptor.settingsSchema) { + const error = SchemaValidator.validate( + descriptor.settingsSchema, + entry['settings'], + ); + if (error) { + fail(`${where}.settings: ${error}`); + } + } + } + + if (entry['runtime'] !== undefined) { + if (!isPlainRecord(entry['runtime'])) { + fail(`${where}.runtime: must be an object`); + } + rejectUnknownKeys(entry['runtime'], `${where}.runtime`, ['timeoutMs']); + const timeoutMs = entry['runtime']['timeoutMs']; + if (timeoutMs !== undefined) { + requirePositiveInteger(timeoutMs, `${where}.runtime.timeoutMs`); + // §5 staging lifecycle: the startup sweep treats staging entries + // older than STAGING_GRACE_MS as crash leftovers. A tool allowed + // to run longer than the grace window could have its live staging + // directory deleted out from under it by another process. + if ((timeoutMs as number) >= STAGING_GRACE_MS) { + fail( + `${where}.runtime.timeoutMs: must be below the staging sweep ` + + `grace window (${STAGING_GRACE_MS}ms) so a live invocation's ` + + `staging directory is never reclaimed mid-run`, + ); + } + } + } + + const modelAccess = entry['modelAccess']; + if (modelAccess === undefined) continue; + if (!isPlainRecord(modelAccess)) { + fail(`${where}.modelAccess: must be an object`); + } + rejectUnknownKeys(modelAccess, `${where}.modelAccess`, [ + 'enabled', + 'description', + 'defaultArguments', + 'lockedArguments', + 'parameterSchema', + 'output', + ]); + const defaults = modelAccess['defaultArguments']; + const locked = modelAccess['lockedArguments']; + if (defaults !== undefined && !isPlainRecord(defaults)) { + fail(`${where}.modelAccess.defaultArguments: must be an object`); + } + if (locked !== undefined && !isPlainRecord(locked)) { + fail(`${where}.modelAccess.lockedArguments: must be an object`); + } + // §13 #21: a key cannot be both defaulted (model may override) and + // locked (model must not name it) — the combination is contradictory. + if (isPlainRecord(defaults) && isPlainRecord(locked)) { + const conflicts = Object.keys(defaults).filter((k) => + Object.prototype.hasOwnProperty.call(locked, k), + ); + if (conflicts.length > 0) { + fail( + `${where}.modelAccess: ${conflicts + .map((k) => `"${k}"`) + .join(', ')} present in both defaultArguments and ` + + `lockedArguments`, + ); + } + } + const nativeSchema = tool.parameterSchema; + const nativeProperties = + isPlainRecord(nativeSchema) && isPlainRecord(nativeSchema['properties']) + ? nativeSchema['properties'] + : undefined; + // defaultArguments/lockedArguments are merged into every invocation's + // args (defaults + caller args + locked), so a key the tool's native + // schema does not declare, or a value its sub-schema rejects, would + // fail EVERY invocation at build time. Catch the misconfiguration at + // startup instead of per-call. `required` is intentionally dropped: + // these records are partial arg sets, the io params arrive per-run. + if (nativeProperties !== undefined) { + for (const [label, record] of [ + ['defaultArguments', defaults], + ['lockedArguments', locked], + ] as const) { + if (!isPlainRecord(record)) continue; + const error = SchemaValidator.validate( + { + type: 'object', + properties: nativeProperties, + additionalProperties: false, + }, + record, + ); + if (error) { + fail(`${where}.modelAccess.${label}: ${error}`); + } + } + } + // §13 #20: the model-visible projection may only narrow the native + // schema — a property the native schema does not declare cannot be + // introduced by projection. + const projection = modelAccess['parameterSchema']; + if (projection !== undefined) { + if (!isPlainRecord(projection)) { + fail(`${where}.modelAccess.parameterSchema: must be an object`); + } + const projectionProps = isPlainRecord(projection['properties']) + ? Object.keys(projection['properties']) + : []; + const nativeProps = + nativeProperties !== undefined + ? new Set(Object.keys(nativeProperties)) + : new Set(); + const introduced = projectionProps.filter((p) => !nativeProps.has(p)); + if (introduced.length > 0) { + fail( + `${where}.modelAccess.parameterSchema: ${introduced + .map((p) => `"${p}"`) + .join(', ')} not present in the tool's native schema ` + + `(projection may only narrow)`, + ); + } + // §11.2 in full: narrowing covers constraint VALUES, not just the + // property set. The declaration merges each projected property's + // keys over the native ones (model-access.ts), so an override + // carrying a looser bound would promise the model a range the + // native per-call validation then rejects — the exact per-call + // misconfiguration this startup check exists to prevent. + if ( + isPlainRecord(projection['properties']) && + nativeProperties !== undefined + ) { + for (const [prop, override] of Object.entries( + projection['properties'], + )) { + const native = nativeProperties[prop]; + if (!isPlainRecord(override) || !isPlainRecord(native)) continue; + const loosening = findProjectionLoosening(native, override); + if (loosening !== undefined) { + fail( + `${where}.modelAccess.parameterSchema.properties.${prop}: ` + + `${loosening} (projection may only narrow)`, + ); + } + } + } + } + } +} + +/** + * Normalize and validate the full `omni.processing` configuration. + * Called once at startup (after the tool registry exists); throws + * {@link OmniPolicyConfigError} on any violation. + */ +export function normalizeOmniProcessingConfig( + raw: RawOmniProcessingSettings, + tools: OmniPolicyToolLookup, +): NormalizedOmniProcessingConfig { + // §13 #18: the upload byte ceiling cannot exceed the channel's own cap. + if (raw.maxUploadFileBytes !== undefined) { + const bytes = requirePositiveInteger( + raw.maxUploadFileBytes, + 'omni.processing.transportGuard.maxUploadFileBytes', + ); + if (bytes > MAX_UPLOAD_FILE_BYTES_CEILING) { + fail( + `omni.processing.transportGuard.maxUploadFileBytes: ${bytes} exceeds ` + + `the DashScope per-file upload cap (${MAX_UPLOAD_FILE_BYTES_CEILING})`, + ); + } + } + // Token-guard threshold: settings load performs no runtime type checks, + // and guard.ts compares with `<=`/`>` — a string here would make both + // comparisons false and silently disable the guard (fail-open). Reject + // anything but a finite number ≥ 0 (0/unset = guard disabled). + if (raw.maxEstimatedTokens !== undefined) { + const tokens = raw.maxEstimatedTokens; + if (typeof tokens !== 'number' || !Number.isFinite(tokens) || tokens < 0) { + fail( + `omni.processing.transportGuard.maxEstimatedTokens: must be a ` + + `finite number >= 0, where 0 disables the token guard ` + + `(got ${JSON.stringify(tokens)})`, + ); + } + } + // §13 #19: cached URLs must not outlive the channel's 48h validity. + if (raw.urlTtlHours !== undefined) { + const ttl = raw.urlTtlHours; + if ( + typeof ttl !== 'number' || + !Number.isFinite(ttl) || + ttl < 0 || + ttl > MAX_URL_TTL_HOURS + ) { + fail( + `omni.delivery.upload.urlTtlHours: must be a number between 0 and ` + + `${MAX_URL_TTL_HOURS} (got ${JSON.stringify(ttl)})`, + ); + } + } + + const limits = normalizeLimits(raw.limits); + validatePolicyTools(raw.policyTools, tools); + + // No system defaults on the fixedPolicies side (D7 revised): with no + // configuration, preprocessing has zero policies and never runs. + const fixedMap = mergePolicyMaps( + {}, + raw.fixedPolicies, + 'omni.processing.fixedPolicies', + { allowTombstones: true }, + ); + const guardMap = mergePolicyMaps( + systemDefaultTransportGuardPolicies(), + raw.transportGuardPolicies, + 'omni.processing.transportGuard.policies', + { allowTombstones: false }, + ); + + const fixedPolicies = Object.entries(fixedMap).map(([id, entry]) => + normalizePolicy( + id, + entry, + 'preprocessing', + 'omni.processing.fixedPolicies', + tools, + ), + ); + const transportGuardPolicies = Object.entries(guardMap).map(([id, entry]) => + normalizePolicy( + id, + entry, + 'transport_guard', + 'omni.processing.transportGuard.policies', + tools, + ), + ); + + // §13 #15/#16: the merged guard set must exist and must cover every + // modality the pipeline can deliver — a modality without a guard policy + // would fail closed with no degradation path. + if (transportGuardPolicies.length === 0) { + fail( + 'omni.processing.transportGuard.policies: must not be empty ' + + '(the transport guard is mandatory)', + ); + } + const covered = new Set( + transportGuardPolicies.flatMap((policy) => policy.mediaTypes), + ); + const uncovered = MODALITIES.filter((m) => !covered.has(m)); + if (uncovered.length > 0) { + fail( + `omni.processing.transportGuard.policies: no guard policy covers ` + + `${uncovered.join(', ')} — the merged set must cover image, video, ` + + `and audio`, + ); + } + + // `output.reprocessMedia` re-enters derivatives into matching with + // origin 'policy'. Within a set where no policy accepts that origin the + // flag can never take effect — a silent misconfiguration, so it fails + // like every other contradictory setting. Each stage runs with its own + // policy set, so the check is per set. + for (const [where, set] of [ + ['omni.processing.fixedPolicies', fixedPolicies], + ['omni.processing.transportGuard.policies', transportGuardPolicies], + ] as const) { + const inert = set.filter((p) => p.output.reprocessMedia); + if (inert.length > 0 && !set.some((p) => p.origins.includes('policy'))) { + fail( + `${where}: ${inert.map((p) => `"${p.id}"`).join(', ')} sets ` + + `output.reprocessMedia, but no policy in this set accepts ` + + `origin "policy" — derivatives would re-enter matching and ` + + `never match. Add "policy" to some policy's origins or drop ` + + `reprocessMedia.`, + ); + } + } + + return { fixedPolicies, transportGuardPolicies, limits }; +} diff --git a/packages/core/src/omni/policy/degradation-cache.test.ts b/packages/core/src/omni/policy/degradation-cache.test.ts new file mode 100644 index 00000000000..932b424d751 --- /dev/null +++ b/packages/core/src/omni/policy/degradation-cache.test.ts @@ -0,0 +1,300 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { + computePolicyFingerprint, + OmniDegradationCache, +} from './degradation-cache.js'; + +const ORIGINAL = 'a'.repeat(64); +const DEGRADED = 'b'.repeat(64); + +const ENTRY = { + degradedSha256: DEGRADED, + extension: '.jpg', + disclosure: + '原 4096×3072/8.2MB → 1568×1176/0.9MB，质量 75，细节与文字锐度受损', + mimeType: 'image/jpeg', +}; + +describe('computePolicyFingerprint', () => { + it('is stable across key order and identical inputs', () => { + const a = computePolicyFingerprint('omni_downsample_image', { + maxDimension: 1568, + quality: 75, + }); + const b = computePolicyFingerprint('omni_downsample_image', { + quality: 75, + maxDimension: 1568, + }); + expect(a).toBe(b); + expect(a).toMatch(/^[0-9a-f]{64}$/); + }); + + it('ignores the per-invocation io params (inputPath/outputDir)', () => { + const bare = computePolicyFingerprint('omni_downsample_image', { + quality: 75, + }); + const withIo = computePolicyFingerprint('omni_downsample_image', { + quality: 75, + inputPath: '/tmp/a/in.png', + outputDir: '/tmp/staging/deadbeef', + }); + expect(withIo).toBe(bare); + }); + + it('ignores undefined values (absent tunable == undefined tunable)', () => { + expect( + computePolicyFingerprint('t', { quality: 75, maxDimension: undefined }), + ).toBe(computePolicyFingerprint('t', { quality: 75 })); + }); + + it.each([ + ['tool name', ['other_tool', { quality: 75 }, undefined]], + ['argument value', ['t', { quality: 80 }, undefined]], + ['argument set', ['t', { quality: 75, maxDimension: 800 }, undefined]], + ['tool version', ['t', { quality: 75 }, '2']], + ] as Array<[string, [string, Record, string | undefined]]>)( + 'changes when the %s changes', + (_label, [tool, args, version]) => { + const base = computePolicyFingerprint('t', { quality: 75 }); + expect(computePolicyFingerprint(tool, args, version)).not.toBe(base); + }, + ); + + it('sorts keys recursively in nested arguments', () => { + expect( + computePolicyFingerprint('t', { opts: { b: 2, a: [1, { d: 4, c: 3 }] } }), + ).toBe( + computePolicyFingerprint('t', { opts: { a: [1, { c: 3, d: 4 }], b: 2 } }), + ); + }); +}); + +describe('OmniDegradationCache', () => { + let root: string; + let cache: OmniDegradationCache; + const fp = computePolicyFingerprint('omni_downsample_image', { + maxDimension: 1568, + quality: 75, + }); + + beforeEach(async () => { + root = await fs.mkdtemp(path.join(os.tmpdir(), 'omni-degcache-')); + cache = new OmniDegradationCache(root); + }); + + afterEach(async () => { + await fs.rm(root, { recursive: true, force: true }); + }); + + it('round-trips an entry and persists across instances', async () => { + await expect(cache.get(ORIGINAL, fp)).resolves.toBeNull(); + await cache.put(ORIGINAL, fp, ENTRY); + const hit = await cache.get(ORIGINAL, fp); + expect(hit).toMatchObject(ENTRY); + expect(Date.parse(hit!.createdAt)).not.toBeNaN(); + + const second = new OmniDegradationCache(root); + await expect(second.get(ORIGINAL, fp)).resolves.toMatchObject(ENTRY); + }); + + it('writes to policy-cache.json under the omni root', async () => { + await cache.put(ORIGINAL, fp, ENTRY); + const raw = JSON.parse( + await fs.readFile(path.join(root, 'policy-cache.json'), 'utf8'), + ); + expect(raw.version).toBe(1); + expect(Object.keys(raw.entries)).toEqual([`${ORIGINAL}|${fp}`]); + }); + + it('misses on a different fingerprint or original hash', async () => { + await cache.put(ORIGINAL, fp, ENTRY); + const otherFp = computePolicyFingerprint('omni_downsample_image', { + maxDimension: 800, + }); + await expect(cache.get(ORIGINAL, otherFp)).resolves.toBeNull(); + await expect(cache.get('c'.repeat(64), fp)).resolves.toBeNull(); + }); + + it('re-put for the same key replaces the entry', async () => { + await cache.put(ORIGINAL, fp, ENTRY); + await cache.put(ORIGINAL, fp, { + ...ENTRY, + degradedSha256: 'd'.repeat(64), + }); + await expect(cache.get(ORIGINAL, fp)).resolves.toMatchObject({ + degradedSha256: 'd'.repeat(64), + }); + }); + + it('round-trips the optional artifact role (a hit must not strip it)', async () => { + await cache.put(ORIGINAL, fp, { ...ENTRY, role: 'thumbnail' }); + await expect(cache.get(ORIGINAL, fp)).resolves.toMatchObject({ + ...ENTRY, + role: 'thumbnail', + }); + // And an entry without a role stays role-less. + const fp2 = computePolicyFingerprint('omni_downsample_image', { + quality: 51, + }); + await cache.put(ORIGINAL, fp2, ENTRY); + const hit = await cache.get(ORIGINAL, fp2); + expect(hit!.role).toBeUndefined(); + }); + + it('removeByOriginalSha256 drops every policy result for the source', async () => { + const fp2 = computePolicyFingerprint('omni_downsample_image', { + quality: 50, + }); + await cache.put(ORIGINAL, fp, ENTRY); + await cache.put(ORIGINAL, fp2, ENTRY); + await cache.put('c'.repeat(64), fp, ENTRY); + + await cache.removeByOriginalSha256(ORIGINAL); + await expect(cache.get(ORIGINAL, fp)).resolves.toBeNull(); + await expect(cache.get(ORIGINAL, fp2)).resolves.toBeNull(); + await expect(cache.get('c'.repeat(64), fp)).resolves.not.toBeNull(); + }); + + it('removeByDegradedSha256 drops every entry pointing at the derivative', async () => { + await cache.put(ORIGINAL, fp, ENTRY); + await cache.put('c'.repeat(64), fp, ENTRY); + await cache.put('e'.repeat(64), fp, { + ...ENTRY, + degradedSha256: 'f'.repeat(64), + }); + + await cache.removeByDegradedSha256(DEGRADED); + await expect(cache.get(ORIGINAL, fp)).resolves.toBeNull(); + await expect(cache.get('c'.repeat(64), fp)).resolves.toBeNull(); + await expect(cache.get('e'.repeat(64), fp)).resolves.not.toBeNull(); + }); + + it('backs up a corrupt cache file and starts fresh (never fatal)', async () => { + const filePath = path.join(root, 'policy-cache.json'); + await fs.writeFile(filePath, '{corrupt'); + await expect(cache.get(ORIGINAL, fp)).resolves.toBeNull(); + const names = await fs.readdir(root); + expect(names.some((n) => n.startsWith('policy-cache.json.corrupt-'))).toBe( + true, + ); + // And the cache is usable again. + await cache.put(ORIGINAL, fp, ENTRY); + await expect(cache.get(ORIGINAL, fp)).resolves.toMatchObject(ENTRY); + }); + + it('writes atomically: no .tmp litter, 0600 file mode', async () => { + await cache.put(ORIGINAL, fp, ENTRY); + const names = await fs.readdir(root); + expect(names.filter((n) => n.endsWith('.tmp'))).toEqual([]); + if (process.platform !== 'win32') { + const stat = await fs.stat(path.join(root, 'policy-cache.json')); + expect(stat.mode & 0o777).toBe(0o600); + } + }); + + it('serializes concurrent puts without losing entries', async () => { + await Promise.all( + Array.from({ length: 8 }, (_, i) => + cache.put(ORIGINAL, computePolicyFingerprint('t', { i }), ENTRY), + ), + ); + const raw = JSON.parse( + await fs.readFile(path.join(root, 'policy-cache.json'), 'utf8'), + ); + expect(Object.keys(raw.entries)).toHaveLength(8); + }); + + describe('poisoned cache file (workspace-controlled input is shape-validated)', () => { + /** Plant one raw entry as a hostile repo could ship it. */ + async function plantEntry(entry: Record): Promise { + await fs.writeFile( + path.join(root, 'policy-cache.json'), + JSON.stringify({ + version: 1, + entries: { [`${ORIGINAL}|${fp}`]: entry }, + }), + ); + } + + it.each([ + [ + 'traversal in degradedSha256', + { ...ENTRY, degradedSha256: '../../../../etc/passwd' }, + ], + [ + 'uppercase hex degradedSha256', + { ...ENTRY, degradedSha256: 'A'.repeat(64) }, + ], + ['short degradedSha256', { ...ENTRY, degradedSha256: 'ab12' }], + [ + 'traversal in extension', + { ...ENTRY, extension: '/../../../../tmp/evil' }, + ], + ['multi-dot extension', { ...ENTRY, extension: '.jpg/../x' }], + ['dotless extension', { ...ENTRY, extension: 'jpg' }], + ['non-string extension', { ...ENTRY, extension: 42 }], + ['empty disclosure (D8 invariant)', { ...ENTRY, disclosure: '' }], + ['missing disclosure', { ...ENTRY, disclosure: undefined }], + [ + 'oversized disclosure (prompt-stuffing channel)', + { ...ENTRY, disclosure: 'x'.repeat(4096) }, + ], + ['empty mimeType', { ...ENTRY, mimeType: '' }], + ['missing mimeType', { ...ENTRY, mimeType: undefined }], + ['empty role', { ...ENTRY, role: '' }], + ['non-string role', { ...ENTRY, role: 42 }], + ])( + 'drops a malformed entry instead of serving it: %s', + async (_label, entry) => { + await plantEntry(entry as Record); + await expect(cache.get(ORIGINAL, fp)).resolves.toBeNull(); + // Self-heal: the malformed entry is deleted, so the next transcode's + // put() rebuilds it from verified data. + const raw = JSON.parse( + await fs.readFile(path.join(root, 'policy-cache.json'), 'utf8'), + ); + expect(raw.entries).toEqual({}); + }, + ); + + it('still serves a planted entry when every field is well-formed', async () => { + await plantEntry({ ...ENTRY, createdAt: new Date().toISOString() }); + await expect(cache.get(ORIGINAL, fp)).resolves.toMatchObject(ENTRY); + }); + + it.each([ + ['null', null], + ['string', 'x'], + ['number', 42], + ['array', [1, 2]], + ])( + 'drops a non-object entry VALUE at load instead of throwing: %s', + async (_label, value) => { + // Value-level shape is validated at load (shared cache-file layer): + // a crafted value like `null` must not surface as TypeErrors from + // field accessors — including scans like removeByDegradedSha256 + // that touch EVERY entry, not just the requested key. + await fs.writeFile( + path.join(root, 'policy-cache.json'), + JSON.stringify({ + version: 1, + entries: { [`${ORIGINAL}|${fp}`]: value, other: ENTRY }, + }), + ); + await expect(cache.get(ORIGINAL, fp)).resolves.toBeNull(); + await expect( + cache.removeByDegradedSha256(ENTRY.degradedSha256), + ).resolves.toBeUndefined(); + }, + ); + }); +}); diff --git a/packages/core/src/omni/policy/degradation-cache.ts b/packages/core/src/omni/policy/degradation-cache.ts new file mode 100644 index 00000000000..d114ba47678 --- /dev/null +++ b/packages/core/src/omni/policy/degradation-cache.ts @@ -0,0 +1,199 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { createHash } from 'node:crypto'; +import path from 'node:path'; +import { OmniJsonCacheFile } from '../json-cache-file.js'; +import { OBJECT_EXTENSION_RE } from '../storage.js'; + +/** + * Identity of one degradation result (decision D2): everything the + * orchestrator needs to reuse a previously transcoded derivative without + * re-running the tool — the derived object's content hash (locating it in + * `objects/`), the extension it was stored under (storage.ts convention: + * leading dot), and the disclosure text that must accompany the lossy + * derivative on every delivery. + */ +export interface DegradationCacheEntry { + degradedSha256: string; + /** Object-store extension INCLUDING the leading dot (".jpg"). */ + extension: string; + /** Disclosure the tool emitted (D8) — redelivered verbatim on reuse. */ + disclosure: string; + mimeType: string; + /** `metadata.omniRole` the tool stamped on the artifact, when any — + * without it a cache hit would strip the role a fresh derivation + * carries, changing downstream artifact matching between the first run + * and every cached rerun. */ + role?: string; + createdAt: string; +} + +/** Ceiling on a cached disclosure's length. Disclosures are one-line + * summaries (tens of characters in practice); the cache file is + * workspace-shippable, and an unbounded field served verbatim into + * model-visible content would hand a hostile repo an arbitrarily large + * prompt-stuffing channel on every cache hit. */ +export const MAX_CACHED_DISCLOSURE_LENGTH = 2048; + +/** Io params are per-invocation plumbing, never policy identity: the same + * policy applied to the same object must hit regardless of where the + * source file sat or which staging dir the run used. */ +const FINGERPRINT_EXCLUDED_KEYS = new Set(['inputPath', 'outputDir']); + +/** Deterministic JSON: objects serialized with sorted keys at every + * depth, so `{a,b}` and `{b,a}` fingerprint identically. */ +function stableStringify(value: unknown): string { + if (Array.isArray(value)) { + return `[${value.map(stableStringify).join(',')}]`; + } + if (typeof value === 'object' && value !== null) { + const record = value as Record; + const body = Object.keys(record) + .sort() + .map((k) => `${JSON.stringify(k)}:${stableStringify(record[k])}`) + .join(','); + return `{${body}}`; + } + return JSON.stringify(value) ?? 'null'; +} + +/** + * `policyFingerprint = sha256(toolName + normalized arguments + tool + * version)` (decision D2). Arguments are normalized by dropping the + * per-invocation io params and key-sorting the rest, so semantically + * identical calls fingerprint identically. `toolVersion` exists to + * invalidate cached results when a tool's transcode behavior changes + * without any argument changing. + */ +export function computePolicyFingerprint( + toolName: string, + args: Record, + toolVersion = '1', +): string { + const tunables: Record = {}; + for (const [k, v] of Object.entries(args)) { + if (!FINGERPRINT_EXCLUDED_KEYS.has(k) && v !== undefined) { + tunables[k] = v; + } + } + return createHash('sha256') + .update(`${toolName}\n${stableStringify(tunables)}\n${toolVersion}`) + .digest('hex'); +} + +/** + * Persistent map from `(originalSha256, policyFingerprint)` to the + * degraded derivative's identity (decision D2). Lives at + * `.qwen/omni/policy-cache.json`; a hit whose object still exists in + * `objects/` lets the orchestrator skip a minutes-long transcode. The + * existence check is the orchestrator's job — this cache only answers + * "what did this policy produce last time". + * + * File mechanics (serialized ops, atomic writes, corrupt backup+rebuild, + * unreadable-file no-op) are shared with the upload cache via + * {@link OmniJsonCacheFile}. Entries carry no TTL: identities are + * content-addressed and never go stale — they are invalidated + * explicitly when the underlying object disappears (GC/corruption). + */ +export class OmniDegradationCache { + private readonly file: OmniJsonCacheFile; + + constructor(omniRootDir: string) { + this.file = new OmniJsonCacheFile( + path.join(omniRootDir, 'policy-cache.json'), + 'omni:policy-cache', + ); + } + + private key(originalSha256: string, policyFingerprint: string): string { + return `${originalSha256}|${policyFingerprint}`; + } + + async get( + originalSha256: string, + policyFingerprint: string, + ): Promise { + return this.file.access(null, (entries) => { + const key = this.key(originalSha256, policyFingerprint); + const entry = entries[key]; + if (!entry) return { result: null }; + // The cache file sits inside the workspace (`.qwen/omni/`), so a + // hostile repository can ship a crafted one. Entries are only + // trusted when every field that later becomes a filesystem path or + // a model-visible text is well-formed: the hash must be exactly + // 64-hex (it addresses the object store), the extension a single + // dotted component (no traversal segments), and the disclosure + // non-empty (lossy reuse without disclosure would silently break + // the D8 invariant) and bounded (an unbounded field served + // verbatim into model-visible content is a prompt-stuffing + // channel). Malformed entries are dropped, not served — + // the orchestrator then re-derives and overwrites them. + if ( + !/^[0-9a-f]{64}$/.test(entry.degradedSha256) || + typeof entry.extension !== 'string' || + !OBJECT_EXTENSION_RE.test(entry.extension) || + typeof entry.disclosure !== 'string' || + entry.disclosure.length === 0 || + entry.disclosure.length > MAX_CACHED_DISCLOSURE_LENGTH || + typeof entry.mimeType !== 'string' || + entry.mimeType.length === 0 || + (entry.role !== undefined && + (typeof entry.role !== 'string' || entry.role.length === 0)) + ) { + delete entries[key]; + return { result: null, changed: true }; + } + return { result: entry }; + }); + } + + async put( + originalSha256: string, + policyFingerprint: string, + entry: Omit, + ): Promise { + return this.file.access(undefined, (entries) => { + entries[this.key(originalSha256, policyFingerprint)] = { + ...entry, + createdAt: new Date().toISOString(), + }; + return { result: undefined, changed: true }; + }); + } + + /** Drop every policy result derived FROM the object (source object + * gone/corrupt — GC cascade). */ + async removeByOriginalSha256(originalSha256: string): Promise { + return this.file.access(undefined, (entries) => { + const prefix = `${originalSha256}|`; + let changed = false; + for (const k of Object.keys(entries)) { + if (k.startsWith(prefix)) { + delete entries[k]; + changed = true; + } + } + return { result: undefined, changed }; + }); + } + + /** Drop every entry POINTING AT the derived object (derivative + * gone/corrupt — the next run must re-transcode, not chase a missing + * object). */ + async removeByDegradedSha256(degradedSha256: string): Promise { + return this.file.access(undefined, (entries) => { + let changed = false; + for (const [k, v] of Object.entries(entries)) { + if (v.degradedSha256 === degradedSha256) { + delete entries[k]; + changed = true; + } + } + return { result: undefined, changed }; + }); + } +} diff --git a/packages/core/src/omni/policy/model-access.test.ts b/packages/core/src/omni/policy/model-access.test.ts new file mode 100644 index 00000000000..1ec778878d9 --- /dev/null +++ b/packages/core/src/omni/policy/model-access.test.ts @@ -0,0 +1,611 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { describe, expect, it } from 'vitest'; +import type { MediaPolicyToolDescriptor } from '../../tools/tools.js'; +import type { OmniPolicyToolsSettings } from './types.js'; +import { + evaluateMediaPolicyToolCall, + isMediaPolicyToolHiddenFromModel, + projectMediaPolicyToolDeclaration, + resolveMediaPolicyModelAccess, + type MediaPolicyConfigView, +} from './model-access.js'; + +const DESCRIPTOR: MediaPolicyToolDescriptor = { + kind: 'media_policy', + inputMediaTypes: ['image'], + outputs: [{ kind: 'media', required: true }], +}; + +const configWith = ( + settings: OmniPolicyToolsSettings | undefined, +): MediaPolicyConfigView => ({ + getOmniPolicyToolsSettings: () => settings, +}); + +const policyTool = (name = 'omni_compress_image') => ({ + name, + mediaPolicyDescriptor: DESCRIPTOR, +}); + +const ordinaryTool = (name = 'run_shell_command') => ({ name }); + +describe('resolveMediaPolicyModelAccess', () => { + it('defaults to disabled with empty projections when settings are absent', () => { + expect(resolveMediaPolicyModelAccess({}, 'omni_compress_image')).toEqual({ + enabled: false, + defaultArguments: {}, + lockedArguments: {}, + }); + }); + + it('defaults to disabled when the tool has no settings entry', () => { + const config = configWith({ + other_tool: { modelAccess: { enabled: true } }, + }); + expect( + resolveMediaPolicyModelAccess(config, 'omni_compress_image').enabled, + ).toBe(false); + }); + + it('reads enabled + argument projections when well-formed', () => { + const config = configWith({ + omni_compress_image: { + modelAccess: { + enabled: true, + defaultArguments: { quality: 80 }, + lockedArguments: { output_dir: '/tmp/objects' }, + }, + }, + }); + expect( + resolveMediaPolicyModelAccess(config, 'omni_compress_image'), + ).toEqual({ + enabled: true, + defaultArguments: { quality: 80 }, + lockedArguments: { output_dir: '/tmp/objects' }, + }); + }); + + it.each([ + ['null tombstone entry', { omni_compress_image: null }], + ['non-object modelAccess', { omni_compress_image: { modelAccess: 'yes' } }], + [ + 'array modelAccess', + { omni_compress_image: { modelAccess: [{ enabled: true }] } }, + ], + [ + 'truthy non-boolean enabled', + { omni_compress_image: { modelAccess: { enabled: 'true' } } }, + ], + ])('fails closed on malformed settings: %s', (_label, raw) => { + const config = configWith(raw as unknown as OmniPolicyToolsSettings); + expect( + resolveMediaPolicyModelAccess(config, 'omni_compress_image').enabled, + ).toBe(false); + }); + + it('ignores malformed argument projections but keeps enabled', () => { + const config = configWith({ + omni_compress_image: { + modelAccess: { + enabled: true, + defaultArguments: 'quality=80', + lockedArguments: ['output_dir'], + }, + }, + } as unknown as OmniPolicyToolsSettings); + expect( + resolveMediaPolicyModelAccess(config, 'omni_compress_image'), + ).toEqual({ enabled: true, defaultArguments: {}, lockedArguments: {} }); + }); + + it('reads description and parameterSchema when well-formed', () => { + const config = configWith({ + omni_compress_image: { + modelAccess: { + enabled: true, + description: 'Compress an image.', + parameterSchema: { properties: { quality: { maximum: 90 } } }, + }, + }, + }); + const access = resolveMediaPolicyModelAccess(config, 'omni_compress_image'); + expect(access.description).toBe('Compress an image.'); + expect(access.parameterSchema).toEqual({ + properties: { quality: { maximum: 90 } }, + }); + }); + + it.each([ + ['empty description', { description: '' }], + ['non-string description', { description: 42 }], + ['array parameterSchema', { parameterSchema: [] }], + ['string parameterSchema', { parameterSchema: '{}' }], + ])('drops a malformed declaration projection: %s', (_label, modelAccess) => { + const config = configWith({ + omni_compress_image: { modelAccess }, + } as unknown as OmniPolicyToolsSettings); + const access = resolveMediaPolicyModelAccess(config, 'omni_compress_image'); + expect(access.description).toBeUndefined(); + expect(access.parameterSchema).toBeUndefined(); + }); +}); + +describe('projectMediaPolicyToolDeclaration', () => { + const NATIVE = { + name: 'omni_compress_image', + description: 'Native description.', + parametersJsonSchema: { + type: 'object', + properties: { + inputPath: { type: 'string', description: 'Source path.' }, + outputDir: { type: 'string' }, + maxDimension: { type: 'number', minimum: 1 }, + quality: { type: 'number', minimum: 1, maximum: 100 }, + }, + required: ['inputPath', 'outputDir'], + additionalProperties: false, + }, + }; + + it('returns the native declaration unchanged without modelAccess settings', () => { + expect(projectMediaPolicyToolDeclaration({}, NATIVE)).toEqual(NATIVE); + }); + + it('removes lockedArguments keys from properties AND required', () => { + const config = configWith({ + omni_compress_image: { + modelAccess: { + enabled: true, + lockedArguments: { outputDir: '/staging' }, + }, + }, + }); + expect(projectMediaPolicyToolDeclaration(config, NATIVE)).toEqual({ + name: 'omni_compress_image', + description: 'Native description.', + parametersJsonSchema: { + type: 'object', + properties: { + inputPath: { type: 'string', description: 'Source path.' }, + maxDimension: { type: 'number', minimum: 1 }, + quality: { type: 'number', minimum: 1, maximum: 100 }, + }, + required: ['inputPath'], + additionalProperties: false, + }, + }); + }); + + it('narrows to parameterSchema properties, merging overrides over native constraints', () => { + const config = configWith({ + omni_compress_image: { + modelAccess: { + enabled: true, + lockedArguments: { inputPath: '/x', outputDir: '/y' }, + parameterSchema: { + properties: { + maxDimension: { maximum: 4096, description: 'Longest edge.' }, + }, + }, + }, + }, + }); + expect(projectMediaPolicyToolDeclaration(config, NATIVE)).toEqual({ + name: 'omni_compress_image', + description: 'Native description.', + parametersJsonSchema: { + type: 'object', + properties: { + maxDimension: { + type: 'number', // native constraint preserved… + minimum: 1, + maximum: 4096, // …override merged on top + description: 'Longest edge.', + }, + }, + required: [], + additionalProperties: false, + }, + }); + }); + + it('is narrowing-only: a projection property with no native counterpart is ignored', () => { + const config = configWith({ + omni_compress_image: { + modelAccess: { + enabled: true, + parameterSchema: { + properties: { + quality: {}, + madeUp: { type: 'string' }, + }, + }, + }, + }, + }); + const declaration = projectMediaPolicyToolDeclaration(config, NATIVE); + const schema = declaration.parametersJsonSchema as { + properties: Record; + }; + expect(Object.keys(schema.properties)).toEqual(['quality']); + }); + + it('never re-adds a locked key even when parameterSchema names it', () => { + const config = configWith({ + omni_compress_image: { + modelAccess: { + enabled: true, + lockedArguments: { outputDir: '/staging' }, + parameterSchema: { + properties: { outputDir: {}, quality: {} }, + }, + }, + }, + }); + const declaration = projectMediaPolicyToolDeclaration(config, NATIVE); + const schema = declaration.parametersJsonSchema as { + properties: Record; + }; + expect(Object.keys(schema.properties)).toEqual(['quality']); + }); + + it('overrides the description when configured', () => { + const config = configWith({ + omni_compress_image: { + modelAccess: { enabled: true, description: 'Model-facing text.' }, + }, + }); + expect(projectMediaPolicyToolDeclaration(config, NATIVE).description).toBe( + 'Model-facing text.', + ); + }); + + it('passes a non-record native schema through, still applying the description override', () => { + const config = configWith({ + omni_compress_image: { + modelAccess: { + enabled: true, + description: 'Overridden.', + lockedArguments: { outputDir: '/staging' }, + }, + }, + }); + const native = { + name: 'omni_compress_image', + description: 'Native description.', + parametersJsonSchema: undefined, + }; + expect(projectMediaPolicyToolDeclaration(config, native)).toEqual({ + name: 'omni_compress_image', + description: 'Overridden.', + parametersJsonSchema: undefined, + }); + }); + + it('hides descriptor operatorOnlyParams like locked keys, even with no modelAccess settings', () => { + // Endpoint/credential parameters must never be model-visible: with an + // enabled-but-unprojected modelAccess config (and even with NO config + // at all) the operator-only keys are removed from properties and + // required exactly like lockedArguments keys. + const native = { + name: 'omni_transcribe_audio', + description: 'Transcribe.', + parametersJsonSchema: { + type: 'object', + properties: { + inputPath: { type: 'string' }, + baseUrl: { type: 'string' }, + apiKeyEnv: { type: 'string' }, + }, + required: ['inputPath', 'baseUrl'], + additionalProperties: false, + }, + operatorOnlyParams: ['baseUrl', 'apiKeyEnv'] as const, + }; + for (const config of [ + {}, + configWith({ omni_transcribe_audio: { modelAccess: { enabled: true } } }), + ]) { + const declaration = projectMediaPolicyToolDeclaration(config, native); + const schema = declaration.parametersJsonSchema as { + properties: Record; + required: string[]; + }; + expect(Object.keys(schema.properties)).toEqual(['inputPath']); + expect(schema.required).toEqual(['inputPath']); + } + }); +}); + +describe('isMediaPolicyToolHiddenFromModel', () => { + it('never hides ordinary tools', () => { + expect(isMediaPolicyToolHiddenFromModel({}, ordinaryTool())).toBe(false); + }); + + it('hides media-policy tools by default', () => { + expect(isMediaPolicyToolHiddenFromModel({}, policyTool())).toBe(true); + }); + + it('reveals media-policy tools when modelAccess.enabled is true', () => { + const config = configWith({ + omni_compress_image: { modelAccess: { enabled: true } }, + }); + expect(isMediaPolicyToolHiddenFromModel(config, policyTool())).toBe(false); + }); +}); + +describe('evaluateMediaPolicyToolCall', () => { + it('passes ordinary tools untouched regardless of settings', () => { + const args = { command: 'ls' }; + const result = evaluateMediaPolicyToolCall({ + config: configWith({ + run_shell_command: { modelAccess: { enabled: false } }, + }), + tool: ordinaryTool(), + args, + executionOrigin: { kind: 'model' }, + }); + expect(result).toEqual({ outcome: 'pass', args }); + }); + + it('treats a missing origin as a model call (fail closed)', () => { + const result = evaluateMediaPolicyToolCall({ + config: {}, + tool: policyTool(), + args: {}, + executionOrigin: undefined, + }); + expect(result).toMatchObject({ + outcome: 'reject', + reason: 'execution_denied', + }); + }); + + it('rejects model calls of media-policy tools by default, citing the setting', () => { + const result = evaluateMediaPolicyToolCall({ + config: {}, + tool: policyTool(), + args: {}, + executionOrigin: { kind: 'model' }, + }); + expect(result).toMatchObject({ + outcome: 'reject', + reason: 'execution_denied', + }); + expect((result as { message: string }).message).toContain( + '"omni.processing.policyTools.omni_compress_image.modelAccess.enabled": true', + ); + }); + + it('rejects client-origin calls the same as model calls when disabled', () => { + const result = evaluateMediaPolicyToolCall({ + config: {}, + tool: policyTool(), + args: {}, + executionOrigin: { kind: 'client' }, + }); + expect(result).toMatchObject({ + outcome: 'reject', + reason: 'execution_denied', + }); + }); + + it('rejects a forged fixed_policy origin on a non-media-policy tool', () => { + const result = evaluateMediaPolicyToolCall({ + config: {}, + tool: ordinaryTool(), + args: { command: 'rm -rf /' }, + executionOrigin: { + kind: 'fixed_policy', + policyId: 'forged', + stage: 'preprocessing', + }, + }); + expect(result).toMatchObject({ + outcome: 'reject', + reason: 'execution_denied', + }); + expect((result as { message: string }).message).toContain( + 'not a media policy tool', + ); + }); + + it('passes fixed_policy calls of media-policy tools untouched, ignoring modelAccess', () => { + const args = { quality: 55, output_dir: '/staging' }; + const result = evaluateMediaPolicyToolCall({ + // Disabled + locked keys present in args: neither applies to + // fixed-policy calls. + config: configWith({ + omni_compress_image: { + modelAccess: { + enabled: false, + lockedArguments: { output_dir: '/elsewhere' }, + }, + }, + }), + tool: policyTool(), + args, + executionOrigin: { + kind: 'fixed_policy', + policyId: 'image-compress-v1', + stage: 'preprocessing', + }, + }); + expect(result).toEqual({ outcome: 'pass', args }); + expect((result as { args: Record }).args).toBe(args); + }); + + it('rejects explicit lockedArguments keys as invalid_params, naming the keys', () => { + const result = evaluateMediaPolicyToolCall({ + config: configWith({ + omni_compress_image: { + modelAccess: { + enabled: true, + lockedArguments: { output_dir: '/tmp', format: 'webp' }, + }, + }, + }), + tool: policyTool(), + args: { output_dir: '/evil', format: 'exe', quality: 50 }, + executionOrigin: { kind: 'model' }, + }); + expect(result).toMatchObject({ + outcome: 'reject', + reason: 'invalid_params', + }); + const message = (result as { message: string }).message; + expect(message).toContain('"output_dir"'); + expect(message).toContain('"format"'); + }); + + it('rejects a locked key even when passed as undefined', () => { + const result = evaluateMediaPolicyToolCall({ + config: configWith({ + omni_compress_image: { + modelAccess: { + enabled: true, + lockedArguments: { output_dir: '/tmp' }, + }, + }, + }), + tool: policyTool(), + args: { output_dir: undefined }, + executionOrigin: { kind: 'model' }, + }); + expect(result).toMatchObject({ + outcome: 'reject', + reason: 'invalid_params', + }); + }); + + it('merges defaults < model args < lockedArguments on pass', () => { + const result = evaluateMediaPolicyToolCall({ + config: configWith({ + omni_compress_image: { + modelAccess: { + enabled: true, + defaultArguments: { quality: 80, format: 'jpeg' }, + lockedArguments: { output_dir: '/objects' }, + }, + }, + }), + tool: policyTool(), + args: { quality: 55, source: 'a.png' }, + executionOrigin: { kind: 'model' }, + }); + expect(result).toEqual({ + outcome: 'pass', + args: { + quality: 55, // model overrides default + format: 'jpeg', // default fills omitted + source: 'a.png', // model-only key preserved + output_dir: '/objects', // locked always injected + }, + }); + }); + + it('passes enabled tools with no projections through unchanged', () => { + const result = evaluateMediaPolicyToolCall({ + config: configWith({ + omni_compress_image: { modelAccess: { enabled: true } }, + }), + tool: policyTool(), + args: { source: 'a.png' }, + executionOrigin: { kind: 'model' }, + }); + expect(result).toEqual({ outcome: 'pass', args: { source: 'a.png' } }); + }); + + const exfilTool = (name = 'omni_transcribe_audio') => ({ + name, + mediaPolicyDescriptor: { + ...DESCRIPTOR, + operatorOnlyParams: ['baseUrl', 'apiKeyEnv'], + } satisfies MediaPolicyToolDescriptor, + }); + + it('rejects gated calls naming a descriptor operator-only key (credential exfiltration)', () => { + // The attack this gate exists for: injected content telling the model + // to point another provider's key at an attacker host. + for (const originKind of ['model', 'client'] as const) { + const result = evaluateMediaPolicyToolCall({ + config: configWith({ + omni_transcribe_audio: { modelAccess: { enabled: true } }, + }), + tool: exfilTool(), + args: { + inputPath: '/a.wav', + baseUrl: 'https://evil.example/v1', + apiKeyEnv: 'OPENAI_API_KEY', + }, + executionOrigin: { kind: originKind }, + }); + expect(result).toMatchObject({ + outcome: 'reject', + reason: 'invalid_params', + }); + const message = (result as { message: string }).message; + expect(message).toContain('"baseUrl"'); + expect(message).toContain('"apiKeyEnv"'); + expect(message).toContain('operator-only'); + } + }); + + it('rejects an operator-only key even when passed as undefined', () => { + const result = evaluateMediaPolicyToolCall({ + config: configWith({ + omni_transcribe_audio: { modelAccess: { enabled: true } }, + }), + tool: exfilTool(), + args: { inputPath: '/a.wav', apiKeyEnv: undefined }, + executionOrigin: { kind: 'model' }, + }); + expect(result).toMatchObject({ + outcome: 'reject', + reason: 'invalid_params', + }); + }); + + it('still allows operator-only values via defaultArguments and fixed_policy args', () => { + // Operator surfaces stay functional: modelAccess.defaultArguments may + // inject the endpoint config the caller is forbidden to name… + const gated = evaluateMediaPolicyToolCall({ + config: configWith({ + omni_transcribe_audio: { + modelAccess: { + enabled: true, + defaultArguments: { baseUrl: 'https://asr.corp/v1' }, + }, + }, + }), + tool: exfilTool(), + args: { inputPath: '/a.wav' }, + executionOrigin: { kind: 'model' }, + }); + expect(gated).toEqual({ + outcome: 'pass', + args: { inputPath: '/a.wav', baseUrl: 'https://asr.corp/v1' }, + }); + + // …and fixed-policy calls (operator-authored settings.json arguments) + // bypass the gate entirely, operator-only keys included. + const args = { inputPath: '/a.wav', apiKeyEnv: 'CORP_ASR_KEY' }; + const fixed = evaluateMediaPolicyToolCall({ + config: configWith(undefined), + tool: exfilTool(), + args, + executionOrigin: { + kind: 'fixed_policy', + policyId: 'audio-transcribe-v1', + stage: 'preprocessing', + }, + }); + expect(fixed).toEqual({ outcome: 'pass', args }); + }); +}); diff --git a/packages/core/src/omni/policy/model-access.ts b/packages/core/src/omni/policy/model-access.ts new file mode 100644 index 00000000000..8bb6d7c5ef7 --- /dev/null +++ b/packages/core/src/omni/policy/model-access.ts @@ -0,0 +1,288 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import type { FunctionDeclaration } from '@google/genai'; +import type { ToolExecutionOrigin } from '../../core/turn.js'; +import type { MediaPolicyToolDescriptor } from '../../tools/tools.js'; +import { isPlainRecord } from './types.js'; +import type { + MediaPolicyToolConfigView, + OmniPolicyToolModelAccessSettings, +} from './types.js'; + +/** + * Shared modelAccess resolver + call gate for omni media-policy tools. + * + * Media-policy tools are always registered (the fixed-policy orchestrator + * must be able to find them), but they are fixed-policy-only by default: + * only `omni.processing.policyTools..modelAccess.enabled: true` + * makes them callable by the model or by direct client calls. The gate + * must hold on every surface at once — declaration lists, ToolSearch + * keyword + select, the CoreToolScheduler, and ACP's Session.runTool() — + * so all of them call into this module rather than re-deriving the rule. + */ + +/** Historical name this module exported for its config view; kept as an + * alias of the shared type so existing importers stay valid. */ +export type MediaPolicyConfigView = MediaPolicyToolConfigView; + +/** Resolved modelAccess for one tool: always concrete (defaults applied). */ +export interface ResolvedMediaPolicyModelAccess { + /** Whether model/client-origin calls are allowed. Default false. */ + enabled: boolean; + defaultArguments: Record; + lockedArguments: Record; + /** Model-facing description override for the declaration projection. */ + description?: string; + /** Narrowing-only projection over the native parameter schema. */ + parameterSchema?: Record; +} + +/** + * Read `omni.processing.policyTools..modelAccess` leniently: + * anything absent or malformed resolves to the fail-closed default + * (`enabled: false`, no argument projection). + */ +export function resolveMediaPolicyModelAccess( + config: MediaPolicyConfigView, + toolName: string, +): ResolvedMediaPolicyModelAccess { + const entry = config.getOmniPolicyToolsSettings?.()?.[toolName]; + const modelAccess: OmniPolicyToolModelAccessSettings | undefined = + isPlainRecord(entry) && isPlainRecord(entry['modelAccess']) + ? (entry['modelAccess'] as OmniPolicyToolModelAccessSettings) + : undefined; + return { + enabled: modelAccess?.enabled === true, + defaultArguments: isPlainRecord(modelAccess?.defaultArguments) + ? modelAccess.defaultArguments + : {}, + lockedArguments: isPlainRecord(modelAccess?.lockedArguments) + ? modelAccess.lockedArguments + : {}, + description: + typeof modelAccess?.description === 'string' && + modelAccess.description !== '' + ? modelAccess.description + : undefined, + parameterSchema: isPlainRecord(modelAccess?.parameterSchema) + ? modelAccess.parameterSchema + : undefined, + }; +} + +/** + * Model-visible declaration for a media-policy tool (decision D6, policy + * design §9.4): the projection is applied at the SINGLE `schema` getter + * the declaration surfaces read, while validation keeps using the native + * schema (the harness-injected arguments the projection hides must stay + * valid). + * + * Shape: the native parametersJsonSchema minus every + * `modelAccess.lockedArguments` key (removed from `properties` and + * `required` — the model must not see arguments it is forbidden to pass), + * then — when `modelAccess.parameterSchema` is configured — narrowed to + * the properties it names, with each named property's constraints merged + * over the native ones. A projection property with no native counterpart + * is ignored (narrowing-only: the projection can never ADD surface). + * `modelAccess.description` overrides the tool description when set. + */ +export function projectMediaPolicyToolDeclaration( + config: MediaPolicyConfigView, + native: { + name: string; + description: string; + parametersJsonSchema: unknown; + /** Descriptor-declared operator-only keys — hidden from the model + * exactly like lockedArguments (the model must not see arguments the + * gate forbids it to pass). */ + operatorOnlyParams?: readonly string[]; + }, +): FunctionDeclaration { + const access = resolveMediaPolicyModelAccess(config, native.name); + const description = access.description ?? native.description; + const schema = isPlainRecord(native.parametersJsonSchema) + ? native.parametersJsonSchema + : undefined; + const nativeProps = + schema && isPlainRecord(schema['properties']) + ? schema['properties'] + : undefined; + if (!schema || !nativeProps) { + return { + name: native.name, + description, + parametersJsonSchema: native.parametersJsonSchema, + }; + } + const lockedKeys = new Set([ + ...Object.keys(access.lockedArguments), + ...(native.operatorOnlyParams ?? []), + ]); + const narrowProps = + access.parameterSchema && + isPlainRecord(access.parameterSchema['properties']) + ? (access.parameterSchema['properties'] as Record) + : undefined; + const properties: Record = {}; + for (const [key, value] of Object.entries(nativeProps)) { + if (lockedKeys.has(key)) continue; + if (narrowProps && !(key in narrowProps)) continue; + const override = narrowProps?.[key]; + properties[key] = + isPlainRecord(override) && isPlainRecord(value) + ? { ...value, ...override } + : value; + } + const required = Array.isArray(schema['required']) + ? (schema['required'] as unknown[]).filter( + (key): key is string => typeof key === 'string' && key in properties, + ) + : undefined; + return { + name: native.name, + description, + parametersJsonSchema: { + ...schema, + properties, + ...(required !== undefined ? { required } : {}), + }, + }; +} + +/** + * Whether a tool must be hidden from model-facing declaration surfaces + * (initial declarations, subagent filtered declarations, ToolSearch + * keyword candidates and exact-select). True iff the tool is a + * media-policy tool whose modelAccess is not enabled. + */ +export function isMediaPolicyToolHiddenFromModel( + config: MediaPolicyConfigView, + tool: { name: string; mediaPolicyDescriptor?: MediaPolicyToolDescriptor }, +): boolean { + if (!tool.mediaPolicyDescriptor) return false; + return !resolveMediaPolicyModelAccess(config, tool.name).enabled; +} + +/** Outcome of {@link evaluateMediaPolicyToolCall}. */ +export type MediaPolicyCallGateResult = + | { + outcome: 'pass'; + /** Arguments to build the invocation with. For gated model/client + * calls this is defaults + caller args + lockedArguments; for + * everything else it is the caller args unchanged. */ + args: Record; + } + | { + outcome: 'reject'; + /** 'execution_denied' → the call may not run at all; + * 'invalid_params' → a parameter-level error the model can fix. */ + reason: 'execution_denied' | 'invalid_params'; + message: string; + }; + +/** + * Execution-time gate applied by CoreToolScheduler and ACP Session.runTool + * before an invocation is built: + * + * - a `fixed_policy` origin on a NON-media-policy tool is rejected + * (defense in depth — origins are never deserialized, but a forged + * origin must not become a permission bypass for Shell/Edit/MCP); + * - a `fixed_policy` origin on a media-policy tool passes untouched (the + * orchestrator already resolved its own `arguments`; modelAccess does + * not apply to fixed calls); + * - a model/client-origin call of a media-policy tool requires + * `modelAccess.enabled`, must not name any lockedArguments key + * explicitly, and gets defaults + lockedArguments merged in; + * - everything else passes untouched. + * + * A missing origin fails closed as `{ kind: 'model' }`. + */ +export function evaluateMediaPolicyToolCall(params: { + config: MediaPolicyConfigView; + tool: { name: string; mediaPolicyDescriptor?: MediaPolicyToolDescriptor }; + args: Record; + executionOrigin: ToolExecutionOrigin | undefined; +}): MediaPolicyCallGateResult { + const { config, tool, args } = params; + const origin = params.executionOrigin ?? { kind: 'model' }; + + if (origin.kind === 'fixed_policy') { + if (!tool.mediaPolicyDescriptor) { + return { + outcome: 'reject', + reason: 'execution_denied', + message: + `Tool "${tool.name}" cannot run with a fixed-policy execution ` + + `origin: it is not a media policy tool.`, + }; + } + return { outcome: 'pass', args }; + } + + if (!tool.mediaPolicyDescriptor) { + return { outcome: 'pass', args }; + } + + const access = resolveMediaPolicyModelAccess(config, tool.name); + if (!access.enabled) { + return { + outcome: 'reject', + reason: 'execution_denied', + message: + `Tool "${tool.name}" is an omni media policy tool reserved for ` + + `fixed-policy orchestration. Direct calls require ` + + `"omni.processing.policyTools.${tool.name}.modelAccess.enabled": true.`, + }; + } + + const lockedKeys = Object.keys(access.lockedArguments); + const violations = lockedKeys.filter((key) => + Object.prototype.hasOwnProperty.call(args, key), + ); + if (violations.length > 0) { + return { + outcome: 'reject', + reason: 'invalid_params', + message: + `Invalid parameters for tool "${tool.name}": ` + + `${violations.map((k) => `"${k}"`).join(', ')} ` + + `${violations.length === 1 ? 'is' : 'are'} locked by configuration ` + + `and must not be provided. Remove ${ + violations.length === 1 ? 'it' : 'them' + } and retry.`, + }; + } + + // Operator-only parameters (descriptor-declared, e.g. endpoint base URL + // + credential env-var name): a gated caller must never set them — a + // model-controlled endpoint/credential pair would let injected content + // exfiltrate arbitrary environment secrets to an attacker host. They + // remain settable through settings / defaultArguments / lockedArguments + // (operator-controlled surfaces only). + const operatorViolations = ( + tool.mediaPolicyDescriptor.operatorOnlyParams ?? [] + ).filter((key) => Object.prototype.hasOwnProperty.call(args, key)); + if (operatorViolations.length > 0) { + return { + outcome: 'reject', + reason: 'invalid_params', + message: + `Invalid parameters for tool "${tool.name}": ` + + `${operatorViolations.map((k) => `"${k}"`).join(', ')} ` + + `${operatorViolations.length === 1 ? 'is' : 'are'} operator-only ` + + `(set via omni.processing.policyTools.${tool.name} configuration) ` + + `and must not be provided by the caller. Remove ${ + operatorViolations.length === 1 ? 'it' : 'them' + } and retry.`, + }; + } + + return { + outcome: 'pass', + args: { ...access.defaultArguments, ...args, ...access.lockedArguments }, + }; +} diff --git a/packages/core/src/omni/policy/orchestrator.test.ts b/packages/core/src/omni/policy/orchestrator.test.ts new file mode 100644 index 00000000000..0c906e0a6b7 --- /dev/null +++ b/packages/core/src/omni/policy/orchestrator.test.ts @@ -0,0 +1,1805 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { createHash } from 'node:crypto'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import type { Config } from '../../config/config.js'; +import type { ToolCallRequestInfo } from '../../core/turn.js'; +import type { MediaPolicyToolDescriptor } from '../../tools/tools.js'; +import type { RecognizedMedia } from '../recognition.js'; +import { OmniObjectStore } from '../storage.js'; +import { + computePolicyFingerprint, + OmniDegradationCache, +} from './degradation-cache.js'; +import { + MAX_FILE_ARTIFACT_BYTES, + OmniPolicyExecutionError, + runFixedPolicies, + type PolicySourceResource, +} from './orchestrator.js'; +import type { + FixedPolicyCondition, + NormalizedFixedPolicy, + NormalizedOmniProcessingLimits, +} from './types.js'; + +// The orchestrator resolves the executor with a dynamic import; vitest +// intercepts it the same as a static one. +const executeToolCallMock = vi.hoisted(() => vi.fn()); +vi.mock('../../core/nonInteractiveToolExecutor.js', () => ({ + executeToolCall: executeToolCallMock, +})); + +// Partial mock: recognizeMediaFile would need ffprobe; everything else +// (hashFileSha256 in particular — putFile re-hashes for real) stays real. +const recognizeMediaFileMock = vi.hoisted(() => vi.fn()); +vi.mock('../recognition.js', async (importOriginal) => ({ + ...(await importOriginal()), + recognizeMediaFile: recognizeMediaFileMock, +})); + +const SOURCE_BYTES = 'original-image-bytes'; +const DEGRADED_BYTES = 'degraded-image-bytes'; + +function sha256Of(text: string): string { + return createHash('sha256').update(text).digest('hex'); +} + +function recognizedImage( + overrides: Partial = {}, +): RecognizedMedia { + return { + modality: 'image', + detectedMimeType: 'image/png', + sizeBytes: SOURCE_BYTES.length, + metadata: { width: 4000, height: 3000 }, + ...overrides, + }; +} + +const DEGRADED_RECOGNIZED: RecognizedMedia = { + modality: 'image', + detectedMimeType: 'image/jpeg', + sizeBytes: DEGRADED_BYTES.length, + metadata: { width: 1568, height: 1176 }, +}; + +const DESCRIPTOR: MediaPolicyToolDescriptor = { + kind: 'media_policy', + inputMediaTypes: ['image'], + outputs: [ + { kind: 'media', mimeTypes: ['image/jpeg'], required: true, lossy: true }, + { kind: 'text', role: 'disclosure', required: true }, + ], +}; + +function makePolicy( + overrides: Partial = {}, +): NormalizedFixedPolicy { + return { + id: 'img-downsample', + priority: 0, + mediaTypes: ['image'], + origins: ['user', 'tool'], + onConditionUnavailable: 'skip', + toolName: 'omni_downsample_image', + arguments: { maxDimension: 1568 }, + maxRunsPerLineage: 1, + onFailure: 'continue', + output: { + reprocessMedia: false, + source: 'omit', + artifacts: { '*': 'include' }, + }, + stage: 'preprocessing', + ...overrides, + }; +} + +function makeConfig( + descriptorByTool: Record, + policyToolsSettings?: unknown, +) { + return { + getToolRegistry: () => ({ + getTool: (name: string) => + descriptorByTool[name] + ? { mediaPolicyDescriptor: descriptorByTool[name] } + : undefined, + }), + getOmniPolicyToolsSettings: () => policyToolsSettings, + } as unknown as Config; +} + +/** System defaults (P §12.2) with per-test overrides. */ +function limitsWith( + overrides: Partial, +): NormalizedOmniProcessingLimits { + return { + maxConcurrentResources: 1, + reservedOutputTokens: 8192, + maxLineageDepth: 8, + maxPolicyRunsPerRoot: 64, + maxArtifactsPerRoot: 256, + maxDerivedBytesPerRoot: 1073741824, + maxTransportPasses: 3, + ...overrides, + }; +} + +describe('runFixedPolicies', () => { + let tmpDir: string; + let store: OmniObjectStore; + let sourcePath: string; + let source: PolicySourceResource; + let config: Config; + + /** Default success behavior: write a degraded artifact into the staging + * dir and return it as a workspace policy artifact with a disclosure. */ + function mockToolSuccess( + options: { + bytes?: string; + disclosure?: string | undefined; + fileName?: string; + } = {}, + ): void { + const bytes = options.bytes ?? DEGRADED_BYTES; + const fileName = options.fileName ?? 'out.jpg'; + const disclosure = + 'disclosure' in options + ? options.disclosure + : 'Downsampled from 4000x3000 to 1568x1176.'; + executeToolCallMock.mockImplementation( + async (_config: Config, request: ToolCallRequestInfo) => { + const outputDir = request.args['outputDir'] as string; + await fs.writeFile(path.join(outputDir, fileName), bytes); + return { + callId: request.callId, + responseParts: [], + resultDisplay: undefined, + error: undefined, + errorType: undefined, + policyArtifacts: { + toolName: request.name, + invocationId: request.callId, + executionOrigin: request.executionOrigin, + artifacts: [ + { + kind: 'image', + storage: 'workspace', + title: fileName, + workspacePath: fileName, + mimeType: 'image/jpeg', + ...(disclosure !== undefined + ? { metadata: { omniDisclosure: disclosure } } + : {}), + }, + ], + }, + }; + }, + ); + } + + beforeEach(async () => { + vi.clearAllMocks(); + tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'omni-orchestrator-')); + store = new OmniObjectStore(path.join(tmpDir, '.qwen')); + sourcePath = path.join(tmpDir, 'photo.png'); + await fs.writeFile(sourcePath, SOURCE_BYTES); + source = { + filePath: sourcePath, + recognized: recognizedImage(), + displayName: 'photo.png', + origin: 'user', + }; + config = makeConfig({ omni_downsample_image: DESCRIPTOR }); + recognizeMediaFileMock.mockImplementation(async (filePath: string) => { + if (filePath.endsWith('.jpg')) return DEGRADED_RECOGNIZED; + return recognizedImage(); + }); + }); + + afterEach(async () => { + await fs.rm(tmpDir, { recursive: true, force: true }); + }); + + it('delivers the source untouched when no policy matches its modality', async () => { + const { deliveries, records } = await runFixedPolicies(config, source, { + store, + policies: [makePolicy({ mediaTypes: ['video'] })], + }); + expect(deliveries).toEqual([ + { + filePath: sourcePath, + recognized: recognizedImage(), + sha256: undefined, + disclosure: undefined, + degraded: undefined, + }, + ]); + expect(records).toEqual([]); + expect(executeToolCallMock).not.toHaveBeenCalled(); + }); + + it('skips policies whose origins exclude the resource provenance', async () => { + const { deliveries, records } = await runFixedPolicies(config, source, { + store, + policies: [makePolicy({ origins: ['tool'] })], + }); + expect(deliveries).toHaveLength(1); + expect(deliveries[0].filePath).toBe(sourcePath); + expect(records).toEqual([]); + expect(executeToolCallMock).not.toHaveBeenCalled(); + }); + + it('executes a matching policy via the executor protocol and promotes the artifact', async () => { + mockToolSuccess(); + const { deliveries, records } = await runFixedPolicies(config, source, { + store, + policies: [makePolicy()], + }); + + // Exact executor protocol (the "complete minimal protocol" contract). + expect(executeToolCallMock).toHaveBeenCalledTimes(1); + const [calledConfig, request, signal, opts] = + executeToolCallMock.mock.calls[0]; + expect(calledConfig).toBe(config); + expect(signal).toBeInstanceOf(AbortSignal); + expect(opts).toEqual({ recordToolResult: false }); + const req = request as ToolCallRequestInfo; + expect(req.name).toBe('omni_downsample_image'); + expect(req.isClientInitiated).toBe(true); + expect(req.callId).toMatch(/^[0-9a-f]{16}$/); + expect(req.prompt_id).toBe(`omni-fixed-policy-${req.callId}`); + expect(req.executionOrigin).toEqual({ + kind: 'fixed_policy', + policyId: 'img-downsample', + stage: 'preprocessing', + }); + const stagingDir = path.join(store.getStagingDir(), req.callId); + expect(req.args).toEqual({ + maxDimension: 1568, + inputPath: sourcePath, + outputDir: stagingDir, + }); + + // Derivative promoted into objects/, source omitted. + const degradedSha = sha256Of(DEGRADED_BYTES); + const objectPath = store.objectPathFor(degradedSha, '.jpg'); + expect(deliveries).toEqual([ + { + filePath: objectPath, + recognized: DEGRADED_RECOGNIZED, + sha256: degradedSha, + disclosure: 'Downsampled from 4000x3000 to 1568x1176.', + degraded: true, + }, + ]); + await expect(fs.readFile(objectPath, 'utf8')).resolves.toBe(DEGRADED_BYTES); + + // Staging cleaned up; degradation cache written. + await expect(fs.readdir(store.getStagingDir())).resolves.toEqual([]); + const cache = new OmniDegradationCache(store.getOmniRootDir()); + const entry = await cache.get( + sha256Of(SOURCE_BYTES), + computePolicyFingerprint('omni_downsample_image', { maxDimension: 1568 }), + ); + expect(entry).toMatchObject({ + degradedSha256: degradedSha, + extension: '.jpg', + disclosure: 'Downsampled from 4000x3000 to 1568x1176.', + mimeType: 'image/jpeg', + }); + + expect(records).toEqual([ + { + policyId: 'img-downsample', + toolName: 'omni_downsample_image', + outcome: 'succeeded', + resource: 'photo.png', + }, + ]); + }); + + it('keeps the source alongside the derivative when output.source is keep', async () => { + mockToolSuccess(); + const { deliveries } = await runFixedPolicies(config, source, { + store, + policies: [ + makePolicy({ + output: { + reprocessMedia: false, + source: 'keep', + artifacts: { '*': 'include' }, + }, + }), + ], + }); + expect(deliveries.map((d) => d.filePath)).toEqual([ + sourcePath, + store.objectPathFor(sha256Of(DEGRADED_BYTES), '.jpg'), + ]); + }); + + it('reuses a degradation-cache hit without invoking the tool', async () => { + const degradedSha = sha256Of(DEGRADED_BYTES); + const objectPath = store.objectPathFor(degradedSha, '.jpg'); + await fs.mkdir(path.dirname(objectPath), { recursive: true }); + await fs.writeFile(objectPath, DEGRADED_BYTES); + const cache = new OmniDegradationCache(store.getOmniRootDir()); + await cache.put( + sha256Of(SOURCE_BYTES), + computePolicyFingerprint('omni_downsample_image', { maxDimension: 1568 }), + { + degradedSha256: degradedSha, + extension: '.jpg', + disclosure: 'cached disclosure', + mimeType: 'image/jpeg', + }, + ); + + const { deliveries, records } = await runFixedPolicies(config, source, { + store, + policies: [makePolicy()], + }); + expect(executeToolCallMock).not.toHaveBeenCalled(); + expect(deliveries).toEqual([ + { + filePath: objectPath, + recognized: DEGRADED_RECOGNIZED, + sha256: degradedSha, + disclosure: 'cached disclosure', + degraded: true, + }, + ]); + expect(records[0]).toMatchObject({ outcome: 'cache_hit' }); + }); + + it('drops a stale cache entry (object missing) and re-executes', async () => { + const staleSha = sha256Of('stale-derivative'); + const cache = new OmniDegradationCache(store.getOmniRootDir()); + const fingerprint = computePolicyFingerprint('omni_downsample_image', { + maxDimension: 1568, + }); + await cache.put(sha256Of(SOURCE_BYTES), fingerprint, { + degradedSha256: staleSha, + extension: '.jpg', + disclosure: 'stale disclosure', + mimeType: 'image/jpeg', + }); + mockToolSuccess(); + + const { deliveries, records } = await runFixedPolicies(config, source, { + store, + policies: [makePolicy()], + }); + expect(executeToolCallMock).toHaveBeenCalledTimes(1); + expect(records[0]).toMatchObject({ outcome: 'succeeded' }); + expect(deliveries[0].sha256).toBe(sha256Of(DEGRADED_BYTES)); + // The stale entry was replaced by the fresh derivative's identity. + const entry = await cache.get(sha256Of(SOURCE_BYTES), fingerprint); + expect(entry?.degradedSha256).toBe(sha256Of(DEGRADED_BYTES)); + }); + + it('drops an unverifiable cache entry (probe throws) and re-executes instead of failing', async () => { + const plantedBytes = 'previously-degraded-bytes'; + const plantedSha = sha256Of(plantedBytes); + const plantedPath = store.objectPathFor(plantedSha, '.jpg'); + await fs.mkdir(path.dirname(plantedPath), { recursive: true }); + await fs.writeFile(plantedPath, plantedBytes); + const cache = new OmniDegradationCache(store.getOmniRootDir()); + const fingerprint = computePolicyFingerprint('omni_downsample_image', { + maxDimension: 1568, + }); + await cache.put(sha256Of(SOURCE_BYTES), fingerprint, { + degradedSha256: plantedSha, + extension: '.jpg', + disclosure: 'cached disclosure', + mimeType: 'image/jpeg', + }); + // The cached derivative's bytes hash correctly but its probe fails + // (corrupted container, ffprobe I/O race). Verification must drop the + // entry and fall through to a fresh transcode — not abort the run. + recognizeMediaFileMock.mockImplementation(async (filePath: string) => { + if (filePath === plantedPath) throw new Error('probe failed'); + if (filePath.endsWith('.jpg')) return DEGRADED_RECOGNIZED; + return recognizedImage(); + }); + mockToolSuccess(); + + const { deliveries, records } = await runFixedPolicies(config, source, { + store, + policies: [makePolicy()], + }); + expect(executeToolCallMock).toHaveBeenCalledTimes(1); + expect(records).toEqual([ + expect.objectContaining({ outcome: 'succeeded' }), + ]); + expect(deliveries[0].sha256).toBe(sha256Of(DEGRADED_BYTES)); + // Self-heal: the unverifiable entry was dropped and re-written with + // the fresh derivative's identity. + const entry = await cache.get(sha256Of(SOURCE_BYTES), fingerprint); + expect(entry?.degradedSha256).toBe(sha256Of(DEGRADED_BYTES)); + }); + + it('keys the degradation cache with the descriptor version (D2)', async () => { + config = makeConfig({ + omni_downsample_image: { ...DESCRIPTOR, version: '7' }, + }); + mockToolSuccess(); + await runFixedPolicies(config, source, { + store, + policies: [makePolicy()], + }); + const cache = new OmniDegradationCache(store.getOmniRootDir()); + // The entry lives under the versioned fingerprint only: bumping the + // tool version must invalidate derivatives produced by older code. + await expect( + cache.get( + sha256Of(SOURCE_BYTES), + computePolicyFingerprint( + 'omni_downsample_image', + { maxDimension: 1568 }, + '7', + ), + ), + ).resolves.toMatchObject({ degradedSha256: sha256Of(DEGRADED_BYTES) }); + await expect( + cache.get( + sha256Of(SOURCE_BYTES), + computePolicyFingerprint('omni_downsample_image', { + maxDimension: 1568, + }), + ), + ).resolves.toBeNull(); + }); + + it('excludes animated images (frameCount > 1) from policy matching (D9)', async () => { + source = { + ...source, + recognized: recognizedImage({ + metadata: { width: 4000, height: 3000, frameCount: 12 }, + }), + }; + const { deliveries, records } = await runFixedPolicies(config, source, { + store, + policies: [makePolicy()], + }); + expect(executeToolCallMock).not.toHaveBeenCalled(); + expect(records).toEqual([]); + expect(deliveries).toHaveLength(1); + expect(deliveries[0].filePath).toBe(sourcePath); + expect(deliveries[0].degraded).toBeUndefined(); + }); + + it('still matches a single-frame image with an explicit frameCount of 1', async () => { + source = { + ...source, + recognized: recognizedImage({ + metadata: { width: 4000, height: 3000, frameCount: 1 }, + }), + }; + mockToolSuccess(); + const { records } = await runFixedPolicies(config, source, { + store, + policies: [makePolicy()], + }); + expect(executeToolCallMock).toHaveBeenCalledTimes(1); + expect(records[0]).toMatchObject({ outcome: 'succeeded' }); + }); + + it('treats a hash-identical output as a no-op: source delivered, nothing cached', async () => { + mockToolSuccess({ bytes: SOURCE_BYTES }); + // Identical bytes hash identically even though the mock labels the + // artifact image/jpeg — the fixed-point check runs on content hashes. + const { deliveries, records } = await runFixedPolicies(config, source, { + store, + policies: [makePolicy()], // source: 'omit' must NOT apply on no-op + }); + expect(records[0]).toMatchObject({ outcome: 'no_op' }); + expect(deliveries).toHaveLength(1); + expect(deliveries[0].filePath).toBe(sourcePath); + const cache = new OmniDegradationCache(store.getOmniRootDir()); + await expect( + cache.get( + sha256Of(SOURCE_BYTES), + computePolicyFingerprint('omni_downsample_image', { + maxDimension: 1568, + }), + ), + ).resolves.toBeNull(); + }); + + it('silently skips a policy whose `when` does not match', async () => { + const { deliveries, records } = await runFixedPolicies(config, source, { + store, + policies: [ + makePolicy({ + when: ['>', ['field', 'resource.width'], 5000], + }), + ], + }); + expect(records).toEqual([]); + expect(deliveries[0].filePath).toBe(sourcePath); + expect(executeToolCallMock).not.toHaveBeenCalled(); + }); + + it('records condition_unavailable with the missing fields when skipping', async () => { + const { records } = await runFixedPolicies(config, source, { + store, + policies: [ + makePolicy({ + when: ['>', ['field', 'resource.durationMs'], 1000], + }), + ], + }); + expect(records).toEqual([ + { + policyId: 'img-downsample', + toolName: 'omni_downsample_image', + outcome: 'condition_unavailable', + resource: 'photo.png', + missingFields: ['resource.durationMs'], + }, + ]); + expect(executeToolCallMock).not.toHaveBeenCalled(); + }); + + it('runs anyway on an undecidable condition when onConditionUnavailable is run', async () => { + mockToolSuccess(); + const { records } = await runFixedPolicies(config, source, { + store, + policies: [ + makePolicy({ + onConditionUnavailable: 'run', + when: ['>', ['field', 'resource.durationMs'], 1000], + }), + ], + }); + expect(executeToolCallMock).toHaveBeenCalledTimes(1); + expect(records[0]).toMatchObject({ outcome: 'succeeded' }); + }); + + describe('condition namespaces (request./session., policy design §8.3)', () => { + // The 4000×3000 root estimates to ceil(4000*3000/2048) = 5860 tokens; + // the 1568×1176 derivative to ceil(1568*1176/2048) = 901. + const requestWhen = ( + operator: '>' | '==', + value: number, + ): FixedPolicyCondition => [ + operator, + ['field', 'request.totalEstimatedMediaTokens'], + value, + ]; + + it('computes request.totalEstimatedMediaTokens from the pending delivery set', async () => { + mockToolSuccess(); + const { records } = await runFixedPolicies(config, source, { + store, + policies: [makePolicy({ when: requestWhen('>', 5859) })], + }); + expect(executeToolCallMock).toHaveBeenCalledTimes(1); + expect(records[0]).toMatchObject({ outcome: 'succeeded' }); + }); + + it('does not match when the pending total is not above the threshold', async () => { + const { records } = await runFixedPolicies(config, source, { + store, + policies: [makePolicy({ when: requestWhen('>', 5860) })], + }); + expect(records).toEqual([]); + expect(executeToolCallMock).not.toHaveBeenCalled(); + }); + + it('prefers a caller-supplied request namespace over the internal sum', async () => { + const { records } = await runFixedPolicies(config, source, { + store, + policies: [makePolicy({ when: requestWhen('>', 5859) })], + conditionContext: { request: { totalEstimatedMediaTokens: 1 } }, + }); + expect(records).toEqual([]); + expect(executeToolCallMock).not.toHaveBeenCalled(); + }); + + it('reads unavailable (never a partial sum) when a pending resource is unestimable', async () => { + source = { ...source, recognized: recognizedImage({ metadata: {} }) }; + const { records } = await runFixedPolicies(config, source, { + store, + policies: [makePolicy({ when: requestWhen('>', 0) })], + }); + expect(records).toEqual([ + { + policyId: 'img-downsample', + toolName: 'omni_downsample_image', + outcome: 'condition_unavailable', + resource: 'photo.png', + missingFields: ['request.totalEstimatedMediaTokens'], + }, + ]); + expect(executeToolCallMock).not.toHaveBeenCalled(); + }); + + it('recomputes the request namespace as derivatives enter the next pass', async () => { + mockToolSuccess(); + const { records } = await runFixedPolicies(config, source, { + store, + policies: [ + makePolicy({ + id: 'a-derive', + output: { + reprocessMedia: true, + source: 'omit', + artifacts: { '*': 'include' }, + }, + }), + // Runs only on the derivative pass: eq 901 is the DERIVATIVE + // total after the root left the delivery set — the root pass + // total was 5860, so a start-of-run snapshot would never match. + makePolicy({ + id: 'b-on-derivative', + origins: ['policy'], + arguments: { maxDimension: 800 }, + when: requestWhen('==', 901), + }), + ], + }); + expect(executeToolCallMock).toHaveBeenCalledTimes(2); + // b-on-derivative EXECUTED (its `when` matched the recomputed total); + // the mock returns bytes identical to its input, so the run records + // as a no-op rather than a degradation — execution is the assertion. + expect(records.map((r) => [r.policyId, r.outcome])).toEqual([ + ['a-derive', 'succeeded'], + ['b-on-derivative', 'no_op'], + ]); + }); + + it('evaluates session.* from the caller-supplied per-delivery snapshot', async () => { + mockToolSuccess(); + const sessionWhen: FixedPolicyCondition = [ + '<=', + ['field', 'session.availableContextTokens'], + 700, + ]; + const { records } = await runFixedPolicies(config, source, { + store, + policies: [makePolicy({ when: sessionWhen })], + conditionContext: { session: { availableContextTokens: 700 } }, + }); + expect(executeToolCallMock).toHaveBeenCalledTimes(1); + expect(records[0]).toMatchObject({ outcome: 'succeeded' }); + }); + + it('reads session.* as unavailable when no snapshot was supplied', async () => { + const { records } = await runFixedPolicies(config, source, { + store, + policies: [ + makePolicy({ + when: ['<=', ['field', 'session.availableContextTokens'], 700], + }), + ], + }); + expect(records).toEqual([ + { + policyId: 'img-downsample', + toolName: 'omni_downsample_image', + outcome: 'condition_unavailable', + resource: 'photo.png', + missingFields: ['session.availableContextTokens'], + }, + ]); + expect(executeToolCallMock).not.toHaveBeenCalled(); + }); + }); + + it('caps re-derivation per lineage via maxRunsPerLineage', async () => { + mockToolSuccess(); + const { deliveries } = await runFixedPolicies(config, source, { + store, + policies: [ + makePolicy({ + origins: ['user', 'tool', 'policy'], + maxRunsPerLineage: 1, + output: { + reprocessMedia: true, + source: 'omit', + artifacts: { '*': 'include' }, + }, + }), + ], + }); + // The derivative re-enters matching but the lineage already spent the + // policy's single run — exactly one execution, derivative delivered. + expect(executeToolCallMock).toHaveBeenCalledTimes(1); + expect(deliveries).toHaveLength(1); + expect(deliveries[0].degraded).toBe(true); + }); + + it('keeps the source and continues on failure when onFailure is continue', async () => { + executeToolCallMock.mockResolvedValue({ + callId: 'x', + responseParts: [], + resultDisplay: undefined, + error: new Error('ffmpeg exploded'), + errorType: undefined, + }); + const { deliveries, records } = await runFixedPolicies(config, source, { + store, + policies: [makePolicy()], + }); + expect(deliveries).toHaveLength(1); + expect(deliveries[0].filePath).toBe(sourcePath); + expect(records).toEqual([ + { + policyId: 'img-downsample', + toolName: 'omni_downsample_image', + outcome: 'failed', + resource: 'photo.png', + error: 'ffmpeg exploded', + }, + ]); + // Failure never leaves partial staging state (D10 Stage A). + await expect(fs.readdir(store.getStagingDir())).resolves.toEqual([]); + }); + + it('throws OmniPolicyExecutionError when onFailure is abort', async () => { + executeToolCallMock.mockResolvedValue({ + callId: 'x', + responseParts: [], + resultDisplay: undefined, + error: new Error('ffmpeg exploded'), + errorType: undefined, + }); + await expect( + runFixedPolicies(config, source, { + store, + policies: [makePolicy({ onFailure: 'abort' })], + }), + ).rejects.toMatchObject({ + name: 'OmniPolicyExecutionError', + policyId: 'img-downsample', + message: 'Fixed policy img-downsample failed: ffmpeg exploded', + }); + }); + + it('fails closed on transport_guard-stage failures regardless of onFailure', async () => { + executeToolCallMock.mockResolvedValue({ + callId: 'x', + responseParts: [], + resultDisplay: undefined, + error: new Error('guard tool crashed'), + errorType: undefined, + }); + await expect( + runFixedPolicies(config, source, { + store, + policies: [ + makePolicy({ onFailure: 'continue', stage: 'transport_guard' }), + ], + }), + ).rejects.toBeInstanceOf(OmniPolicyExecutionError); + }); + + it('rejects an artifact whose workspacePath escapes the staging directory', async () => { + const evilPath = path.join(tmpDir, 'evil.jpg'); + executeToolCallMock.mockImplementation( + async (_config: Config, request: ToolCallRequestInfo) => { + await fs.writeFile(evilPath, DEGRADED_BYTES); + return { + callId: request.callId, + responseParts: [], + resultDisplay: undefined, + error: undefined, + errorType: undefined, + policyArtifacts: { + toolName: request.name, + invocationId: request.callId, + executionOrigin: request.executionOrigin, + artifacts: [ + { + kind: 'image', + storage: 'workspace', + title: 'evil.jpg', + workspacePath: path.relative( + path.join(store.getStagingDir(), request.callId), + evilPath, + ), + metadata: { omniDisclosure: 'x' }, + }, + ], + }, + }; + }, + ); + const { deliveries, records } = await runFixedPolicies(config, source, { + store, + policies: [makePolicy()], + }); + expect(records[0]).toMatchObject({ outcome: 'failed' }); + expect(records[0].error).toContain('escapes the staging directory'); + expect(deliveries[0].filePath).toBe(sourcePath); + }); + + it('rejects a lossy artifact that carries no omniDisclosure', async () => { + mockToolSuccess({ disclosure: undefined }); + const { records } = await runFixedPolicies(config, source, { + store, + policies: [makePolicy()], + }); + expect(records[0]).toMatchObject({ outcome: 'failed' }); + expect(records[0].error).toContain('lossy but carries no omniDisclosure'); + }); + + it('fails the run when the tool succeeds but returns no policy artifacts', async () => { + executeToolCallMock.mockImplementation( + async (_config: Config, request: ToolCallRequestInfo) => ({ + callId: request.callId, + responseParts: [], + resultDisplay: undefined, + error: undefined, + errorType: undefined, + // No policyArtifacts at all — e.g. a tool that "succeeded" without + // emitting through the artifact protocol. + policyArtifacts: undefined, + }), + ); + const { deliveries, records } = await runFixedPolicies(config, source, { + store, + policies: [makePolicy()], + }); + expect(records[0]).toMatchObject({ outcome: 'failed' }); + expect(records[0].error).toContain('produced no policy artifacts'); + expect(deliveries[0].filePath).toBe(sourcePath); + }); + + it('rejects an artifact whose recognized media type is not declared by the descriptor', async () => { + // The tool writes a GIF, but the descriptor only declares image/jpeg + // outputs. Recognition of the actual bytes is authoritative — the + // artifact's own declared mimeType never enters the check. + recognizeMediaFileMock.mockImplementation(async (filePath: string) => { + if (filePath.endsWith('.gif')) { + return { + modality: 'image' as const, + detectedMimeType: 'image/gif', + sizeBytes: DEGRADED_BYTES.length, + metadata: {}, + }; + } + return recognizedImage(); + }); + mockToolSuccess({ fileName: 'out.gif' }); + const { deliveries, records } = await runFixedPolicies(config, source, { + store, + policies: [makePolicy()], + }); + expect(records[0]).toMatchObject({ outcome: 'failed' }); + expect(records[0].error).toContain('undeclared media type image/gif'); + expect(deliveries[0].filePath).toBe(sourcePath); + }); + + it('rejects an artifact whose declared kind mismatches the recognized content', async () => { + // Bytes recognize as image/jpeg (declared by the descriptor), but the + // artifact claims to be audio — the cross-check must fail closed. + executeToolCallMock.mockImplementation( + async (_config: Config, request: ToolCallRequestInfo) => { + const outputDir = request.args['outputDir'] as string; + await fs.writeFile(path.join(outputDir, 'out.jpg'), DEGRADED_BYTES); + return { + callId: request.callId, + responseParts: [], + resultDisplay: undefined, + error: undefined, + errorType: undefined, + policyArtifacts: { + toolName: request.name, + invocationId: request.callId, + executionOrigin: request.executionOrigin, + artifacts: [ + { + kind: 'audio', + storage: 'workspace', + title: 'out.jpg', + workspacePath: 'out.jpg', + metadata: { omniDisclosure: 'x' }, + }, + ], + }, + }; + }, + ); + const { records } = await runFixedPolicies(config, source, { + store, + policies: [makePolicy()], + }); + expect(records[0]).toMatchObject({ outcome: 'failed' }); + expect(records[0].error).toContain( + 'declares kind audio but contains image content', + ); + }); + + it('fails the run when a required media output was not produced (§5 completeness)', async () => { + // Descriptor: jpeg is required, png is an optional lossless extra. The + // tool only produces the png — it validates fine on its own, so only + // assertRequiredOutputsPresent can catch the missing jpeg. + const twoOutputDescriptor: MediaPolicyToolDescriptor = { + kind: 'media_policy', + inputMediaTypes: ['image'], + outputs: [ + { + kind: 'media', + mimeTypes: ['image/jpeg'], + required: true, + lossy: true, + }, + { kind: 'media', mimeTypes: ['image/png'], required: false }, + ], + }; + const twoOutputConfig = makeConfig({ + omni_downsample_image: twoOutputDescriptor, + }); + mockToolSuccess({ fileName: 'out.png', disclosure: undefined }); + const { deliveries, records } = await runFixedPolicies( + twoOutputConfig, + source, + { store, policies: [makePolicy()] }, + ); + expect(records[0]).toMatchObject({ outcome: 'failed' }); + expect(records[0].error).toContain( + 'did not produce its required media image/jpeg output', + ); + expect(deliveries[0].filePath).toBe(sourcePath); + }); + + it('rejects a tool without a media-policy descriptor', async () => { + const { records } = await runFixedPolicies(makeConfig({}), source, { + store, + policies: [makePolicy()], + }); + expect(records[0]).toMatchObject({ outcome: 'failed' }); + expect(records[0].error).toContain('not a registered media-policy tool'); + }); + + it('executes policies in priority order, ties broken by id', async () => { + mockToolSuccess(); + // Distinct arguments per policy: identical arguments would fingerprint + // identically and turn the later runs into degradation-cache hits. + await runFixedPolicies(config, source, { + store, + policies: [ + makePolicy({ + id: 'b-low', + priority: 1, + arguments: { maxDimension: 100 }, + output: { + reprocessMedia: false, + source: 'keep', + artifacts: { '*': 'include' }, + }, + }), + makePolicy({ + id: 'a-high', + priority: 10, + arguments: { maxDimension: 300 }, + output: { + reprocessMedia: false, + source: 'keep', + artifacts: { '*': 'include' }, + }, + }), + makePolicy({ + id: 'a-low', + priority: 1, + arguments: { maxDimension: 200 }, + output: { + reprocessMedia: false, + source: 'keep', + artifacts: { '*': 'include' }, + }, + }), + ], + }); + const order = executeToolCallMock.mock.calls.map((call) => { + const origin = (call[1] as ToolCallRequestInfo).executionOrigin; + return origin?.kind === 'fixed_policy' ? origin.policyId : undefined; + }); + expect(order).toEqual(['a-high', 'a-low', 'b-low']); + }); + + it('stops BEFORE executing once maxPolicyRunsPerRoot is spent, recording budget_exhausted', async () => { + mockToolSuccess(); + // Distinct arguments: identical ones would fingerprint identically and + // make the second policy a (budget-free) degradation-cache hit. + const { deliveries, records } = await runFixedPolicies(config, source, { + store, + policies: [ + makePolicy({ + id: 'a-first', + arguments: { maxDimension: 100 }, + output: { + reprocessMedia: false, + source: 'keep', + artifacts: { '*': 'include' }, + }, + }), + makePolicy({ + id: 'b-second', + arguments: { maxDimension: 200 }, + output: { + reprocessMedia: false, + source: 'keep', + artifacts: { '*': 'include' }, + }, + }), + ], + limits: limitsWith({ maxPolicyRunsPerRoot: 1 }), + }); + expect(executeToolCallMock).toHaveBeenCalledTimes(1); + expect(records).toEqual([ + { + policyId: 'a-first', + toolName: 'omni_downsample_image', + outcome: 'succeeded', + resource: 'photo.png', + }, + { + policyId: 'b-second', + toolName: 'omni_downsample_image', + outcome: 'budget_exhausted', + resource: 'photo.png', + error: 'maxPolicyRunsPerRoot (1) reached', + }, + ]); + // The committed delivery stands (no rollback): source + derivative. + expect(deliveries.map((d) => d.filePath)).toEqual([ + sourcePath, + store.objectPathFor(sha256Of(DEGRADED_BYTES), '.jpg'), + ]); + }); + + it('stops deriving when maxArtifactsPerRoot is exceeded but keeps the committed delivery', async () => { + mockToolSuccess(); + const { deliveries, records } = await runFixedPolicies(config, source, { + store, + policies: [ + makePolicy(), + makePolicy({ id: 'never-runs', arguments: { maxDimension: 300 } }), + ], + limits: limitsWith({ maxArtifactsPerRoot: 0 }), + }); + expect(executeToolCallMock).toHaveBeenCalledTimes(1); + expect(records).toEqual([ + { + policyId: 'img-downsample', + toolName: 'omni_downsample_image', + outcome: 'succeeded', + resource: 'photo.png', + }, + { + policyId: 'img-downsample', + toolName: 'omni_downsample_image', + outcome: 'budget_exhausted', + resource: 'photo.png', + error: 'maxArtifactsPerRoot (0) exceeded', + }, + ]); + // source: 'omit' already applied — the derivative alone is delivered. + expect(deliveries).toHaveLength(1); + expect(deliveries[0].degraded).toBe(true); + }); + + it('stops deriving when maxDerivedBytesPerRoot is exceeded', async () => { + mockToolSuccess(); // DEGRADED_BYTES is 20 bytes > the 10-byte budget + const { deliveries, records } = await runFixedPolicies(config, source, { + store, + policies: [ + makePolicy(), + makePolicy({ id: 'never-runs', arguments: { maxDimension: 300 } }), + ], + limits: limitsWith({ maxDerivedBytesPerRoot: 10 }), + }); + expect(executeToolCallMock).toHaveBeenCalledTimes(1); + expect(records[1]).toEqual({ + policyId: 'img-downsample', + toolName: 'omni_downsample_image', + outcome: 'budget_exhausted', + resource: 'photo.png', + error: 'maxDerivedBytesPerRoot (10) exceeded', + }); + expect(deliveries).toHaveLength(1); + expect(deliveries[0].degraded).toBe(true); + }); + + it('clamps reprocessing at maxLineageDepth even when lineage runs remain', async () => { + // Distinct bytes per run: identical output would end the chain as a + // no_op fixed point before the depth clamp could matter. + let round = 0; + executeToolCallMock.mockImplementation( + async (_config: Config, request: ToolCallRequestInfo) => { + round++; + const outputDir = request.args['outputDir'] as string; + await fs.writeFile(path.join(outputDir, 'out.jpg'), `round-${round}`); + return { + callId: request.callId, + responseParts: [], + resultDisplay: undefined, + error: undefined, + errorType: undefined, + policyArtifacts: { + toolName: request.name, + invocationId: request.callId, + executionOrigin: request.executionOrigin, + artifacts: [ + { + kind: 'image', + storage: 'workspace', + title: 'out.jpg', + workspacePath: 'out.jpg', + mimeType: 'image/jpeg', + metadata: { omniDisclosure: `round ${round}` }, + }, + ], + }, + }; + }, + ); + const { deliveries } = await runFixedPolicies(config, source, { + store, + policies: [ + makePolicy({ + origins: ['user', 'tool', 'policy'], + maxRunsPerLineage: 10, + output: { + reprocessMedia: true, + source: 'omit', + artifacts: { '*': 'include' }, + }, + }), + ], + limits: limitsWith({ maxLineageDepth: 2 }), + }); + // root(depth 0) → run 1 → depth-1 child re-enters → run 2 → the + // depth-2 child delivers but does NOT re-enter (2 is not < 2). The + // lineage cap alone (10) would have allowed further runs. + expect(executeToolCallMock).toHaveBeenCalledTimes(2); + expect(deliveries).toHaveLength(1); + expect(deliveries[0].sha256).toBe(sha256Of('round-2')); + }); + + it('quarantines the staging dir with a reason.json when the invocation fails (D10 Stage B)', async () => { + executeToolCallMock.mockResolvedValue({ + callId: 'x', + responseParts: [], + resultDisplay: undefined, + error: new Error('ffmpeg exploded'), + errorType: undefined, + }); + await runFixedPolicies(config, source, { + store, + policies: [makePolicy()], + }); + await expect(fs.readdir(store.getStagingDir())).resolves.toEqual([]); + const quarantined = await fs.readdir(store.getQuarantineDir()); + expect(quarantined).toHaveLength(1); + const reason = JSON.parse( + await fs.readFile( + path.join(store.getQuarantineDir(), quarantined[0], 'reason.json'), + 'utf8', + ), + ) as Record; + expect(reason).toMatchObject({ + policyId: 'img-downsample', + toolName: 'omni_downsample_image', + reason: 'ffmpeg exploded', + }); + expect(typeof reason['failedAt']).toBe('string'); + }); + + it('removes (not quarantines) staging when the failure is a user abort', async () => { + const controller = new AbortController(); + executeToolCallMock.mockImplementation(async () => { + controller.abort(); + throw new Error('aborted mid-flight'); + }); + await expect( + runFixedPolicies(config, source, { + store, + policies: [makePolicy()], + signal: controller.signal, + }), + ).rejects.toThrow('aborted mid-flight'); + await expect(fs.readdir(store.getStagingDir())).resolves.toEqual([]); + await expect( + fs.readdir(store.getQuarantineDir()).catch(() => []), + ).resolves.toEqual([]); + }); + + it('falls back to plain staging removal when quarantining itself fails', async () => { + vi.spyOn(store, 'quarantineInvocation').mockRejectedValue( + new Error('quarantine disk full'), + ); + executeToolCallMock.mockResolvedValue({ + callId: 'x', + responseParts: [], + resultDisplay: undefined, + error: new Error('ffmpeg exploded'), + errorType: undefined, + }); + const { records } = await runFixedPolicies(config, source, { + store, + policies: [makePolicy()], + }); + expect(records[0]).toMatchObject({ + outcome: 'failed', + error: 'ffmpeg exploded', + }); + // The failed invocation still never leaves live staging state behind. + await expect(fs.readdir(store.getStagingDir())).resolves.toEqual([]); + }); + + describe('tool-level settings defaults (omni.processing.policyTools..settings)', () => { + it('merges settings under policy arguments in BOTH the tool call and the cache fingerprint', async () => { + mockToolSuccess(); + const configured = makeConfig( + { omni_downsample_image: DESCRIPTOR }, + { omni_downsample_image: { settings: { quality: 60 } } }, + ); + await runFixedPolicies(configured, source, { + store, + policies: [makePolicy()], + }); + + const req = executeToolCallMock.mock.calls[0][1] as ToolCallRequestInfo; + expect(req.args).toMatchObject({ maxDimension: 1568, quality: 60 }); + + // Fingerprint must include the merged tunables — otherwise editing + // settings would keep serving derivatives made under the old values. + const cache = new OmniDegradationCache(store.getOmniRootDir()); + await expect( + cache.get( + sha256Of(SOURCE_BYTES), + computePolicyFingerprint('omni_downsample_image', { + maxDimension: 1568, + quality: 60, + }), + ), + ).resolves.not.toBeNull(); + await expect( + cache.get( + sha256Of(SOURCE_BYTES), + computePolicyFingerprint('omni_downsample_image', { + maxDimension: 1568, + }), + ), + ).resolves.toBeNull(); + }); + + it('policy arguments override colliding settings keys', async () => { + mockToolSuccess(); + const configured = makeConfig( + { omni_downsample_image: DESCRIPTOR }, + { omni_downsample_image: { settings: { maxDimension: 99 } } }, + ); + await runFixedPolicies(configured, source, { + store, + policies: [makePolicy()], // arguments: { maxDimension: 1568 } + }); + const req = executeToolCallMock.mock.calls[0][1] as ToolCallRequestInfo; + expect(req.args['maxDimension']).toBe(1568); + }); + + it.each([ + ['null tombstone', { omni_downsample_image: null }], + ['non-object settings', { omni_downsample_image: { settings: 'evil' } }], + ['array settings', { omni_downsample_image: { settings: [1, 2] } }], + ['absent map', undefined], + ])('ignores malformed settings entries: %s', async (_label, settings) => { + mockToolSuccess(); + const configured = makeConfig( + { omni_downsample_image: DESCRIPTOR }, + settings, + ); + await runFixedPolicies(configured, source, { + store, + policies: [makePolicy()], + }); + const req = executeToolCallMock.mock.calls[0][1] as ToolCallRequestInfo; + expect(req.args).toEqual({ + maxDimension: 1568, + inputPath: sourcePath, + outputDir: expect.stringContaining(store.getStagingDir()), + }); + }); + }); + + it('re-hashes a cache hit before reuse: poisoned object bytes trigger re-transcode (D2 integrity)', async () => { + const degradedSha = sha256Of(DEGRADED_BYTES); + const objectPath = store.objectPathFor(degradedSha, '.jpg'); + await fs.mkdir(path.dirname(objectPath), { recursive: true }); + // A file EXISTS at the addressed path but its bytes do not hash to the + // entry's identity — planted via a crafted policy-cache.json plus a + // foreign object (or plain store corruption). + await fs.writeFile(objectPath, 'not-the-degraded-bytes'); + const cache = new OmniDegradationCache(store.getOmniRootDir()); + const fingerprint = computePolicyFingerprint('omni_downsample_image', { + maxDimension: 1568, + }); + await cache.put(sha256Of(SOURCE_BYTES), fingerprint, { + degradedSha256: degradedSha, + extension: '.jpg', + disclosure: 'poisoned disclosure', + mimeType: 'image/jpeg', + }); + mockToolSuccess(); + + const { deliveries, records } = await runFixedPolicies(config, source, { + store, + policies: [makePolicy()], + }); + + // The mismatching object was never served: the tool re-ran and the + // store now holds verified bytes under the hash. + expect(executeToolCallMock).toHaveBeenCalledTimes(1); + expect(records[0]).toMatchObject({ outcome: 'succeeded' }); + expect(deliveries[0].sha256).toBe(degradedSha); + expect(deliveries[0].disclosure).toBe( + 'Downsampled from 4000x3000 to 1568x1176.', + ); + await expect(fs.readFile(objectPath, 'utf8')).resolves.toBe(DEGRADED_BYTES); + }); + + describe('file artifacts (transcript protocol, §6.2)', () => { + const TRANSCRIPT_TEXT = '你好，世界'; + const TRANSCRIPT_DISCLOSURE = '原 63s 音频 → 转写文本 5 字'; + + const TRANSCRIPT_DESCRIPTOR: MediaPolicyToolDescriptor = { + kind: 'media_policy', + inputMediaTypes: ['image'], + outputs: [ + { + kind: 'file', + role: 'transcript', + mimeTypes: ['text/plain'], + required: true, + lossy: true, + }, + { kind: 'text', role: 'disclosure', required: true }, + ], + }; + + /** Tool mock producing one `kind: 'file'` artifact. */ + function mockFileArtifact( + options: { + bytes?: Buffer | string; + mimeType?: string; + role?: string; + disclosure?: string | undefined; + } = {}, + ): void { + const bytes = options.bytes ?? TRANSCRIPT_TEXT; + const mimeType = options.mimeType ?? 'text/plain'; + const role = options.role ?? 'transcript'; + const disclosure = + 'disclosure' in options ? options.disclosure : TRANSCRIPT_DISCLOSURE; + executeToolCallMock.mockImplementation( + async (_config: Config, request: ToolCallRequestInfo) => { + const outputDir = request.args['outputDir'] as string; + await fs.writeFile(path.join(outputDir, 'transcript.txt'), bytes); + return { + callId: request.callId, + responseParts: [], + resultDisplay: undefined, + error: undefined, + errorType: undefined, + policyArtifacts: { + toolName: request.name, + invocationId: request.callId, + executionOrigin: request.executionOrigin, + artifacts: [ + { + kind: 'file', + storage: 'workspace', + title: 'Audio transcript', + workspacePath: 'transcript.txt', + mimeType, + metadata: { + ...(disclosure !== undefined + ? { omniDisclosure: disclosure } + : {}), + omniRole: role, + }, + }, + ], + }, + }; + }, + ); + } + + function transcriptPolicy( + overrides: Partial = {}, + ): NormalizedFixedPolicy { + return makePolicy({ + id: 'img-transcribe', + toolName: 'omni_transcribe_stub', + arguments: {}, + output: { + reprocessMedia: false, + source: 'omit', + artifacts: { 'role:transcript': 'include' }, + }, + ...overrides, + }); + } + + beforeEach(() => { + config = makeConfig({ omni_transcribe_stub: TRANSCRIPT_DESCRIPTOR }); + }); + + it('validates and promotes the transcript into fileDeliveries with text + disclosure', async () => { + mockFileArtifact(); + const { deliveries, fileDeliveries, records } = await runFixedPolicies( + config, + source, + { store, policies: [transcriptPolicy()] }, + ); + + // source omitted, no media derivative — pure-transcript outcome. + expect(deliveries).toEqual([]); + expect(records).toEqual([ + { + policyId: 'img-transcribe', + toolName: 'omni_transcribe_stub', + outcome: 'succeeded', + resource: 'photo.png', + }, + ]); + const transcriptSha = sha256Of(TRANSCRIPT_TEXT); + expect(fileDeliveries).toEqual([ + { + filePath: store.objectPathFor(transcriptSha, '.txt'), + role: 'transcript', + mimeType: 'text/plain', + text: TRANSCRIPT_TEXT, + sha256: transcriptSha, + sizeBytes: Buffer.byteLength(TRANSCRIPT_TEXT, 'utf-8'), + disclosure: TRANSCRIPT_DISCLOSURE, + }, + ]); + // Promoted (content-addressed, immutable) — not left in staging. + await expect( + fs.readFile(store.objectPathFor(transcriptSha, '.txt'), 'utf-8'), + ).resolves.toBe(TRANSCRIPT_TEXT); + }); + + it('a retain selector keeps the transcript out of fileDeliveries', async () => { + mockFileArtifact(); + const { fileDeliveries, records } = await runFixedPolicies( + config, + source, + { + store, + policies: [ + transcriptPolicy({ + output: { + reprocessMedia: false, + source: 'omit', + artifacts: { 'role:transcript': 'retain', '*': 'include' }, + }, + }), + ], + }, + ); + expect(records[0]).toMatchObject({ outcome: 'succeeded' }); + expect(fileDeliveries).toEqual([]); + }); + + it('an artifact no selector matches defaults to retain (no "*" entry)', async () => { + mockFileArtifact(); + const { fileDeliveries, records } = await runFixedPolicies( + config, + source, + { + store, + policies: [ + transcriptPolicy({ + output: { reprocessMedia: false, source: 'omit', artifacts: {} }, + }), + ], + }, + ); + expect(records[0]).toMatchObject({ outcome: 'succeeded' }); + expect(fileDeliveries).toEqual([]); + }); + + it('rejects a file artifact that is not valid UTF-8', async () => { + mockFileArtifact({ bytes: Buffer.from([0xff, 0xfe, 0x80, 0x00]) }); + const { fileDeliveries, records } = await runFixedPolicies( + config, + source, + { store, policies: [transcriptPolicy()] }, + ); + expect(records[0]).toMatchObject({ + outcome: 'failed', + error: expect.stringContaining('is not valid UTF-8 text'), + }); + expect(fileDeliveries).toEqual([]); + }); + + it('rejects a file artifact over MAX_FILE_ARTIFACT_BYTES', async () => { + mockFileArtifact({ bytes: 'a'.repeat(MAX_FILE_ARTIFACT_BYTES + 1) }); + const { records } = await runFixedPolicies(config, source, { + store, + policies: [transcriptPolicy()], + }); + expect(records[0]).toMatchObject({ + outcome: 'failed', + error: expect.stringContaining( + `exceeds the file-artifact size budget (${MAX_FILE_ARTIFACT_BYTES + 1} > ${MAX_FILE_ARTIFACT_BYTES} bytes)`, + ), + }); + }); + + it('rejects a lossy file artifact without omniDisclosure', async () => { + mockFileArtifact({ disclosure: undefined }); + const { records } = await runFixedPolicies(config, source, { + store, + policies: [transcriptPolicy()], + }); + expect(records[0]).toMatchObject({ + outcome: 'failed', + error: expect.stringContaining( + 'is lossy but carries no omniDisclosure', + ), + }); + }); + + it('rejects a file artifact whose mimeType matches no declared file output', async () => { + mockFileArtifact({ mimeType: 'text/markdown' }); + const { records } = await runFixedPolicies(config, source, { + store, + policies: [transcriptPolicy()], + }); + expect(records[0]).toMatchObject({ + outcome: 'failed', + error: expect.stringContaining('matches no declared file output'), + }); + }); + + it('fails the run when the required file output was not produced (§5 completeness)', async () => { + // Descriptor requires BOTH a media and a transcript output; the tool + // only produces the media derivative. + config = makeConfig({ + omni_transcribe_stub: { + kind: 'media_policy', + inputMediaTypes: ['image'], + outputs: [ + { + kind: 'media', + mimeTypes: ['image/jpeg'], + required: true, + lossy: true, + }, + { + kind: 'file', + role: 'transcript', + mimeTypes: ['text/plain'], + required: true, + lossy: true, + }, + { kind: 'text', role: 'disclosure', required: true }, + ], + }, + }); + mockToolSuccess(); + const { records } = await runFixedPolicies(config, source, { + store, + policies: [transcriptPolicy()], + }); + expect(records[0]).toMatchObject({ + outcome: 'failed', + error: expect.stringContaining( + 'did not produce its required file text/plain output', + ), + }); + }); + + it('never serves file artifacts from the degradation cache (re-runs the tool)', async () => { + mockFileArtifact(); + await runFixedPolicies(config, source, { + store, + policies: [transcriptPolicy()], + }); + const second = await runFixedPolicies(config, source, { + store, + policies: [transcriptPolicy()], + }); + expect(executeToolCallMock).toHaveBeenCalledTimes(2); + expect(second.records[0]).toMatchObject({ outcome: 'succeeded' }); + expect(second.fileDeliveries).toHaveLength(1); + }); + + it('counts file artifacts toward maxArtifactsPerRoot', async () => { + mockFileArtifact(); + const { records, fileDeliveries } = await runFixedPolicies( + config, + source, + { + store, + policies: [ + transcriptPolicy({ id: 'transcribe-a' }), + transcriptPolicy({ id: 'transcribe-b' }), + ], + limits: limitsWith({ maxArtifactsPerRoot: 1 }), + }, + ); + // The second run tips the count over the budget: its delivery stands + // but derivation stops with an explicit budget_exhausted record. + expect(records.map((r) => r.outcome)).toEqual([ + 'succeeded', + 'succeeded', + 'budget_exhausted', + ]); + expect(fileDeliveries).toHaveLength(2); + }); + }); + + describe('maxConcurrentResources gates concurrent runs per omni root', () => { + /** Tool mock that parks each invocation on a caller-released latch, + * recording how many invocations are in flight simultaneously. */ + function mockGatedTool() { + let inFlight = 0; + let peak = 0; + const releases: Array<() => void> = []; + executeToolCallMock.mockImplementation( + async (_config: Config, request: ToolCallRequestInfo) => { + inFlight++; + peak = Math.max(peak, inFlight); + await new Promise((resolve) => releases.push(resolve)); + inFlight--; + const outputDir = request.args['outputDir'] as string; + const bytes = `degraded-${request.callId}`; + await fs.writeFile(path.join(outputDir, 'out.jpg'), bytes); + return { + callId: request.callId, + responseParts: [], + resultDisplay: undefined, + error: undefined, + errorType: undefined, + policyArtifacts: { + toolName: request.name, + invocationId: request.callId, + executionOrigin: request.executionOrigin, + artifacts: [ + { + kind: 'image', + storage: 'workspace', + title: 'out.jpg', + workspacePath: 'out.jpg', + mimeType: 'image/jpeg', + metadata: { omniDisclosure: 'gated' }, + }, + ], + }, + }; + }, + ); + return { + peak: () => peak, + started: () => releases.length, + releaseAll: () => { + for (const release of releases.splice(0)) release(); + }, + }; + } + + async function makeSecondSource(): Promise { + const secondPath = path.join(tmpDir, 'photo-2.png'); + await fs.writeFile(secondPath, 'second-image-bytes'); + return { + filePath: secondPath, + recognized: recognizedImage({ sizeBytes: 18 }), + displayName: 'photo-2.png', + origin: 'user', + }; + } + + it('limit 1: the second resource waits until the first fully finishes', async () => { + const gate = mockGatedTool(); + const second = await makeSecondSource(); + const options = { + store, + policies: [makePolicy()], + limits: limitsWith({ maxConcurrentResources: 1 }), + }; + + const run1 = runFixedPolicies(config, source, options); + const run2 = runFixedPolicies(config, second, options); + // Give both runs every chance to start their tool call. + await vi.waitFor(() => expect(gate.started()).toBe(1)); + await new Promise((r) => setTimeout(r, 20)); + expect(gate.started()).toBe(1); // run2 is parked on the gate + + gate.releaseAll(); // finish run1 → slot transfers to run2 + await vi.waitFor(() => expect(gate.started()).toBe(1)); // fresh latch + gate.releaseAll(); + await Promise.all([run1, run2]); + expect(gate.peak()).toBe(1); + expect(executeToolCallMock).toHaveBeenCalledTimes(2); + }); + + it('limit 2: both resources transcode simultaneously', async () => { + const gate = mockGatedTool(); + const second = await makeSecondSource(); + const options = { + store, + policies: [makePolicy()], + limits: limitsWith({ maxConcurrentResources: 2 }), + }; + + const run1 = runFixedPolicies(config, source, options); + const run2 = runFixedPolicies(config, second, options); + await vi.waitFor(() => expect(gate.started()).toBe(2)); + expect(gate.peak()).toBe(2); + gate.releaseAll(); + await Promise.all([run1, run2]); + }); + + it('a failed run releases its slot (no deadlock for the waiter)', async () => { + const second = await makeSecondSource(); + executeToolCallMock + .mockResolvedValueOnce({ + callId: 'x', + responseParts: [], + resultDisplay: undefined, + error: new Error('ffmpeg exploded'), + errorType: undefined, + }) + .mockImplementation( + async (_c: Config, request: ToolCallRequestInfo) => { + const outputDir = request.args['outputDir'] as string; + await fs.writeFile(path.join(outputDir, 'out.jpg'), DEGRADED_BYTES); + return { + callId: request.callId, + responseParts: [], + resultDisplay: undefined, + error: undefined, + errorType: undefined, + policyArtifacts: { + toolName: request.name, + invocationId: request.callId, + executionOrigin: request.executionOrigin, + artifacts: [ + { + kind: 'image', + storage: 'workspace', + title: 'out.jpg', + workspacePath: 'out.jpg', + mimeType: 'image/jpeg', + metadata: { omniDisclosure: 'ok' }, + }, + ], + }, + }; + }, + ); + const options = { + store, + policies: [makePolicy({ onFailure: 'abort' as const })], + limits: limitsWith({ maxConcurrentResources: 1 }), + }; + await expect(runFixedPolicies(config, source, options)).rejects.toThrow( + OmniPolicyExecutionError, + ); + // The waiter (or any later run) must still get the slot. + const { records } = await runFixedPolicies(config, second, options); + expect(records[0]).toMatchObject({ outcome: 'succeeded' }); + }); + }); +}); diff --git a/packages/core/src/omni/policy/orchestrator.ts b/packages/core/src/omni/policy/orchestrator.ts new file mode 100644 index 00000000000..e29f61f496d --- /dev/null +++ b/packages/core/src/omni/policy/orchestrator.ts @@ -0,0 +1,1064 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { createHash, randomBytes } from 'node:crypto'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import type { Config } from '../../config/config.js'; +import type { ToolCallRequestInfo } from '../../core/turn.js'; +import type { + MediaPolicyToolDescriptor, + ToolArtifact, +} from '../../tools/tools.js'; +import { createDebugLogger } from '../../utils/debugLogger.js'; +import { estimateRawResourceTokens } from '../estimation.js'; +import { + extensionForMime, + hashFileSha256, + recognizeMediaFile, + type RecognizedMedia, +} from '../recognition.js'; +import type { OmniObjectStore } from '../storage.js'; +import { + evaluateFixedPolicyCondition, + type FixedPolicyConditionContext, + type RequestConditionField, + type ResourceConditionField, +} from './conditions.js'; +import { + computePolicyFingerprint, + OmniDegradationCache, +} from './degradation-cache.js'; +import { DEFAULT_OMNI_PROCESSING_LIMITS } from './config.js'; +import { resolvePolicyToolSettings } from './tools/media-policy-tool.js'; +import type { + FixedPolicyOrigin, + NormalizedFixedPolicy, + NormalizedOmniProcessingLimits, +} from './types.js'; + +const debugLogger = createDebugLogger('omni:policy'); + +/** One resource in the final delivery set produced by the orchestrator. */ +export interface PolicyDeliveryResource { + /** Absolute path of the deliverable file (the original input, or a + * promoted derivative inside `objects/`). */ + filePath: string; + recognized: RecognizedMedia; + /** Content hash when already known (always set for derivatives; set on + * the source only if a policy run had to hash it). */ + sha256?: string; + /** Disclosure that must accompany the resource (lossy derivatives). */ + disclosure?: string; + /** True when the resource is a lossy derivative of the user's input. */ + degraded?: boolean; +} + +/** Size ceiling for non-media (`kind: 'file'`) policy artifacts — the + * "bounded" in upstream P's transcript protocol. Text this size is far + * beyond any real transcript; anything bigger is a runaway tool. */ +export const MAX_FILE_ARTIFACT_BYTES = 256 * 1024; + +/** + * One included non-media file artifact (upstream P §6.2 transcript + * protocol): NOT a media resource — it skipped media recognition, was + * validated as bounded UTF-8 text instead, and is delivered as a text + * Part rather than uploaded. `text` carries the full content, read within + * {@link MAX_FILE_ARTIFACT_BYTES} at validation time, so delivery never + * re-reads the file. + */ +export interface PolicyFileDelivery { + /** Promoted object path (registration/record; content is in `text`). */ + filePath: string; + /** `metadata.omniRole` of the artifact (e.g. 'transcript'). */ + role?: string; + mimeType: string; + text: string; + sha256: string; + sizeBytes: number; + /** Disclosure that must accompany the text (lossy derivations). */ + disclosure?: string; +} + +/** Debug/telemetry record of one policy decision that did real work (or + * failed to). Pure matching misses are deliberately unrecorded — with the + * system defaults active on every delivery, zero-policy runs must stay + * zero-noise. */ +export interface PolicyRunRecord { + policyId: string; + toolName: string; + outcome: + | 'succeeded' + | 'cache_hit' + | 'no_op' + | 'failed' + | 'condition_unavailable' + | 'budget_exhausted'; + /** Display label of the resource the policy ran against. */ + resource: string; + /** Fields that made a `when` condition undecidable. */ + missingFields?: string[]; + error?: string; +} + +export interface RunFixedPoliciesOptions { + store: OmniObjectStore; + policies: NormalizedFixedPolicy[]; + signal?: AbortSignal; + /** Request/session condition namespaces from the caller. The resource + * namespace is always derived from each item's recognition, and + * `request.totalEstimatedMediaTokens` is computed internally from the + * pending delivery set unless the caller supplies its own `request` + * (a future multi-root caller that knows the full request set). */ + conditionContext?: Pick; + /** Injectable for tests; defaults to the store-rooted cache. */ + degradationCache?: OmniDegradationCache; + /** Per-root derivation budgets (decision D11); the system defaults + * apply when the caller has no normalized processing config. */ + limits?: NormalizedOmniProcessingLimits; +} + +/** Root resource entering the orchestrator. */ +export interface PolicySourceResource { + filePath: string; + recognized: RecognizedMedia; + /** User-recognizable name for records and error messages. */ + displayName: string; + origin: Extract; +} + +/** Thrown when a policy invocation fails and the failure must abort the + * delivery (`onFailure: 'abort'`, or any transport-guard policy). */ +export class OmniPolicyExecutionError extends Error { + constructor( + message: string, + readonly policyId: string, + options?: { cause?: unknown }, + ) { + super(message, options); + this.name = 'OmniPolicyExecutionError'; + } +} + +/** Work-queue item: a resource that may still be matched by policies. */ +interface WorkItem { + filePath: string; + recognized: RecognizedMedia; + label: string; + origin: FixedPolicyOrigin; + sha256?: string; + disclosure?: string; + degraded?: boolean; + /** Per-derivation-chain run counts (policy id → runs). Copied — never + * shared — on derivation, so sibling branches cap independently. */ + lineageRuns: Map; + /** Derivation-chain length from the root (root = 0). */ + depth: number; + deliver: boolean; + /** Whether the item enters policy matching (`output.reprocessMedia`). */ + process: boolean; +} + +/** Result of one actual policy execution. */ +interface PolicyExecution { + outcome: 'succeeded' | 'cache_hit' | 'no_op'; + derived: Array<{ + filePath: string; + recognized: RecognizedMedia; + sha256: string; + disclosure?: string; + degraded: boolean; + /** `metadata.omniRole`, when the tool labeled the artifact. */ + role?: string; + }>; + /** Non-media file artifacts (transcripts). Never re-enter matching. */ + derivedFiles: PolicyFileDelivery[]; +} + +/** + * Delivery decision for one artifact under the policy's + * `output.artifacts` selector map: most-specific selector wins + * (`role:` > `kind:` > `*`); an artifact nothing matches is retained + * (upstream P default). + */ +function resolveArtifactSelector( + artifacts: Record, + kind: string, + role: string | undefined, +): 'include' | 'retain' { + if (role !== undefined) { + const byRole = artifacts[`role:${role}`]; + if (byRole) return byRole; + } + const byKind = artifacts[`kind:${kind}`]; + if (byKind) return byKind; + return artifacts['*'] ?? 'retain'; +} + +function resourceConditionContext( + recognized: RecognizedMedia, + shared?: Pick, +): FixedPolicyConditionContext { + const resource: Partial> = {}; + const set = ( + field: ResourceConditionField, + value: number | undefined, + ): void => { + if (typeof value === 'number' && !Number.isNaN(value)) { + resource[field] = value; + } + }; + const m = recognized.metadata; + set('sizeBytes', recognized.sizeBytes); + set('durationMs', m.durationMs); + set('width', m.width); + set('height', m.height); + // The probe reports the (single) primary stream's dimensions, so the + // max* aliases resolve to the same values. + set('maxWidth', m.width); + set('maxHeight', m.height); + set('frameRate', m.frameRate); + set('frameCount', m.frameCount); + set('bitRate', m.bitRate); + set('sampleRateHz', m.sampleRateHz); + set('channels', m.channels); + const estimate = estimateRawResourceTokens(recognized); + if (estimate.status === 'ok') { + set('estimatedTokenCount', estimate.estimatedTokenCount); + } + return { resource, ...shared }; +} + +/** + * `request.totalEstimatedMediaTokens` (policy design §8.3): the estimated + * token sum over ALL media currently pending delivery for this root. + * Computed when an item enters matching (pass start) — and therefore + * recomputed as derivatives join or replace the delivery set. Undefined + * (→ `unavailable`) when any pending resource cannot be estimated: a + * partial sum silently reading as a smaller total would flip threshold + * conditions the permissive way. + */ +function requestConditionNamespace( + items: WorkItem[], +): Partial> | undefined { + let total = 0; + for (const item of items) { + if (!item.deliver) continue; + const estimate = estimateRawResourceTokens(item.recognized); + if (estimate.status !== 'ok') return undefined; + total += estimate.estimatedTokenCount; + } + return { totalEstimatedMediaTokens: total }; +} + +/** Deterministic execution order: priority descending, id ascending. */ +function sortPolicies( + policies: NormalizedFixedPolicy[], +): NormalizedFixedPolicy[] { + return [...policies].sort( + (a, b) => b.priority - a.priority || a.id.localeCompare(b.id), + ); +} + +/** Per-omni-root counting semaphore bounding how many resources are + * inside fixed-policy processing at once (`maxConcurrentResources`, + * decision D11). Keyed by omni root so distinct stores in one process + * (multi-project setups, tests) never throttle each other. Callers of + * one root share a config, so the per-call limit is stable per key. */ +const resourceGates = new Map< + string, + { active: number; waiters: Array<() => void> } +>(); + +/** + * Take one processing slot for `rootDir`, waiting FIFO when `limit` are + * already taken. Returns an idempotent release function. On release the + * slot transfers directly to the next waiter (no decrement/re-increment + * gap another caller could slip through), so at most `limit` holders + * ever run concurrently. + */ +async function acquireResourceSlot( + rootDir: string, + limit: number, +): Promise<() => void> { + const effectiveLimit = Math.max(1, Math.floor(limit)); + let gate = resourceGates.get(rootDir); + if (!gate) { + gate = { active: 0, waiters: [] }; + resourceGates.set(rootDir, gate); + } + const heldGate = gate; + if (heldGate.active >= effectiveLimit) { + await new Promise((resolve) => heldGate.waiters.push(resolve)); + // Slot transferred by the releaser — `active` already counts us. + } else { + heldGate.active++; + } + let released = false; + return () => { + if (released) return; + released = true; + const next = heldGate.waiters.shift(); + if (next) { + next(); + return; + } + heldGate.active--; + if (heldGate.active === 0 && heldGate.waiters.length === 0) { + resourceGates.delete(rootDir); + } + }; +} + +/** + * Run the fixed-policy pipeline over one recognized media resource + * (decisions D1/D3/D5): match each policy in priority order, execute the + * matched media-policy tool through the ordinary scheduler path inside an + * exclusive staging directory, validate the artifacts against the tool's + * descriptor, promote them into the content-addressed store, and return + * the final delivery set plus records of the work performed. + * + * Termination is structural AND budgeted: each policy runs at most + * `maxRunsPerLineage` times per derivation chain and the policy set is + * finite, so the derived tree is finite; on top of that the per-root + * budgets (decision D11 — `maxPolicyRunsPerRoot`, `maxArtifactsPerRoot`, + * `maxDerivedBytesPerRoot`, `maxLineageDepth`) stop further derivation + * when exceeded. A budget stop is not a failure: already-committed + * delivery decisions stand (no rollback), the stop is recorded with the + * exhausted budget as its reason, and the transport guard still judges + * the final set. + * + * Failure semantics (decision D10): a failed invocation never leaves + * partial state in staging/ — its staging directory is moved to + * quarantine/ with a `reason.json` for postmortem (Stage B; sweeps apply + * retention). `onFailure: 'continue'` keeps the source in the delivery + * set (the transport guard remains the backstop), while `'abort'` — and + * any transport-guard-stage failure — throws + * {@link OmniPolicyExecutionError}. + */ +export async function runFixedPolicies( + config: Config, + source: PolicySourceResource, + options: RunFixedPoliciesOptions, +): Promise<{ + deliveries: PolicyDeliveryResource[]; + /** Included non-media file artifacts (transcripts), delivered as text + * Parts by the caller — never uploaded. */ + fileDeliveries: PolicyFileDelivery[]; + records: PolicyRunRecord[]; +}> { + const policies = sortPolicies(options.policies); + const limits = options.limits ?? DEFAULT_OMNI_PROCESSING_LIMITS; + // `maxConcurrentResources` (decision D11): each call processes one + // root, so bounding concurrent calls per omni root bounds simultaneous + // transcode work (ffmpeg/sharp processes, staging disk churn). + const releaseSlot = await acquireResourceSlot( + options.store.getOmniRootDir(), + limits.maxConcurrentResources, + ); + try { + return await runFixedPoliciesUnbounded(config, source, options, { + policies, + limits, + }); + } finally { + releaseSlot(); + } +} + +/** Body of {@link runFixedPolicies}, after the per-root concurrency slot + * has been taken. */ +async function runFixedPoliciesUnbounded( + config: Config, + source: PolicySourceResource, + options: RunFixedPoliciesOptions, + normalized: { + policies: NormalizedFixedPolicy[]; + limits: NormalizedOmniProcessingLimits; + }, +): Promise<{ + deliveries: PolicyDeliveryResource[]; + fileDeliveries: PolicyFileDelivery[]; + records: PolicyRunRecord[]; +}> { + const { policies, limits } = normalized; + const cache = + options.degradationCache ?? + new OmniDegradationCache(options.store.getOmniRootDir()); + const records: PolicyRunRecord[] = []; + const fileDeliveries: PolicyFileDelivery[] = []; + const items: WorkItem[] = [ + { + filePath: source.filePath, + recognized: source.recognized, + label: source.displayName, + origin: source.origin, + lineageRuns: new Map(), + depth: 0, + deliver: true, + process: true, + }, + ]; + + // Per-root budget counters (decision D11). One runFixedPolicies call + // processes exactly one root, so the counters live here. + let runsUsed = 0; + let artifactsProduced = 0; + let derivedBytesProduced = 0; + let budgetExhausted = false; + const stopOnBudget = ( + policy: NormalizedFixedPolicy, + item: WorkItem, + reason: string, + ): void => { + budgetExhausted = true; + records.push({ + policyId: policy.id, + toolName: policy.toolName, + outcome: 'budget_exhausted', + resource: item.label, + error: reason, + }); + debugLogger.debug( + `per-root policy budget exhausted on ${item.label}: ${reason}; ` + + `no further derivation for this root (committed deliveries stand)`, + ); + }; + + // Index-based: executions append derived items behind the cursor. + for (let i = 0; i < items.length && !budgetExhausted; i++) { + const item = items[i]; + if (!item.process) continue; + // D9: animated images (frameCount > 1) never enter image-policy + // matching — sharp multi-frame re-encoding is out of scope, and a + // silent single-frame flattening must be impossible. An over-limit + // animated image is handled by the transport guard's explicit + // fail-closed omission instead. Still images are unaffected: probes + // report no frameCount for them, which reads as a single frame. + if ( + item.recognized.modality === 'image' && + (item.recognized.metadata.frameCount ?? 1) > 1 + ) { + debugLogger.debug( + `animated image ${item.label} ` + + `(${item.recognized.metadata.frameCount} frames) excluded from ` + + `policy matching (D9)`, + ); + continue; + } + // Pass-start condition snapshot (policy design §8.3): the request + // namespace is recomputed as each item enters matching so it reflects + // derivatives added by earlier passes; the session namespace is the + // caller's per-delivery snapshot and never changes mid-run. + const sharedConditionContext: Pick< + FixedPolicyConditionContext, + 'request' | 'session' + > = { + request: + options.conditionContext?.request ?? requestConditionNamespace(items), + session: options.conditionContext?.session, + }; + for (const policy of policies) { + if (!policy.mediaTypes.includes(item.recognized.modality)) continue; + if (!policy.origins.includes(item.origin)) continue; + const runs = item.lineageRuns.get(policy.id) ?? 0; + if (runs >= policy.maxRunsPerLineage) continue; + if (policy.when) { + const evaluation = evaluateFixedPolicyCondition( + policy.when, + resourceConditionContext(item.recognized, sharedConditionContext), + ); + if (evaluation.outcome === 'no_match') continue; + if ( + evaluation.outcome === 'unavailable' && + policy.onConditionUnavailable === 'skip' + ) { + records.push({ + policyId: policy.id, + toolName: policy.toolName, + outcome: 'condition_unavailable', + resource: item.label, + missingFields: evaluation.missingFields, + }); + continue; + } + // 'unavailable' + onConditionUnavailable 'run' falls through. + } + if (runsUsed >= limits.maxPolicyRunsPerRoot) { + stopOnBudget( + policy, + item, + `maxPolicyRunsPerRoot (${limits.maxPolicyRunsPerRoot}) reached`, + ); + break; + } + runsUsed++; + item.lineageRuns.set(policy.id, runs + 1); + try { + const execution = await executePolicy( + config, + item, + policy, + options.store, + cache, + options.signal, + ); + records.push({ + policyId: policy.id, + toolName: policy.toolName, + outcome: execution.outcome, + resource: item.label, + }); + if (execution.outcome === 'no_op') continue; + if (policy.output.source === 'omit') item.deliver = false; + const childDepth = item.depth + 1; + const depthAllowsReprocess = childDepth < limits.maxLineageDepth; + if (policy.output.reprocessMedia && !depthAllowsReprocess) { + debugLogger.debug( + `maxLineageDepth (${limits.maxLineageDepth}) reached under ${item.label}; ` + + `derivatives deliver but do not re-enter policy matching`, + ); + } + for (const derived of execution.derived) { + artifactsProduced++; + derivedBytesProduced += derived.recognized.sizeBytes; + items.push({ + ...derived, + label: `${item.label} → ${policy.id}`, + origin: 'policy', + lineageRuns: new Map(item.lineageRuns), + depth: childDepth, + // `output.artifacts` selector decides delivery (upstream P): + // an included derivative enters the delivery set, a retained + // one is only registered in objects/. + deliver: + resolveArtifactSelector( + policy.output.artifacts, + derived.recognized.modality, + derived.role, + ) === 'include', + process: policy.output.reprocessMedia && depthAllowsReprocess, + }); + } + for (const file of execution.derivedFiles) { + // File artifacts (transcripts) count against the same budgets + // but never re-enter policy matching — they are not media. + artifactsProduced++; + derivedBytesProduced += file.sizeBytes; + if ( + resolveArtifactSelector( + policy.output.artifacts, + 'file', + file.role, + ) === 'include' + ) { + fileDeliveries.push(file); + } + } + if (artifactsProduced > limits.maxArtifactsPerRoot) { + stopOnBudget( + policy, + item, + `maxArtifactsPerRoot (${limits.maxArtifactsPerRoot}) exceeded`, + ); + break; + } + if (derivedBytesProduced > limits.maxDerivedBytesPerRoot) { + stopOnBudget( + policy, + item, + `maxDerivedBytesPerRoot (${limits.maxDerivedBytesPerRoot}) exceeded`, + ); + break; + } + } catch (err) { + if (options.signal?.aborted) throw err; + const message = err instanceof Error ? err.message : String(err); + records.push({ + policyId: policy.id, + toolName: policy.toolName, + outcome: 'failed', + resource: item.label, + error: message, + }); + debugLogger.debug( + `fixed policy ${policy.id} (${policy.toolName}) failed on ${item.label}: ${message}`, + ); + if ( + policy.onFailure === 'abort' || + policy.stage === 'transport_guard' + ) { + throw new OmniPolicyExecutionError( + `Fixed policy ${policy.id} failed: ${message}`, + policy.id, + { cause: err }, + ); + } + // 'continue': the source stays in the delivery set; the transport + // guard remains the backstop for oversized content. + } + } + } + + return { + deliveries: items + .filter((item) => item.deliver) + .map((item) => ({ + filePath: item.filePath, + recognized: item.recognized, + sha256: item.sha256, + disclosure: item.disclosure, + degraded: item.degraded, + })), + fileDeliveries, + records, + }; +} + +/** Validated view of one media artifact after descriptor/staging checks. */ +interface ValidatedMediaArtifact { + kind: 'media'; + absolutePath: string; + recognized: RecognizedMedia; + sha256: string; + disclosure?: string; + lossy: boolean; + /** `metadata.omniRole`, when the tool labeled the artifact. */ + role?: string; +} + +/** Validated view of one non-media file artifact (transcript protocol, + * upstream P §6.2): bounded UTF-8 text, never probed as media. */ +interface ValidatedFileArtifact { + kind: 'file'; + absolutePath: string; + mimeType: string; + text: string; + sizeBytes: number; + sha256: string; + disclosure?: string; + lossy: boolean; + role?: string; +} + +type ValidatedArtifact = ValidatedMediaArtifact | ValidatedFileArtifact; + +/** + * Execute one policy against one work item: degradation-cache lookup, + * otherwise a real tool invocation in a fresh staging directory followed + * by artifact validation and promotion (staging lifecycle §5, order D12: + * promote first, then substitute, then delete staging). + */ +async function executePolicy( + config: Config, + item: WorkItem, + policy: NormalizedFixedPolicy, + store: OmniObjectStore, + cache: OmniDegradationCache, + signal: AbortSignal | undefined, +): Promise { + const tool = config.getToolRegistry().getTool(policy.toolName); + const descriptor = tool?.mediaPolicyDescriptor; + if (!descriptor) { + throw new Error( + `tool ${policy.toolName} is not a registered media-policy tool`, + ); + } + + // The source hash keys the degradation cache; computed lazily so runs + // without matching policies never pay it. + item.sha256 ??= await hashFileSha256(item.filePath, signal); + // Effective tunables: tool-level defaults from + // `omni.processing.policyTools..settings` (validated against the + // descriptor's settingsSchema at startup) underneath the policy's own + // arguments. Merged HERE — the single point feeding both the tool call + // and the cache fingerprint — so a settings change also invalidates + // cached derivatives produced under the old values. + const settingsDefaults = resolvePolicyToolSettings(config, policy.toolName); + const effectiveArguments = { ...settingsDefaults, ...policy.arguments }; + const fingerprint = computePolicyFingerprint( + policy.toolName, + effectiveArguments, + descriptor.version, + ); + const hit = await cache.get(item.sha256, fingerprint); + if (hit) { + try { + const objectPath = store.objectPathFor(hit.degradedSha256, hit.extension); + const stat = await fs.lstat(objectPath).catch(() => undefined); + if (stat?.isFile() && !stat.isSymbolicLink()) { + // Content verification before reuse: the cache file lives in the + // workspace and is only shape-validated on load, so the bytes at + // the addressed path must actually hash to the entry's identity — + // otherwise a poisoned cache (or a corrupted store) would silently + // substitute foreign media as "the degraded derivative". + const actualSha256 = await hashFileSha256(objectPath, signal); + if (actualSha256 === hit.degradedSha256) { + const recognized = await recognizeMediaFile(objectPath, { signal }); + // Same cross-check a fresh derivation gets in validateArtifact: + // the recognized bytes must be a media type this tool DECLARES + // producing. The cache file is workspace-shippable, so without + // this a crafted entry could route an arbitrary store object — + // wrong modality included — through a policy that never made + // it, skipping every per-derivation validation gate. + const declared = descriptor.outputs.some( + (o) => + o.kind === 'media' && + o.mimeTypes?.includes(recognized.detectedMimeType), + ); + if (declared) { + debugLogger.debug( + `degradation cache hit: policy=${policy.id} sha256=${item.sha256.slice(0, 12)}…`, + ); + return { + outcome: 'cache_hit', + derived: [ + { + filePath: objectPath, + recognized, + sha256: hit.degradedSha256, + disclosure: hit.disclosure, + degraded: true, + role: hit.role, + }, + ], + derivedFiles: [], + }; + } + debugLogger.debug( + `degradation cache hit for policy=${policy.id} recognized as ` + + `undeclared media type ${recognized.detectedMimeType}; ` + + `dropping the entry and re-transcoding`, + ); + } + } + } catch (err) { + // Verification errors (hash/probe I/O races, a hostile entry whose + // components objectPathFor rejects) must not abort the run: the + // entry is dropped below and the policy re-transcodes from source. + // A caller abort is not a verification failure — propagate it. + if (signal?.aborted) throw err; + debugLogger.debug( + `degradation cache hit could not be verified for policy=${policy.id}: ` + + `${err instanceof Error ? err.message : String(err)}; ` + + `dropping the entry and re-transcoding`, + ); + } + // Stale, mismatching, or unverifiable: the derivative left the store + // (GC, manual deletion) or its bytes no longer match the entry. Drop + // every entry pointing at it and re-transcode. + await cache.removeByDegradedSha256(hit.degradedSha256); + } + + const invocationId = randomBytes(8).toString('hex'); + const stagingDir = await store.createStagingDir(invocationId); + let failure: unknown; + try { + const request: ToolCallRequestInfo = { + callId: invocationId, + name: policy.toolName, + args: { + ...effectiveArguments, + inputPath: item.filePath, + outputDir: stagingDir, + }, + isClientInitiated: true, + prompt_id: `omni-fixed-policy-${invocationId}`, + executionOrigin: { + kind: 'fixed_policy', + policyId: policy.id, + stage: policy.stage, + }, + }; + // Dynamic import: the executor pulls in the scheduler, whose module + // graph reaches back into omni surfaces — the runtime dependency is + // resolved at call time (same pattern as the scheduler's tool-result + // funnel import) to keep module evaluation cycle-free. + const { executeToolCall } = await import( + '../../core/nonInteractiveToolExecutor.js' + ); + const response = await executeToolCall( + config, + request, + signal ?? new AbortController().signal, + { recordToolResult: false }, + ); + if (response.error) { + throw new Error(response.error.message, { cause: response.error }); + } + const batch = response.policyArtifacts; + if (!batch || batch.artifacts.length === 0) { + throw new Error( + `tool ${policy.toolName} succeeded but produced no policy artifacts`, + ); + } + + // Artifacts are independent files in the same staging dir — validate + // them concurrently; map keeps the batch order. + const validated = await Promise.all( + batch.artifacts.map((artifact) => + validateArtifact(artifact, descriptor, stagingDir, signal), + ), + ); + assertRequiredOutputsPresent(descriptor, validated, policy.toolName); + + // Fixed-point: identical output means this iteration changed nothing — + // deliver the source and stop deriving (no cache entry either; a no-op + // is a property of this input, re-derivable cheaply). + if (validated.every((a) => a.sha256 === item.sha256)) { + return { outcome: 'no_op', derived: [], derivedFiles: [] }; + } + + // Promotion first (D12): once an artifact is in objects/ it is + // content-addressed and immutable; only then substitute + cache. + // Independent files promote concurrently (the store is + // content-addressed: tmp + atomic rename); map keeps the batch order. + const promoted = await Promise.all( + validated.map(async (artifact) => { + const mimeType = + artifact.kind === 'media' + ? artifact.recognized.detectedMimeType + : artifact.mimeType; + const put = await store.putFile( + artifact.absolutePath, + artifact.sha256, + extensionForMime(mimeType), + signal, + ); + return { artifact, objectPath: put.objectPath }; + }), + ); + const derived: PolicyExecution['derived'] = []; + const derivedFiles: PolicyExecution['derivedFiles'] = []; + for (const { artifact, objectPath } of promoted) { + if (artifact.kind === 'media') { + derived.push({ + filePath: objectPath, + recognized: artifact.recognized, + sha256: artifact.sha256, + disclosure: artifact.disclosure, + degraded: artifact.lossy, + role: artifact.role, + }); + } else { + derivedFiles.push({ + filePath: objectPath, + role: artifact.role, + mimeType: artifact.mimeType, + text: artifact.text, + sha256: artifact.sha256, + sizeBytes: artifact.sizeBytes, + disclosure: artifact.disclosure, + }); + } + } + // The cache maps one input to ONE media derivative; multi-output tools + // and file artifacts (whose cache-hit path depends on media + // re-recognition) are simply not cached — re-run instead of guessing. + if ( + validated.length === 1 && + validated[0].kind === 'media' && + validated[0].disclosure + ) { + await cache.put(item.sha256, fingerprint, { + degradedSha256: validated[0].sha256, + extension: extensionForMime(validated[0].recognized.detectedMimeType), + disclosure: validated[0].disclosure, + mimeType: validated[0].recognized.detectedMimeType, + // Persist the artifact's role so a cache hit reconstructs the SAME + // derived shape as the fresh derivation above. + ...(validated[0].role !== undefined ? { role: validated[0].role } : {}), + }); + } + return { outcome: 'succeeded', derived, derivedFiles }; + } catch (err) { + failure = err; + throw err; + } finally { + if (failure === undefined || signal?.aborted) { + // Success, no_op, and user aborts end without a staging dir — there + // is nothing to diagnose. + await store.removeStagingDir(invocationId).catch(() => {}); + } else { + // Failure (decision D10 Stage B): move the staging dir — partial + // outputs included — into quarantine/ with a reason.json for + // postmortem; the startup sweeps apply retention/size budgets. If + // quarantining itself fails, fall back to plain removal so a failed + // invocation still never leaves live staging state behind. + try { + await store.quarantineInvocation(invocationId, { + policyId: policy.id, + toolName: policy.toolName, + reason: failure instanceof Error ? failure.message : String(failure), + }); + } catch { + await store.removeStagingDir(invocationId).catch(() => {}); + } + } + } +} + +/** + * Validate one artifact against the staging contract (§5) and the tool's + * descriptor (D8): workspace-storage with a path strictly inside the + * staging dir, a regular non-symlink file, recognized content matching a + * declared media output — or, for `kind: 'file'` artifacts (transcript + * protocol), bounded strict-UTF-8 text matching a declared file output — + * and, for lossy outputs, a non-empty `metadata.omniDisclosure`. + */ +async function validateArtifact( + artifact: ToolArtifact, + descriptor: MediaPolicyToolDescriptor, + stagingDir: string, + signal: AbortSignal | undefined, +): Promise { + if (artifact.storage !== 'workspace' || !artifact.workspacePath) { + throw new Error( + `policy artifact "${artifact.title}" is not a workspace file`, + ); + } + const absolutePath = path.resolve(stagingDir, artifact.workspacePath); + if (!absolutePath.startsWith(stagingDir + path.sep)) { + throw new Error( + `policy artifact "${artifact.title}" escapes the staging directory`, + ); + } + const stat = await fs.lstat(absolutePath).catch(() => undefined); + if (!stat?.isFile() || stat.isSymbolicLink()) { + throw new Error( + `policy artifact "${artifact.title}" is missing or not a regular file`, + ); + } + const rawRole = artifact.metadata?.['omniRole']; + const role = typeof rawRole === 'string' && rawRole ? rawRole : undefined; + const rawDisclosure = artifact.metadata?.['omniDisclosure']; + const disclosure = + typeof rawDisclosure === 'string' && rawDisclosure + ? rawDisclosure + : undefined; + + if (artifact.kind === 'file') { + // Non-media artifact (upstream P §6.2): validated as bounded strict + // UTF-8 text against a declared `kind: 'file'` output — no media + // probe. Only tools whose descriptor declares a file output can pass + // here, and the declared mimeType must be one the spec allows. + const spec = descriptor.outputs.find( + (o) => + o.kind === 'file' && + (o.role === undefined || o.role === role) && + artifact.mimeType !== undefined && + o.mimeTypes?.includes(artifact.mimeType), + ); + if (!spec) { + throw new Error( + `policy artifact "${artifact.title}" (file, role ${role ?? 'none'}, ` + + `${artifact.mimeType ?? 'no mimeType'}) matches no declared file output`, + ); + } + if (stat.size > MAX_FILE_ARTIFACT_BYTES) { + throw new Error( + `policy artifact "${artifact.title}" exceeds the file-artifact ` + + `size budget (${stat.size} > ${MAX_FILE_ARTIFACT_BYTES} bytes)`, + ); + } + const bytes = await fs.readFile(absolutePath); + let text: string; + try { + text = new TextDecoder('utf-8', { fatal: true }).decode(bytes); + } catch { + throw new Error( + `policy artifact "${artifact.title}" is not valid UTF-8 text`, + ); + } + if (spec.lossy && !disclosure) { + throw new Error( + `policy artifact "${artifact.title}" is lossy but carries no omniDisclosure`, + ); + } + return { + kind: 'file', + absolutePath, + mimeType: artifact.mimeType as string, + text, + sizeBytes: bytes.byteLength, + // The bytes are already in memory for the UTF-8 check — hash them + // directly instead of streaming the file a second time. + sha256: createHash('sha256').update(bytes).digest('hex'), + disclosure, + lossy: spec.lossy === true, + role, + }; + } + + // Authoritative recognition of the actual bytes — the tool's declared + // mimeType/kind are cross-checked, never trusted. + const recognized = await recognizeMediaFile(absolutePath, { signal }); + const spec = descriptor.outputs.find( + (o) => + o.kind === 'media' && o.mimeTypes?.includes(recognized.detectedMimeType), + ); + if (!spec) { + throw new Error( + `policy artifact "${artifact.title}" has undeclared media type ${recognized.detectedMimeType}`, + ); + } + if (artifact.kind !== recognized.modality) { + throw new Error( + `policy artifact "${artifact.title}" declares kind ${String(artifact.kind)} but contains ${recognized.modality} content`, + ); + } + if (spec.lossy && !disclosure) { + throw new Error( + `policy artifact "${artifact.title}" is lossy but carries no omniDisclosure`, + ); + } + return { + kind: 'media', + absolutePath, + recognized, + sha256: await hashFileSha256(absolutePath, signal), + disclosure, + lossy: spec.lossy === true, + role, + }; +} + +/** Every required media/file output declared by the descriptor must have + * been produced (§5 completeness check). */ +function assertRequiredOutputsPresent( + descriptor: MediaPolicyToolDescriptor, + validated: ValidatedArtifact[], + toolName: string, +): void { + for (const spec of descriptor.outputs) { + if (!spec.required) continue; + let produced: boolean; + if (spec.kind === 'media') { + produced = validated.some( + (a) => + a.kind === 'media' && + spec.mimeTypes?.includes(a.recognized.detectedMimeType), + ); + } else if (spec.kind === 'file') { + produced = validated.some( + (a) => + a.kind === 'file' && + (spec.role === undefined || a.role === spec.role) && + spec.mimeTypes?.includes(a.mimeType), + ); + } else { + // Text outputs (disclosures) travel as artifact metadata, not as + // artifacts of their own — validated per lossy artifact above. + continue; + } + if (!produced) { + throw new Error( + `tool ${toolName} did not produce its required ${spec.kind} ` + + `${spec.mimeTypes?.join('/') ?? spec.role ?? ''} output`, + ); + } + } +} diff --git a/packages/core/src/omni/policy/session-context.test.ts b/packages/core/src/omni/policy/session-context.test.ts new file mode 100644 index 00000000000..64844bc44e1 --- /dev/null +++ b/packages/core/src/omni/policy/session-context.test.ts @@ -0,0 +1,97 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { describe, expect, it } from 'vitest'; +import { + buildSessionConditionNamespace, + type SessionConditionConfigView, +} from './session-context.js'; + +const fullConfig = ( + contextWindowSize: number | undefined, + lastPromptTokenCount: number, +): SessionConditionConfigView => ({ + getContentGeneratorConfig: () => ({ contextWindowSize }), + getGeminiClient: () => ({ + getChat: () => ({ getLastPromptTokenCount: () => lastPromptTokenCount }), + }), +}); + +describe('buildSessionConditionNamespace (§8.3 session.*)', () => { + it('snapshots all four fields with the exact subtraction', () => { + expect( + buildSessionConditionNamespace(fullConfig(131072, 20000), 8192), + ).toEqual({ + reservedOutputTokens: 8192, + contextWindowTokens: 131072, + promptTokenCount: 20000, + availableContextTokens: 131072 - 20000 - 8192, + }); + }); + + it('clamps availableContextTokens at 0 when the window is exhausted', () => { + expect(buildSessionConditionNamespace(fullConfig(1000, 900), 200)).toEqual({ + reservedOutputTokens: 200, + contextWindowTokens: 1000, + promptTokenCount: 900, + availableContextTokens: 0, + }); + }); + + it('treats a fresh chat (0 prompt tokens) as a real value, not absence', () => { + expect(buildSessionConditionNamespace(fullConfig(1000, 0), 200)).toEqual({ + reservedOutputTokens: 200, + contextWindowTokens: 1000, + promptTokenCount: 0, + availableContextTokens: 800, + }); + }); + + it.each([ + ['undefined', undefined], + ['zero', 0], + ['negative', -5], + ['non-finite', Number.POSITIVE_INFINITY], + ])( + 'omits contextWindowTokens AND availableContextTokens for a %s window', + (_name, windowSize) => { + expect( + buildSessionConditionNamespace(fullConfig(windowSize, 20000), 8192), + ).toEqual({ reservedOutputTokens: 8192, promptTokenCount: 20000 }); + }, + ); + + it('omits promptTokenCount AND availableContextTokens when getChat throws', () => { + const config: SessionConditionConfigView = { + getContentGeneratorConfig: () => ({ contextWindowSize: 131072 }), + getGeminiClient: () => ({ + getChat: () => { + throw new Error('Chat not initialized'); + }, + }), + }; + expect(buildSessionConditionNamespace(config, 8192)).toEqual({ + reservedOutputTokens: 8192, + contextWindowTokens: 131072, + }); + }); + + it('omits promptTokenCount for a negative or non-finite chat count', () => { + expect(buildSessionConditionNamespace(fullConfig(1000, -1), 200)).toEqual({ + reservedOutputTokens: 200, + contextWindowTokens: 1000, + }); + expect( + buildSessionConditionNamespace(fullConfig(1000, Number.NaN), 200), + ).toEqual({ reservedOutputTokens: 200, contextWindowTokens: 1000 }); + }); + + it('yields only reservedOutputTokens on a bare stub config', () => { + expect(buildSessionConditionNamespace({}, 8192)).toEqual({ + reservedOutputTokens: 8192, + }); + }); +}); diff --git a/packages/core/src/omni/policy/session-context.ts b/packages/core/src/omni/policy/session-context.ts new file mode 100644 index 00000000000..6c4940c0f43 --- /dev/null +++ b/packages/core/src/omni/policy/session-context.ts @@ -0,0 +1,86 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import type { SessionConditionField } from './conditions.js'; + +/** + * Structural view over Config used to snapshot the `session.*` condition + * namespace. Every member is optional so partial/stub configs (tests, + * embedders that skip initialization) simply yield fewer fields — which + * the condition evaluator reads as `unavailable`, never as zero. + */ +export interface SessionConditionConfigView { + getContentGeneratorConfig?: () => { contextWindowSize?: number } | undefined; + /** May throw (GeminiClient.getChat throws before initialization). */ + getGeminiClient?: () => + | { getChat?: () => { getLastPromptTokenCount(): number } } + | undefined; +} + +/** + * Snapshot the `session.*` condition namespace (policy design §8.3) + * BEFORE fixed-policy execution. The snapshot is taken once per media + * delivery and stays constant across every pass of that delivery + * (preprocessing and transport-guard alike). + * + * - `contextWindowTokens`: the active model's context window, read from + * `contentGeneratorConfig.contextWindowSize` — the post-initialization + * authority for the resolved window size. + * - `promptTokenCount`: the CURRENT chat's last reported prompt token + * count (`GeminiChat.getLastPromptTokenCount()`), per-chat by design — + * a subagent's media must be judged against the subagent's own context, + * never a global UI telemetry counter. + * - `reservedOutputTokens`: `omni.processing.limits.reservedOutputTokens`. + * - `availableContextTokens`: + * `max(0, contextWindowTokens − promptTokenCount − reservedOutputTokens)`, + * computed only when both inputs are known — a partial subtraction must + * surface as `unavailable`, never as a permissive large number. + */ +export function buildSessionConditionNamespace( + config: SessionConditionConfigView, + reservedOutputTokens: number, +): Partial> { + const session: Partial> = { + reservedOutputTokens, + }; + + const windowSize = config.getContentGeneratorConfig?.()?.contextWindowSize; + const contextWindowTokens = + typeof windowSize === 'number' && + Number.isFinite(windowSize) && + windowSize > 0 + ? windowSize + : undefined; + if (contextWindowTokens !== undefined) { + session.contextWindowTokens = contextWindowTokens; + } + + let promptTokenCount: number | undefined; + try { + const count = config + .getGeminiClient?.() + ?.getChat?.() + ?.getLastPromptTokenCount(); + // 0 is legitimate (nothing sent yet on this chat); negative or + // non-finite values are stub garbage and read as absent. + if (typeof count === 'number' && Number.isFinite(count) && count >= 0) { + promptTokenCount = count; + } + } catch { + // Chat not initialized: the field stays absent (→ unavailable). + } + if (promptTokenCount !== undefined) { + session.promptTokenCount = promptTokenCount; + } + + if (contextWindowTokens !== undefined && promptTokenCount !== undefined) { + session.availableContextTokens = Math.max( + 0, + contextWindowTokens - promptTokenCount - reservedOutputTokens, + ); + } + return session; +} diff --git a/packages/core/src/omni/policy/tools/clip-video.test.ts b/packages/core/src/omni/policy/tools/clip-video.test.ts new file mode 100644 index 00000000000..1f024d9f22e --- /dev/null +++ b/packages/core/src/omni/policy/tools/clip-video.test.ts @@ -0,0 +1,251 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import type { MediaProbeResult } from '../../ffmpeg.js'; +import type { ToolResult } from '../../../tools/tools.js'; +import { DEFAULT_POLICY_TOOL_TIMEOUT_MS } from './media-policy-tool.js'; +import { + CLIP_VIDEO_DEFAULTS, + OMNI_CLIP_VIDEO_TOOL_NAME, + OmniClipVideoTool, +} from './clip-video.js'; + +const mocks = vi.hoisted(() => ({ + probeMediaMetadata: vi.fn(), + runFfmpeg: vi.fn(), +})); + +vi.mock('../../ffmpeg.js', () => ({ + probeMediaMetadata: mocks.probeMediaMetadata, + runFfmpeg: mocks.runFfmpeg, +})); + +const OUTPUT_SIZE = 900 * 1024; + +describe('OmniClipVideoTool', () => { + let root: string; + let inputPath: string; + let outputDir: string; + + const tool = new OmniClipVideoTool({}); + + const probe = (result: Partial): void => { + mocks.probeMediaMetadata.mockResolvedValue(result as MediaProbeResult); + }; + + const ffmpegSucceeds = (): void => { + mocks.runFfmpeg.mockImplementation(async (args: string[]) => { + await fs.writeFile(args[args.length - 1], Buffer.alloc(OUTPUT_SIZE)); + return { code: 0, stderr: '' }; + }); + }; + + const run = async ( + params: Record, + ): Promise<{ result: ToolResult; signal: AbortSignal }> => { + const invocation = tool.build({ + inputPath, + outputDir, + ...params, + } as never); + const signal = new AbortController().signal; + return { result: await invocation.execute(signal), signal }; + }; + + beforeEach(async () => { + vi.clearAllMocks(); + root = await fs.mkdtemp(path.join(os.tmpdir(), 'omni-cv-')); + inputPath = path.join(root, 'clip.mp4'); + await fs.writeFile(inputPath, Buffer.alloc(1024)); + outputDir = path.join(root, 'staging'); + await fs.mkdir(outputDir); + ffmpegSucceeds(); + }); + + afterEach(async () => { + await fs.rm(root, { recursive: true, force: true }); + }); + + it('declares the media-policy descriptor and defaults', () => { + expect(tool.name).toBe(OMNI_CLIP_VIDEO_TOOL_NAME); + expect(tool.mediaPolicyDescriptor).toEqual({ + kind: 'media_policy', + version: '1', + inputMediaTypes: ['video'], + outputs: [ + { + kind: 'media', + mimeTypes: ['video/mp4'], + required: true, + lossy: true, + }, + { kind: 'text', role: 'disclosure', required: true }, + ], + settingsSchema: expect.objectContaining({ type: 'object' }), + }); + expect(CLIP_VIDEO_DEFAULTS).toEqual({ + crf: 23, + preset: 'veryfast', + audioBitrateKbps: 128, + }); + }); + + it('cuts [startSec, startSec+durationSec] with a frame-accurate re-encode', async () => { + probe({ durationMs: 63_000 }); + const { result, signal } = await run({ startSec: 10, durationSec: 15 }); + + expect(mocks.probeMediaMetadata).toHaveBeenCalledWith( + inputPath, + 'video', + signal, + ); + const outputPath = path.join(outputDir, 'clip.mp4'); + expect(mocks.runFfmpeg).toHaveBeenCalledWith( + [ + '-y', + '-ss', + '10', + '-t', + '15', + '-i', + inputPath, + '-vf', + 'scale=trunc(iw/2)*2:trunc(ih/2)*2', + '-c:v', + 'libx264', + '-crf', + '23', + '-preset', + 'veryfast', + '-c:a', + 'aac', + '-b:a', + '128k', + '-movflags', + '+faststart', + outputPath, + ], + { signal, timeoutMs: DEFAULT_POLICY_TOOL_TIMEOUT_MS }, + ); + + expect(result.error).toBeUndefined(); + expect(result.artifacts).toEqual([ + { + kind: 'video', + storage: 'workspace', + title: 'Clipped video', + workspacePath: 'clip.mp4', + mimeType: 'video/mp4', + sizeBytes: OUTPUT_SIZE, + metadata: { + omniDisclosure: '原 63s → 片段 [10s–25s] 15s，片段外内容全部丢弃', + }, + }, + ]); + }); + + it('clips from startSec to the end when durationSec is absent', async () => { + probe({ durationMs: 63_000 }); + const { result } = await run({ startSec: 10 }); + const args = mocks.runFfmpeg.mock.calls[0][0] as string[]; + expect(args).not.toContain('-t'); + expect(result.artifacts?.[0]?.metadata?.['omniDisclosure']).toBe( + '原 63s → 片段 [10s–63s] 53s，片段外内容全部丢弃', + ); + }); + + it('clamps the disclosed end to the video length', async () => { + probe({ durationMs: 63_000 }); + const { result } = await run({ startSec: 50, durationSec: 100 }); + expect(result.artifacts?.[0]?.metadata?.['omniDisclosure']).toBe( + '原 63s → 片段 [50s–63s] 13s，片段外内容全部丢弃', + ); + }); + + it('discloses an unknown original duration without clamping', async () => { + probe({}); + const { result } = await run({ startSec: 10, durationSec: 15 }); + expect(result.artifacts?.[0]?.metadata?.['omniDisclosure']).toBe( + '原 未知时长 → 片段 [10s–25s] 15s，片段外内容全部丢弃', + ); + }); + + it('rejects a start at or beyond the end of the video without transcoding', async () => { + probe({ durationMs: 63_000 }); + const { result } = await run({ startSec: 63 }); + expect(result.error?.message).toBe( + 'startSec (63s) is at or beyond the end of the video (63s)', + ); + expect(mocks.runFfmpeg).not.toHaveBeenCalled(); + }); + + it('reports the ffmpeg error', async () => { + probe({ durationMs: 63_000 }); + mocks.runFfmpeg.mockResolvedValue({ code: 187, stderr: 'boom' }); + const { result } = await run({ startSec: 10 }); + expect(result.error?.message).toMatch(/ffmpeg failed \(exit 187\)/); + expect(result.error?.message).toContain('boom'); + }); + + it('reports an aborted run', async () => { + probe({ durationMs: 63_000 }); + const controller = new AbortController(); + mocks.runFfmpeg.mockImplementation(async () => { + controller.abort(); + return { code: 0, stderr: '' }; + }); + const invocation = tool.build({ inputPath, outputDir, startSec: 10 }); + const result = await invocation.execute(controller.signal); + expect(result.error?.message).toBe('video clipping aborted'); + }); + + it('threads policyTools..runtime.timeoutMs into runFfmpeg', async () => { + probe({ durationMs: 63_000 }); + const configured = new OmniClipVideoTool({ + getOmniPolicyToolsSettings: () => ({ + [OMNI_CLIP_VIDEO_TOOL_NAME]: { + runtime: { timeoutMs: 120_000 }, + }, + }), + }); + const invocation = configured.build({ + inputPath, + outputDir, + startSec: 10, + }); + await invocation.execute(new AbortController().signal); + expect(mocks.runFfmpeg).toHaveBeenCalledWith( + expect.any(Array), + expect.objectContaining({ timeoutMs: 120_000 }), + ); + }); + + it.each([ + ['both startSec and durationSec absent', {}], + ['explicit startSec 0 without durationSec (no-op clip)', { startSec: 0 }], + ['negative startSec', { startSec: -1 }], + ['zero durationSec', { durationSec: 0 }], + ['relative outputDir', { startSec: 10, outputDir: 'staging' }], + ['unknown property', { startSec: 10, extra: 1 }], + ])('build rejects %s', (_label, overrides) => { + expect(() => + tool.build({ inputPath, outputDir, ...overrides } as never), + ).toThrow(); + }); + + it('rejects a probed full-span request without transcoding', async () => { + // startSec 0 + a duration covering the whole video is a no-op clip: + // nothing outside the span exists to discard, only re-encode damage. + probe({ durationMs: 63_000 }); + const { result } = await run({ startSec: 0, durationSec: 63 }); + expect(result.error?.message).toMatch(/covers the entire video/); + expect(mocks.runFfmpeg).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/core/src/omni/policy/tools/clip-video.ts b/packages/core/src/omni/policy/tools/clip-video.ts new file mode 100644 index 00000000000..214a8382f42 --- /dev/null +++ b/packages/core/src/omni/policy/tools/clip-video.ts @@ -0,0 +1,267 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import fs from 'node:fs/promises'; +import path from 'node:path'; +import type { + MediaPolicyToolDescriptor, + ToolInvocation, + ToolResult, +} from '../../../tools/tools.js'; +import { Kind } from '../../../tools/tools.js'; +import { ToolNames } from '../../../tools/tool-names.js'; +import { probeMediaMetadata, runFfmpeg } from '../../ffmpeg.js'; +import { + assertMediaPolicyIo, + BaseMediaPolicyTool, + ffmpegFailureMessage, + BaseMediaPolicyToolInvocation, + MEDIA_POLICY_IO_SCHEMA_PROPERTIES, + mediaPolicyToolError, + mediaPolicyToolFailure, + mediaPolicyToolSuccess, + resolvePolicyToolTimeoutMs, + type MediaPolicyIoParams, + type MediaPolicyToolConfigView, +} from './media-policy-tool.js'; + +export const OMNI_CLIP_VIDEO_TOOL_NAME = ToolNames.OMNI_CLIP_VIDEO; + +/** Fixed-call default encode parameters (mapping doc §6.1): clip is a + * time-axis cut, NOT a degradation — crf 23 preserves quality; lowering + * resolution/bit rate is omni_downscale_video's job. */ +export const CLIP_VIDEO_DEFAULTS = { + crf: 23, + preset: 'veryfast', + audioBitrateKbps: 128, +} as const; + +const OUTPUT_FILE_NAME = 'clip.mp4'; + +export interface ClipVideoParams extends MediaPolicyIoParams { + /** Clip start in seconds (default 0). */ + startSec?: number; + /** Clip duration in seconds (default: to the end of the video). */ + durationSec?: number; +} + +const TUNABLE_SCHEMA_PROPERTIES = { + startSec: { + type: 'number', + description: 'Clip start position in seconds. Default 0.', + minimum: 0, + }, + durationSec: { + type: 'number', + description: 'Clip duration in seconds. Default: from startSec to the end.', + exclusiveMinimum: 0, + }, +} as const; + +const DESCRIPTOR: MediaPolicyToolDescriptor = { + kind: 'media_policy', + version: '1', + inputMediaTypes: ['video'], + outputs: [ + { + kind: 'media', + mimeTypes: ['video/mp4'], + required: true, + lossy: true, + }, + { kind: 'text', role: 'disclosure', required: true }, + ], + settingsSchema: { + type: 'object', + properties: TUNABLE_SCHEMA_PROPERTIES, + additionalProperties: false, + }, +}; + +/** "12s" / "12.4s" — seconds with at most one decimal. */ +function formatSeconds(seconds: number): string { + return `${Math.round(seconds * 10) / 10}s`; +} + +class ClipVideoInvocation extends BaseMediaPolicyToolInvocation { + constructor( + params: ClipVideoParams, + private readonly timeoutMs: number, + ) { + super(params); + } + + getDescription(): string { + const start = this.params.startSec ?? 0; + const span = + this.params.durationSec !== undefined + ? `${formatSeconds(start)}–${formatSeconds(start + this.params.durationSec)}` + : `${formatSeconds(start)}–end`; + return `Clip ${path.basename(this.params.inputPath)} to [${span}]`; + } + + async execute(signal: AbortSignal): Promise { + const startSec = this.params.startSec ?? 0; + const durationSec = this.params.durationSec; + try { + await assertMediaPolicyIo(this.params); + const probe = await probeMediaMetadata( + this.params.inputPath, + 'video', + signal, + ); + const totalSeconds = + probe.durationMs !== undefined ? probe.durationMs / 1000 : undefined; + if (totalSeconds !== undefined && startSec >= totalSeconds) { + return mediaPolicyToolError( + `startSec (${formatSeconds(startSec)}) is at or beyond the end of the video (${formatSeconds(totalSeconds)})`, + ); + } + // Detectable full-span no-op: the effective window covers the whole + // video, so "clipping" would only run a lossy re-encode and the + // disclosure would falsely claim content outside the span was + // discarded. This is a time-axis cut, NOT a degradation tool. + if ( + totalSeconds !== undefined && + startSec === 0 && + durationSec !== undefined && + durationSec >= totalSeconds + ) { + return mediaPolicyToolError( + `the requested span [0–${formatSeconds(durationSec)}] covers the ` + + `entire video (${formatSeconds(totalSeconds)}) — a no-op clip ` + + `that would only re-encode (and damage) the input`, + ); + } + + const outputPath = path.join(this.params.outputDir, OUTPUT_FILE_NAME); + // Input-side -ss/-t plus a full re-encode: frame-accurate cuts + // regardless of keyframe placement (`-c copy` snaps to keyframes). + // The scale filter only forces even dimensions (libx264 hard + // requirement); resolution is otherwise preserved. + const run = await runFfmpeg( + [ + '-y', + '-ss', + String(startSec), + ...(durationSec !== undefined ? ['-t', String(durationSec)] : []), + '-i', + this.params.inputPath, + '-vf', + 'scale=trunc(iw/2)*2:trunc(ih/2)*2', + '-c:v', + 'libx264', + '-crf', + String(CLIP_VIDEO_DEFAULTS.crf), + '-preset', + CLIP_VIDEO_DEFAULTS.preset, + '-c:a', + 'aac', + '-b:a', + `${CLIP_VIDEO_DEFAULTS.audioBitrateKbps}k`, + '-movflags', + '+faststart', + outputPath, + ], + { signal, timeoutMs: this.timeoutMs }, + ); + if (signal.aborted) { + return mediaPolicyToolError('video clipping aborted'); + } + if (run.code !== 0) { + return mediaPolicyToolError( + ffmpegFailureMessage(run, 'clipping', this.params.inputPath), + ); + } + + const outputSizeBytes = (await fs.stat(outputPath)).size; + const endSec = + durationSec !== undefined + ? totalSeconds !== undefined + ? Math.min(startSec + durationSec, totalSeconds) + : startSec + durationSec + : totalSeconds; + const original = + totalSeconds !== undefined ? formatSeconds(totalSeconds) : '未知时长'; + const endText = endSec !== undefined ? formatSeconds(endSec) : '结尾'; + const spanText = + endSec !== undefined ? ` ${formatSeconds(endSec - startSec)}` : ''; + const disclosure = `原 ${original} → 片段 [${formatSeconds(startSec)}–${endText}]${spanText}，片段外内容全部丢弃`; + + return mediaPolicyToolSuccess({ + outputDir: this.params.outputDir, + outputFileName: OUTPUT_FILE_NAME, + artifactKind: 'video', + title: 'Clipped video', + mimeType: 'video/mp4', + sizeBytes: outputSizeBytes, + disclosure, + }); + } catch (error) { + return mediaPolicyToolFailure(error); + } + } +} + +/** + * `omni_clip_video` — time-axis cut (ffmpeg): input-side seek plus a + * frame-accurate re-encode of the selected span (mapping doc §6.1). + * Everything outside the span is discarded — lossy by definition, with + * the span disclosed. + */ +export class OmniClipVideoTool extends BaseMediaPolicyTool { + constructor(config: MediaPolicyToolConfigView) { + super( + OMNI_CLIP_VIDEO_TOOL_NAME, + 'ClipVideo', + 'Cuts a time span out of a video (frame-accurate re-encode), discarding everything outside the span, with a disclosure of the cut.', + Kind.Other, + { + type: 'object', + properties: { + ...MEDIA_POLICY_IO_SCHEMA_PROPERTIES, + ...TUNABLE_SCHEMA_PROPERTIES, + }, + required: ['inputPath', 'outputDir'], + additionalProperties: false, + }, + config, + ); + } + + override get mediaPolicyDescriptor(): MediaPolicyToolDescriptor { + return DESCRIPTOR; + } + + protected override validateToolParamValues( + params: ClipVideoParams, + ): string | null { + const ioError = super.validateToolParamValues(params); + if (ioError) return ioError; + // A no-op invocation (full-length "clip") must be rejected at the + // parameter layer instead of burning a full lossy re-encode on it: + // both absent, or an explicit startSec of 0 with no duration bound. + if (params.startSec === undefined && params.durationSec === undefined) { + return 'at least one of startSec / durationSec must be provided'; + } + if (params.startSec === 0 && params.durationSec === undefined) { + return ( + 'startSec: 0 without durationSec selects the whole video — a no-op ' + + 'clip that would only re-encode (and damage) the input' + ); + } + return null; + } + + protected createInvocation( + params: ClipVideoParams, + ): ToolInvocation { + return new ClipVideoInvocation( + params, + resolvePolicyToolTimeoutMs(this.configView, this.name), + ); + } +} diff --git a/packages/core/src/omni/policy/tools/convert-image.test.ts b/packages/core/src/omni/policy/tools/convert-image.test.ts new file mode 100644 index 00000000000..413fa9e8342 --- /dev/null +++ b/packages/core/src/omni/policy/tools/convert-image.test.ts @@ -0,0 +1,293 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import type { MediaProbeResult } from '../../ffmpeg.js'; +import type { ToolResult } from '../../../tools/tools.js'; +import { DEFAULT_POLICY_TOOL_TIMEOUT_MS } from './media-policy-tool.js'; +import { + CONVERT_IMAGE_DEFAULTS, + OMNI_CONVERT_IMAGE_TOOL_NAME, + OmniConvertImageTool, +} from './convert-image.js'; + +const mocks = vi.hoisted(() => ({ + probeMediaMetadata: vi.fn(), + runFfmpeg: vi.fn(), + sharpCreate: vi.fn(), +})); + +vi.mock('../../ffmpeg.js', () => ({ + probeMediaMetadata: mocks.probeMediaMetadata, + runFfmpeg: mocks.runFfmpeg, +})); + +vi.mock('sharp', () => ({ + default: (...args: unknown[]) => mocks.sharpCreate(...args), +})); + +const INPUT_SIZE = 2 * 1024 ** 2; // "2MB" +const OUTPUT_SIZE = 300 * 1024; // "300KB" + +describe('OmniConvertImageTool', () => { + let root: string; + let inputPath: string; + let outputDir: string; + let toFile: ReturnType; + let jpeg: ReturnType; + let png: ReturnType; + let webp: ReturnType; + let rotate: ReturnType; + let timeout: ReturnType; + + const tool = new OmniConvertImageTool(); + + const probe = (result: Partial): void => { + mocks.probeMediaMetadata.mockResolvedValue(result as MediaProbeResult); + }; + + const run = async ( + params: Record = {}, + ): Promise<{ result: ToolResult; signal: AbortSignal }> => { + const invocation = tool.build({ + inputPath, + outputDir, + ...params, + } as never); + const signal = new AbortController().signal; + return { result: await invocation.execute(signal), signal }; + }; + + beforeEach(async () => { + vi.clearAllMocks(); + root = await fs.mkdtemp(path.join(os.tmpdir(), 'omni-ci-')); + inputPath = path.join(root, 'photo.png'); + await fs.writeFile(inputPath, Buffer.alloc(INPUT_SIZE)); + outputDir = path.join(root, 'staging'); + await fs.mkdir(outputDir); + + toFile = vi + .fn() + .mockResolvedValue({ width: 800, height: 600, size: OUTPUT_SIZE }); + jpeg = vi.fn(() => ({ toFile })); + png = vi.fn(() => ({ toFile })); + webp = vi.fn(() => ({ toFile })); + rotate = vi.fn(() => ({ jpeg, png, webp })); + timeout = vi.fn(() => ({ rotate })); + mocks.sharpCreate.mockReturnValue({ + timeout, + // Second animated-input gate: a bare metadata() call precedes the + // encode pipeline; single-frame by default. + metadata: vi.fn().mockResolvedValue({ pages: 1 }), + }); + }); + + afterEach(async () => { + await fs.rm(root, { recursive: true, force: true }); + }); + + it('declares the media-policy descriptor and defaults', () => { + expect(tool.name).toBe(OMNI_CONVERT_IMAGE_TOOL_NAME); + expect(tool.mediaPolicyDescriptor).toEqual({ + kind: 'media_policy', + version: '1', + inputMediaTypes: ['image'], + outputs: [ + { + kind: 'media', + mimeTypes: ['image/jpeg', 'image/png', 'image/webp'], + required: true, + lossy: true, + }, + { kind: 'text', role: 'disclosure', required: true }, + ], + settingsSchema: expect.objectContaining({ type: 'object' }), + }); + expect(CONVERT_IMAGE_DEFAULTS).toEqual({ format: 'jpeg', quality: 90 }); + }); + + it('converts to JPEG by default with orientation baked in', async () => { + probe({ codec: 'png', frameCount: 1 }); + const { result, signal } = await run(); + + expect(mocks.probeMediaMetadata).toHaveBeenCalledWith( + inputPath, + 'image', + signal, + ); + expect(mocks.sharpCreate).toHaveBeenCalledWith(inputPath, { + failOn: 'error', + limitInputPixels: true, + }); + expect(rotate).toHaveBeenCalledOnce(); + expect(jpeg).toHaveBeenCalledWith({ quality: 90 }); + expect(toFile).toHaveBeenCalledWith(path.join(outputDir, 'converted.jpg')); + + expect(result.error).toBeUndefined(); + expect(result.artifacts).toEqual([ + { + kind: 'image', + storage: 'workspace', + title: 'Converted image', + workspacePath: 'converted.jpg', + mimeType: 'image/jpeg', + sizeBytes: OUTPUT_SIZE, + metadata: { + omniDisclosure: + '原 PNG/2MB → JPEG 质量 90/300KB，透明通道与元数据丢弃', + }, + }, + ]); + }); + + it('omits the alpha clause when the input cannot carry alpha (JPEG→JPEG)', async () => { + // A JPEG source structurally has no alpha channel — the disclosure + // must not assert a loss that cannot have occurred (D8). + probe({ codec: 'mjpeg', frameCount: 1 }); + const { result } = await run(); + expect(result.artifacts?.[0]?.metadata?.['omniDisclosure']).toBe( + '原 JPEG/2MB → JPEG 质量 90/300KB，元数据丢弃', + ); + }); + + it('softens the alpha clause when the input codec is unknown', async () => { + probe({ frameCount: 1 }); + const { result } = await run(); + expect(result.artifacts?.[0]?.metadata?.['omniDisclosure']).toBe( + '原 未知格式/2MB → JPEG 质量 90/300KB，透明通道（如有）与元数据丢弃', + ); + }); + + it('converts to PNG without a quality clause', async () => { + probe({ codec: 'mjpeg', frameCount: 1 }); + const { result } = await run({ format: 'png' }); + expect(png).toHaveBeenCalledWith(); + expect(jpeg).not.toHaveBeenCalled(); + expect(toFile).toHaveBeenCalledWith(path.join(outputDir, 'converted.png')); + expect(result.artifacts?.[0]).toMatchObject({ + workspacePath: 'converted.png', + mimeType: 'image/png', + }); + expect(result.artifacts?.[0]?.metadata?.['omniDisclosure']).toBe( + '原 JPEG/2MB → PNG/300KB，元数据丢弃', + ); + }); + + it('converts to WEBP with the quality override', async () => { + probe({ codec: 'png', frameCount: 1 }); + const { result } = await run({ format: 'webp', quality: 60 }); + expect(webp).toHaveBeenCalledWith({ quality: 60 }); + expect(result.artifacts?.[0]).toMatchObject({ + workspacePath: 'converted.webp', + mimeType: 'image/webp', + }); + expect(result.artifacts?.[0]?.metadata?.['omniDisclosure']).toBe( + '原 PNG/2MB → WEBP 质量 60/300KB，元数据丢弃', + ); + }); + + it('bounds sharp processing with the default timeout (whole seconds)', async () => { + probe({ codec: 'png', frameCount: 1 }); + await run(); + expect(timeout).toHaveBeenCalledWith({ + seconds: DEFAULT_POLICY_TOOL_TIMEOUT_MS / 1000, + }); + }); + + it('threads policyTools..runtime.timeoutMs into sharp, rounded up to seconds', async () => { + probe({ codec: 'png', frameCount: 1 }); + const configured = new OmniConvertImageTool({ + getOmniPolicyToolsSettings: () => ({ + [OMNI_CONVERT_IMAGE_TOOL_NAME]: { + runtime: { timeoutMs: 90_500 }, + }, + }), + }); + const invocation = configured.build({ inputPath, outputDir } as never); + await invocation.execute(new AbortController().signal); + expect(timeout).toHaveBeenCalledWith({ seconds: 91 }); + }); + + it('falls back to the upper-cased codec, then 未知格式, for unmapped codecs', async () => { + probe({ codec: 'jp2', frameCount: 1 }); + const first = await run(); + expect(first.result.artifacts?.[0]?.metadata?.['omniDisclosure']).toContain( + '原 JP2/', + ); + + probe({ frameCount: 1 }); + const second = await run(); + expect( + second.result.artifacts?.[0]?.metadata?.['omniDisclosure'], + ).toContain('原 未知格式/'); + }); + + it('refuses animated images instead of silently keeping one frame', async () => { + probe({ codec: 'gif', frameCount: 12 }); + const { result } = await run(); + expect(result.error?.message).toMatch( + /animated image \(12 frames\) is not supported/, + ); + expect(mocks.sharpCreate).not.toHaveBeenCalled(); + expect(result.artifacts).toBeUndefined(); + }); + + it('refuses animated images the probe missed via sharp page count', async () => { + // ffprobe reports no frame count (animated WebP/APNG headers carry + // none) — sharp's metadata() is the independent second gate. + probe({ codec: 'webp' }); + mocks.sharpCreate.mockReturnValue({ + metadata: vi.fn().mockResolvedValue({ pages: 12 }), + }); + const { result } = await run(); + expect(result.error?.message).toMatch( + /animated image \(12 frames\) is not supported/, + ); + expect(result.artifacts).toBeUndefined(); + }); + + it('returns an error result when the input file is missing', async () => { + await fs.rm(inputPath); + const { result } = await run(); + expect(result.error?.message).toMatch(/input file not found/); + }); + + it.each([ + ['relative outputDir', { outputDir: 'staging' }], + ['unknown property', { extra: 1 }], + ['unknown format', { format: 'avif' }], + ['quality out of range', { quality: 150 }], + ])('build rejects %s', (_label, overrides) => { + expect(() => + tool.build({ inputPath, outputDir, ...overrides } as never), + ).toThrow(); + }); + + it('returns an error result when sharp cannot be loaded (D9)', async () => { + vi.resetModules(); + vi.doMock('sharp', () => { + throw new Error("Cannot find module 'sharp'"); + }); + try { + const { OmniConvertImageTool: FreshTool } = await import( + './convert-image.js' + ); + probe({ codec: 'png', frameCount: 1 }); + const invocation = new FreshTool().build({ inputPath, outputDir }); + const result = await invocation.execute(new AbortController().signal); + expect(result.error?.message).toMatch( + /"sharp" image module could not be loaded/, + ); + expect(mocks.sharpCreate).not.toHaveBeenCalled(); + } finally { + vi.doUnmock('sharp'); + vi.resetModules(); + } + }); +}); diff --git a/packages/core/src/omni/policy/tools/convert-image.ts b/packages/core/src/omni/policy/tools/convert-image.ts new file mode 100644 index 00000000000..fde77e3036c --- /dev/null +++ b/packages/core/src/omni/policy/tools/convert-image.ts @@ -0,0 +1,271 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import path from 'node:path'; +import type { + MediaPolicyToolDescriptor, + ToolInvocation, + ToolResult, +} from '../../../tools/tools.js'; +import { Kind } from '../../../tools/tools.js'; +import { ToolNames } from '../../../tools/tool-names.js'; +import { probeMediaMetadata } from '../../ffmpeg.js'; +import { + assertMediaPolicyIo, + BaseMediaPolicyTool, + BaseMediaPolicyToolInvocation, + formatBytesShort, + MEDIA_POLICY_IO_SCHEMA_PROPERTIES, + mediaPolicyToolError, + mediaPolicyToolFailure, + mediaPolicyToolSuccess, + resolvePolicyToolTimeoutMs, + sharpTimeoutSeconds, + type MediaPolicyIoParams, + type MediaPolicyToolConfigView, +} from './media-policy-tool.js'; +import { loadSharp, type SharpPipeline } from './sharp-module.js'; + +export const OMNI_CONVERT_IMAGE_TOOL_NAME = ToolNames.OMNI_CONVERT_IMAGE; + +/** Fixed-call default parameters (mapping doc §6.1). */ +export const CONVERT_IMAGE_DEFAULTS = { + format: 'jpeg', + quality: 90, +} as const; + +interface OutputFormat { + fileName: string; + mimeType: string; + label: string; + /** The disclosure's loss clause for this target format, given the + * probed input codec (undefined when the probe could not tell). */ + lossNote(inputCodec: string | undefined): string; + encode(pipeline: SharpPipeline, quality: number): SharpPipeline; +} + +/** Probed codecs whose format can structurally carry an alpha channel — + * the only sources for which a JPEG target's disclosure may assert + * 透明通道丢弃 (a JPEG→JPEG re-encode cannot lose what never existed). */ +const ALPHA_CAPABLE_CODECS = new Set(['png', 'webp', 'gif', 'tiff']); + +const OUTPUT_FORMATS: Record = { + jpeg: { + fileName: 'converted.jpg', + mimeType: 'image/jpeg', + label: 'JPEG', + lossNote: (codec) => + codec === undefined + ? '透明通道（如有）与元数据丢弃' + : ALPHA_CAPABLE_CODECS.has(codec) + ? '透明通道与元数据丢弃' + : '元数据丢弃', + encode: (p, quality) => p.jpeg({ quality }), + }, + png: { + fileName: 'converted.png', + mimeType: 'image/png', + label: 'PNG', + lossNote: () => '元数据丢弃', + encode: (p) => p.png(), + }, + webp: { + fileName: 'converted.webp', + mimeType: 'image/webp', + label: 'WEBP', + lossNote: () => '元数据丢弃', + encode: (p, quality) => p.webp({ quality }), + }, +}; + +/** ffprobe codec → human format label for the disclosure's "原 X" part. */ +const CODEC_LABELS: Record = { + mjpeg: 'JPEG', + jpeg: 'JPEG', + png: 'PNG', + webp: 'WEBP', + gif: 'GIF', + bmp: 'BMP', + tiff: 'TIFF', +}; + +export interface ConvertImageParams extends MediaPolicyIoParams { + /** Target format. */ + format?: 'jpeg' | 'png' | 'webp'; + /** Quality factor (1-100) for jpeg/webp; ignored for png. */ + quality?: number; +} + +const TUNABLE_SCHEMA_PROPERTIES = { + format: { + type: 'string', + enum: ['jpeg', 'png', 'webp'], + description: "Target image format. Default 'jpeg'.", + }, + quality: { + type: 'number', + description: + 'Quality factor (1-100) for jpeg/webp output (ignored for png). Default 90.', + minimum: 1, + maximum: 100, + }, +} as const; + +const DESCRIPTOR: MediaPolicyToolDescriptor = { + kind: 'media_policy', + version: '1', + inputMediaTypes: ['image'], + outputs: [ + { + kind: 'media', + mimeTypes: ['image/jpeg', 'image/png', 'image/webp'], + required: true, + // Uniform lossy declaration (mapping doc §6.1): re-encoding strips + // metadata (and alpha, for JPEG) even when the target codec itself + // is lossless, so every conversion carries a disclosure. + lossy: true, + }, + { kind: 'text', role: 'disclosure', required: true }, + ], + settingsSchema: { + type: 'object', + properties: TUNABLE_SCHEMA_PROPERTIES, + additionalProperties: false, + }, +}; + +class ConvertImageInvocation extends BaseMediaPolicyToolInvocation { + constructor( + params: ConvertImageParams, + private readonly timeoutMs: number, + ) { + super(params); + } + + getDescription(): string { + const format = this.params.format ?? CONVERT_IMAGE_DEFAULTS.format; + return `Convert ${path.basename(this.params.inputPath)} to ${format.toUpperCase()}`; + } + + async execute(signal: AbortSignal): Promise { + const format = this.params.format ?? CONVERT_IMAGE_DEFAULTS.format; + const quality = this.params.quality ?? CONVERT_IMAGE_DEFAULTS.quality; + const output = OUTPUT_FORMATS[format]; + try { + const { inputSizeBytes } = await assertMediaPolicyIo(this.params); + + // Probe BEFORE decoding: the original format feeds the disclosure, + // and animated inputs must be refused outright — sharp would + // silently re-encode only the first frame (decision D9, same guard + // as omni_downsample_image). + const probe = await probeMediaMetadata( + this.params.inputPath, + 'image', + signal, + ); + if ((probe.frameCount ?? 1) > 1) { + return mediaPolicyToolError( + `animated image (${probe.frameCount} frames) is not supported by ${OMNI_CONVERT_IMAGE_TOOL_NAME}`, + ); + } + + let sharp; + try { + sharp = await loadSharp(); + } catch { + return mediaPolicyToolError( + 'the "sharp" image module could not be loaded; image conversion is unavailable', + ); + } + if (signal.aborted) { + return mediaPolicyToolError('image conversion aborted'); + } + + // Second, independent animated-input gate (same rationale as + // omni_downsample_image): ffprobe cannot always report a frame + // count, while sharp's metadata decodes the page count directly. + const pages = (await sharp(this.params.inputPath).metadata()).pages; + if (pages !== undefined && pages > 1) { + return mediaPolicyToolError( + `animated image (${pages} frames) is not supported by ${OMNI_CONVERT_IMAGE_TOOL_NAME}`, + ); + } + + const outputPath = path.join(this.params.outputDir, output.fileName); + // `rotate()` bakes in the EXIF orientation — the orientation tag is + // part of the metadata this conversion strips, so the pixels must + // carry it instead. + const pipeline = sharp(this.params.inputPath, { + failOn: 'error', + limitInputPixels: true, + }) + .timeout({ seconds: sharpTimeoutSeconds(this.timeoutMs) }) + .rotate(); + const info = await output.encode(pipeline, quality).toFile(outputPath); + if (signal.aborted) { + return mediaPolicyToolError('image conversion aborted'); + } + + const originalLabel = + (probe.codec !== undefined ? CODEC_LABELS[probe.codec] : undefined) ?? + probe.codec?.toUpperCase() ?? + '未知格式'; + const qualityPart = format === 'png' ? '' : ` 质量 ${quality}`; + const disclosure = `原 ${originalLabel}/${formatBytesShort(inputSizeBytes)} → ${output.label}${qualityPart}/${formatBytesShort(info.size)}，${output.lossNote(probe.codec)}`; + + return mediaPolicyToolSuccess({ + outputDir: this.params.outputDir, + outputFileName: output.fileName, + artifactKind: 'image', + title: 'Converted image', + mimeType: output.mimeType, + sizeBytes: info.size, + disclosure, + }); + } catch (error) { + return mediaPolicyToolFailure(error); + } + } +} + +/** + * `omni_convert_image` — image format conversion (sharp): re-encode to + * JPEG/PNG/WEBP with EXIF orientation baked in (mapping doc §6.1). No + * resizing — that is omni_downsample_image's job. + */ +export class OmniConvertImageTool extends BaseMediaPolicyTool { + constructor(config: MediaPolicyToolConfigView = {}) { + super( + OMNI_CONVERT_IMAGE_TOOL_NAME, + 'ConvertImage', + 'Converts an image to JPEG, PNG, or WEBP (re-encode, metadata stripped, EXIF orientation baked in), with a disclosure of the change.', + Kind.Other, + { + type: 'object', + properties: { + ...MEDIA_POLICY_IO_SCHEMA_PROPERTIES, + ...TUNABLE_SCHEMA_PROPERTIES, + }, + required: ['inputPath', 'outputDir'], + additionalProperties: false, + }, + config, + ); + } + + override get mediaPolicyDescriptor(): MediaPolicyToolDescriptor { + return DESCRIPTOR; + } + + protected createInvocation( + params: ConvertImageParams, + ): ToolInvocation { + return new ConvertImageInvocation( + params, + resolvePolicyToolTimeoutMs(this.configView, this.name), + ); + } +} diff --git a/packages/core/src/omni/policy/tools/downsample-audio.test.ts b/packages/core/src/omni/policy/tools/downsample-audio.test.ts new file mode 100644 index 00000000000..0dc7e601282 --- /dev/null +++ b/packages/core/src/omni/policy/tools/downsample-audio.test.ts @@ -0,0 +1,252 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import type { MediaProbeResult } from '../../ffmpeg.js'; +import type { ToolResult } from '../../../tools/tools.js'; +import { DEFAULT_POLICY_TOOL_TIMEOUT_MS } from './media-policy-tool.js'; +import { + DOWNSAMPLE_AUDIO_DEFAULTS, + OMNI_DOWNSAMPLE_AUDIO_TOOL_NAME, + OmniDownsampleAudioTool, +} from './downsample-audio.js'; + +const mocks = vi.hoisted(() => ({ + probeMediaMetadata: vi.fn(), + runFfmpeg: vi.fn(), +})); + +vi.mock('../../ffmpeg.js', () => ({ + probeMediaMetadata: mocks.probeMediaMetadata, + runFfmpeg: mocks.runFfmpeg, +})); + +const INPUT_SIZE = 1024 ** 2; // "1MB" +const OUTPUT_SIZE = 120 * 1024; // "120KB" + +describe('OmniDownsampleAudioTool', () => { + let root: string; + let inputPath: string; + let outputDir: string; + + const tool = new OmniDownsampleAudioTool({}); + + const probe = (result: Partial): void => { + mocks.probeMediaMetadata.mockResolvedValue(result as MediaProbeResult); + }; + + const run = async ( + params: Record = {}, + toolInstance: OmniDownsampleAudioTool = tool, + ): Promise<{ result: ToolResult; signal: AbortSignal }> => { + const invocation = toolInstance.build({ + inputPath, + outputDir, + ...params, + } as never); + const signal = new AbortController().signal; + return { result: await invocation.execute(signal), signal }; + }; + + beforeEach(async () => { + vi.clearAllMocks(); + root = await fs.mkdtemp(path.join(os.tmpdir(), 'omni-aud-')); + inputPath = path.join(root, 'track.wav'); + await fs.writeFile(inputPath, Buffer.alloc(INPUT_SIZE)); + outputDir = path.join(root, 'staging'); + await fs.mkdir(outputDir); + mocks.runFfmpeg.mockImplementation(async (args: string[]) => { + await fs.writeFile(args[args.length - 1], Buffer.alloc(OUTPUT_SIZE)); + return { code: 0, stderr: '' }; + }); + }); + + afterEach(async () => { + await fs.rm(root, { recursive: true, force: true }); + }); + + it('declares the media-policy descriptor and defaults', () => { + expect(tool.name).toBe(OMNI_DOWNSAMPLE_AUDIO_TOOL_NAME); + expect(tool.mediaPolicyDescriptor).toEqual({ + kind: 'media_policy', + version: '1', + inputMediaTypes: ['audio'], + outputs: [ + { + kind: 'media', + mimeTypes: ['audio/mp4'], + required: true, + lossy: true, + }, + { kind: 'text', role: 'disclosure', required: true }, + ], + settingsSchema: expect.objectContaining({ type: 'object' }), + }); + expect(DOWNSAMPLE_AUDIO_DEFAULTS).toEqual({ + bitrateKbps: 64, + sampleRateHz: 16_000, + channels: 1, + }); + }); + + it("defaults to 'ask' permission: a model-origin call writes files and must confirm outside yolo", async () => { + const invocation = tool.build({ inputPath, outputDir } as never); + await expect(invocation.getDefaultPermission()).resolves.toBe('ask'); + }); + + it('downsamples with the fixed-call defaults and disclosure (D8)', async () => { + probe({ bitRate: 320_000, sampleRateHz: 48_000, channels: 2 }); + const { result, signal } = await run(); + + expect(mocks.probeMediaMetadata).toHaveBeenCalledWith( + inputPath, + 'audio', + signal, + ); + const outputPath = path.join(outputDir, 'downsampled.m4a'); + expect(mocks.runFfmpeg).toHaveBeenCalledTimes(1); + expect(mocks.runFfmpeg).toHaveBeenCalledWith( + [ + '-y', + '-i', + inputPath, + '-vn', + '-c:a', + 'aac', + '-b:a', + '64k', + '-ar', + '16000', + '-ac', + '1', + outputPath, + ], + { signal, timeoutMs: DEFAULT_POLICY_TOOL_TIMEOUT_MS }, + ); + + expect(result.error).toBeUndefined(); + expect(result.artifacts).toEqual([ + { + kind: 'audio', + storage: 'workspace', + title: 'Downsampled audio', + workspacePath: 'downsampled.m4a', + mimeType: 'audio/mp4', + sizeBytes: OUTPUT_SIZE, + metadata: { + omniDisclosure: + '原 320kbps/48kHz 立体声 → 64kbps/16kHz 单声道，高频细节丢失，声道合并', + }, + }, + ]); + }); + + it('falls back to input byte size when the probe lacks a bit rate', async () => { + probe({ sampleRateHz: 44_100 }); + const { result } = await run(); + expect(result.artifacts?.[0]?.metadata?.['omniDisclosure']).toBe( + '原 1MB/44kHz → 64kbps/16kHz 单声道，高频细节丢失', + ); + }); + + it('threads tunable overrides into the ffmpeg args and disclosure', async () => { + probe({ bitRate: 256_000, sampleRateHz: 48_000, channels: 6 }); + const { result } = await run({ + bitrateKbps: 96, + sampleRateHz: 24_000, + channels: 2, + }); + const args = mocks.runFfmpeg.mock.calls[0][0] as string[]; + expect(args.join(' ')).toContain('-b:a 96k -ar 24000 -ac 2'); + expect(result.artifacts?.[0]?.metadata?.['omniDisclosure']).toBe( + '原 256kbps/48kHz 6声道 → 96kbps/24kHz 立体声，高频细节丢失，声道合并', + ); + }); + + it('clamps every target to the probed source (never "upsamples") and discloses only the re-encode', async () => { + // Source already below every default: 24kbps/8kHz/mono. + probe({ bitRate: 24_000, sampleRateHz: 8000, channels: 1 }); + const { result } = await run(); + const args = mocks.runFfmpeg.mock.calls[0][0] as string[]; + expect(args.join(' ')).toContain('-b:a 24k -ar 8000 -ac 1'); + // No frequency content above the source's own Nyquist was lost — + // claiming 高频细节丢失 here would be a false disclosure (D8). + expect(result.artifacts?.[0]?.metadata?.['omniDisclosure']).toBe( + '原 24kbps/8kHz 单声道 → 24kbps/8kHz 单声道，重新编码压缩', + ); + }); + + it('discloses 声道合并 (not 高频细节丢失) when only the channel count drops', async () => { + probe({ bitRate: 48_000, sampleRateHz: 16_000, channels: 2 }); + const { result } = await run(); + const args = mocks.runFfmpeg.mock.calls[0][0] as string[]; + expect(args.join(' ')).toContain('-b:a 48k -ar 16000 -ac 1'); + expect(result.artifacts?.[0]?.metadata?.['omniDisclosure']).toBe( + '原 48kbps/16kHz 立体声 → 48kbps/16kHz 单声道，声道合并', + ); + }); + + it('threads policyTools..runtime.timeoutMs into runFfmpeg', async () => { + probe({ bitRate: 128_000 }); + const configured = new OmniDownsampleAudioTool({ + getOmniPolicyToolsSettings: () => ({ + [OMNI_DOWNSAMPLE_AUDIO_TOOL_NAME]: { + runtime: { timeoutMs: 90_000 }, + }, + }), + }); + await run({}, configured); + expect(mocks.runFfmpeg).toHaveBeenCalledWith( + expect.any(Array), + expect.objectContaining({ timeoutMs: 90_000 }), + ); + }); + + it('reports the ffmpeg error on a non-zero exit', async () => { + probe({ bitRate: 128_000 }); + mocks.runFfmpeg.mockResolvedValue({ + code: 1, + stderr: 'Invalid data found when processing input', + }); + const { result } = await run(); + expect(result.error?.message).toMatch(/ffmpeg failed \(exit 1\)/); + expect(result.error?.message).toContain('Invalid data found'); + expect(result.artifacts).toBeUndefined(); + }); + + it('reports an aborted run', async () => { + probe({ bitRate: 128_000 }); + const controller = new AbortController(); + mocks.runFfmpeg.mockImplementation(async () => { + controller.abort(); + return { code: 1, stderr: '' }; + }); + const invocation = tool.build({ inputPath, outputDir }); + const result = await invocation.execute(controller.signal); + expect(result.error?.message).toBe('audio downsampling aborted'); + }); + + it('returns an error result when the input is a symlink', async () => { + const link = path.join(root, 'link.wav'); + await fs.symlink(inputPath, link); + const { result } = await run({ inputPath: link }); + expect(result.error?.message).toMatch(/not a regular file/); + expect(mocks.runFfmpeg).not.toHaveBeenCalled(); + }); + + it.each([ + ['relative inputPath', { inputPath: 'track.wav' }], + ['unknown property', { loudness: 5 }], + ['channels out of range', { channels: 3 }], + ])('build rejects %s', (_label, overrides) => { + expect(() => + tool.build({ inputPath, outputDir, ...overrides } as never), + ).toThrow(); + }); +}); diff --git a/packages/core/src/omni/policy/tools/downsample-audio.ts b/packages/core/src/omni/policy/tools/downsample-audio.ts new file mode 100644 index 00000000000..76b48a19a53 --- /dev/null +++ b/packages/core/src/omni/policy/tools/downsample-audio.ts @@ -0,0 +1,250 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import fs from 'node:fs/promises'; +import path from 'node:path'; +import type { + MediaPolicyToolDescriptor, + ToolInvocation, + ToolResult, +} from '../../../tools/tools.js'; +import { Kind } from '../../../tools/tools.js'; +import { ToolNames } from '../../../tools/tool-names.js'; +import { probeMediaMetadata, runFfmpeg } from '../../ffmpeg.js'; +import { + assertMediaPolicyIo, + BaseMediaPolicyTool, + ffmpegFailureMessage, + BaseMediaPolicyToolInvocation, + describeChannels, + formatBytesShort, + MEDIA_POLICY_IO_SCHEMA_PROPERTIES, + mediaPolicyToolError, + mediaPolicyToolFailure, + mediaPolicyToolSuccess, + resolvePolicyToolTimeoutMs, + type MediaPolicyIoParams, + type MediaPolicyToolConfigView, +} from './media-policy-tool.js'; + +export const OMNI_DOWNSAMPLE_AUDIO_TOOL_NAME = ToolNames.OMNI_DOWNSAMPLE_AUDIO; + +/** Fixed-call default parameters (mapping doc §6). */ +export const DOWNSAMPLE_AUDIO_DEFAULTS = { + bitrateKbps: 64, + sampleRateHz: 16_000, + channels: 1, +} as const; + +const OUTPUT_FILE_NAME = 'downsampled.m4a'; + +export interface DownsampleAudioParams extends MediaPolicyIoParams { + /** Output bit rate in kbit/s. */ + bitrateKbps?: number; + /** Output sample rate in Hz. */ + sampleRateHz?: number; + /** Output channel count. */ + channels?: number; +} + +const TUNABLE_SCHEMA_PROPERTIES = { + bitrateKbps: { + type: 'number', + description: 'Output bit rate in kbit/s. Default 64.', + minimum: 8, + }, + sampleRateHz: { + type: 'number', + description: 'Output sample rate in Hz. Default 16000.', + minimum: 8000, + }, + channels: { + type: 'number', + description: 'Output channel count. Default 1 (mono).', + minimum: 1, + maximum: 2, + }, +} as const; + +const DESCRIPTOR: MediaPolicyToolDescriptor = { + kind: 'media_policy', + version: '1', + inputMediaTypes: ['audio'], + outputs: [ + { + kind: 'media', + mimeTypes: ['audio/mp4'], + required: true, + lossy: true, + }, + { kind: 'text', role: 'disclosure', required: true }, + ], + settingsSchema: { + type: 'object', + properties: TUNABLE_SCHEMA_PROPERTIES, + additionalProperties: false, + }, +}; + +class DownsampleAudioInvocation extends BaseMediaPolicyToolInvocation { + constructor( + params: DownsampleAudioParams, + private readonly timeoutMs: number, + ) { + super(params); + } + + getDescription(): string { + const bitrateKbps = + this.params.bitrateKbps ?? DOWNSAMPLE_AUDIO_DEFAULTS.bitrateKbps; + return `Downsample ${path.basename(this.params.inputPath)} to ${bitrateKbps}kbps`; + } + + async execute(signal: AbortSignal): Promise { + const requestedBitrateKbps = + this.params.bitrateKbps ?? DOWNSAMPLE_AUDIO_DEFAULTS.bitrateKbps; + const requestedSampleRateHz = + this.params.sampleRateHz ?? DOWNSAMPLE_AUDIO_DEFAULTS.sampleRateHz; + const requestedChannels = + this.params.channels ?? DOWNSAMPLE_AUDIO_DEFAULTS.channels; + try { + const { inputSizeBytes } = await assertMediaPolicyIo(this.params); + const probe = await probeMediaMetadata( + this.params.inputPath, + 'audio', + signal, + ); + + // Never "upsample": clamp each target to the probed source (same + // guard as downsample-image's withoutEnlargement and + // downscale-video's Math.min against probe.height). Without the + // clamp, a source already below the targets re-encodes into a + // LARGER derivative that the transport guard counts as progress, + // under a disclosure claiming losses that never happened. + const bitrateKbps = + probe.bitRate !== undefined + ? Math.min(requestedBitrateKbps, Math.ceil(probe.bitRate / 1000)) + : requestedBitrateKbps; + const sampleRateHz = + probe.sampleRateHz !== undefined + ? Math.min(requestedSampleRateHz, probe.sampleRateHz) + : requestedSampleRateHz; + const channels = + probe.channels !== undefined + ? Math.min(requestedChannels, probe.channels) + : requestedChannels; + + const outputPath = path.join(this.params.outputDir, OUTPUT_FILE_NAME); + const run = await runFfmpeg( + [ + '-y', + '-i', + this.params.inputPath, + // Audio-only output: a cover-art video stream would otherwise be + // carried along (and can even fail the m4a mux). + '-vn', + '-c:a', + 'aac', + '-b:a', + `${bitrateKbps}k`, + '-ar', + String(sampleRateHz), + '-ac', + String(channels), + outputPath, + ], + { signal, timeoutMs: this.timeoutMs }, + ); + if (signal.aborted) { + return mediaPolicyToolError('audio downsampling aborted'); + } + if (run.code !== 0) { + return mediaPolicyToolError( + ffmpegFailureMessage(run, 'downsampling', this.params.inputPath), + ); + } + + const outputSizeBytes = (await fs.stat(outputPath)).size; + const originalBitrate = + probe.bitRate !== undefined + ? `${Math.round(probe.bitRate / 1000)}kbps` + : formatBytesShort(inputSizeBytes); + const originalRate = + probe.sampleRateHz !== undefined + ? `/${Math.round(probe.sampleRateHz / 1000)}kHz` + : ''; + // D8 accuracy: only claim the losses that actually happened. A + // sample-rate or bit-rate drop removes high-frequency detail; a + // channel drop merges the stereo image; and when the clamps left + // every parameter at the source's own values the only change is + // the lossy re-encode itself. + const drops: string[] = []; + if ( + (probe.bitRate !== undefined && + bitrateKbps < Math.ceil(probe.bitRate / 1000)) || + (probe.sampleRateHz !== undefined && sampleRateHz < probe.sampleRateHz) + ) { + drops.push('高频细节丢失'); + } + if (probe.channels !== undefined && channels < probe.channels) { + drops.push('声道合并'); + } + const lossNote = drops.length > 0 ? drops.join('，') : '重新编码压缩'; + const disclosure = `原 ${originalBitrate}${originalRate}${describeChannels(probe.channels)} → ${bitrateKbps}kbps/${Math.round(sampleRateHz / 1000)}kHz${describeChannels(channels)}，${lossNote}`; + + return mediaPolicyToolSuccess({ + outputDir: this.params.outputDir, + outputFileName: OUTPUT_FILE_NAME, + artifactKind: 'audio', + title: 'Downsampled audio', + mimeType: 'audio/mp4', + sizeBytes: outputSizeBytes, + disclosure, + }); + } catch (error) { + return mediaPolicyToolFailure(error); + } + } +} + +/** + * `omni_downsample_audio` — lossy audio degradation (ffmpeg): re-encode + * to AAC at a low bit rate, sample rate, and channel count (mapping doc + * §6). + */ +export class OmniDownsampleAudioTool extends BaseMediaPolicyTool { + constructor(config: MediaPolicyToolConfigView) { + super( + OMNI_DOWNSAMPLE_AUDIO_TOOL_NAME, + 'DownsampleAudio', + 'Downsamples an audio file to a lower bit rate, sample rate, and channel count, producing a smaller lossy derivative with a disclosure of the degradation.', + Kind.Other, + { + type: 'object', + properties: { + ...MEDIA_POLICY_IO_SCHEMA_PROPERTIES, + ...TUNABLE_SCHEMA_PROPERTIES, + }, + required: ['inputPath', 'outputDir'], + additionalProperties: false, + }, + config, + ); + } + + override get mediaPolicyDescriptor(): MediaPolicyToolDescriptor { + return DESCRIPTOR; + } + + protected createInvocation( + params: DownsampleAudioParams, + ): ToolInvocation { + return new DownsampleAudioInvocation( + params, + resolvePolicyToolTimeoutMs(this.configView, this.name), + ); + } +} diff --git a/packages/core/src/omni/policy/tools/downsample-image.test.ts b/packages/core/src/omni/policy/tools/downsample-image.test.ts new file mode 100644 index 00000000000..8d7e43614af --- /dev/null +++ b/packages/core/src/omni/policy/tools/downsample-image.test.ts @@ -0,0 +1,264 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import type { MediaProbeResult } from '../../ffmpeg.js'; +import type { ToolResult } from '../../../tools/tools.js'; +import { DEFAULT_POLICY_TOOL_TIMEOUT_MS } from './media-policy-tool.js'; +import { + DOWNSAMPLE_IMAGE_DEFAULTS, + OMNI_DOWNSAMPLE_IMAGE_TOOL_NAME, + OmniDownsampleImageTool, +} from './downsample-image.js'; + +const mocks = vi.hoisted(() => ({ + probeMediaMetadata: vi.fn(), + runFfmpeg: vi.fn(), + sharpCreate: vi.fn(), +})); + +vi.mock('../../ffmpeg.js', () => ({ + probeMediaMetadata: mocks.probeMediaMetadata, + runFfmpeg: mocks.runFfmpeg, +})); + +vi.mock('sharp', () => ({ + default: (...args: unknown[]) => mocks.sharpCreate(...args), +})); + +const INPUT_SIZE = 2 * 1024 ** 2; // "2MB" +const OUTPUT_SIZE = 300 * 1024; // "300KB" + +describe('OmniDownsampleImageTool', () => { + let root: string; + let inputPath: string; + let outputDir: string; + let toFile: ReturnType; + let jpeg: ReturnType; + let resize: ReturnType; + let rotate: ReturnType; + let timeout: ReturnType; + + const tool = new OmniDownsampleImageTool(); + + const probe = (result: Partial): void => { + mocks.probeMediaMetadata.mockResolvedValue(result as MediaProbeResult); + }; + + const run = async ( + params: Record = {}, + ): Promise<{ result: ToolResult; signal: AbortSignal }> => { + const invocation = tool.build({ + inputPath, + outputDir, + ...params, + } as never); + const signal = new AbortController().signal; + return { result: await invocation.execute(signal), signal }; + }; + + beforeEach(async () => { + vi.clearAllMocks(); + root = await fs.mkdtemp(path.join(os.tmpdir(), 'omni-img-')); + inputPath = path.join(root, 'photo.png'); + await fs.writeFile(inputPath, Buffer.alloc(INPUT_SIZE)); + outputDir = path.join(root, 'staging'); + await fs.mkdir(outputDir); + + toFile = vi + .fn() + .mockResolvedValue({ width: 1568, height: 1176, size: OUTPUT_SIZE }); + jpeg = vi.fn(() => ({ toFile })); + resize = vi.fn(() => ({ jpeg })); + rotate = vi.fn(() => ({ resize })); + timeout = vi.fn(() => ({ rotate })); + mocks.sharpCreate.mockReturnValue({ + timeout, + // Second animated-input gate: a bare metadata() call precedes the + // encode pipeline; single-frame by default. + metadata: vi.fn().mockResolvedValue({ pages: 1 }), + }); + }); + + afterEach(async () => { + await fs.rm(root, { recursive: true, force: true }); + }); + + it('declares the media-policy descriptor and tool name', () => { + expect(tool.name).toBe(OMNI_DOWNSAMPLE_IMAGE_TOOL_NAME); + expect(tool.mediaPolicyDescriptor).toEqual({ + kind: 'media_policy', + version: '1', + inputMediaTypes: ['image'], + outputs: [ + { + kind: 'media', + mimeTypes: ['image/jpeg'], + required: true, + lossy: true, + }, + { kind: 'text', role: 'disclosure', required: true }, + ], + settingsSchema: expect.objectContaining({ type: 'object' }), + }); + expect(DOWNSAMPLE_IMAGE_DEFAULTS).toEqual({ + maxDimension: 1568, + quality: 75, + }); + }); + + it("defaults to 'ask' permission: a model-origin call writes files and must confirm outside yolo", async () => { + const invocation = tool.build({ inputPath, outputDir } as never); + await expect(invocation.getDefaultPermission()).resolves.toBe('ask'); + }); + + it('downsamples with the fixed-call defaults and disclosure (D8)', async () => { + probe({ width: 4096, height: 3072, frameCount: 1 }); + const { result, signal } = await run(); + + expect(mocks.probeMediaMetadata).toHaveBeenCalledWith( + inputPath, + 'image', + signal, + ); + expect(mocks.sharpCreate).toHaveBeenCalledWith(inputPath, { + failOn: 'error', + limitInputPixels: true, + }); + expect(rotate).toHaveBeenCalledOnce(); + expect(resize).toHaveBeenCalledWith({ + width: 1568, + height: 1568, + fit: 'inside', + withoutEnlargement: true, + }); + expect(jpeg).toHaveBeenCalledWith({ quality: 75 }); + expect(toFile).toHaveBeenCalledWith( + path.join(outputDir, 'downsampled.jpg'), + ); + + expect(result.error).toBeUndefined(); + expect(result.artifacts).toEqual([ + { + kind: 'image', + storage: 'workspace', + title: 'Downsampled image', + workspacePath: 'downsampled.jpg', + mimeType: 'image/jpeg', + sizeBytes: OUTPUT_SIZE, + metadata: { + omniDisclosure: + '原 4096×3072/2MB → 1568×1176/300KB，质量 75，细节与文字锐度受损', + }, + }, + ]); + expect(result.llmContent).toContain('Downsampled image'); + }); + + it('threads tunable overrides into sharp', async () => { + probe({ width: 4000, height: 3000, frameCount: 1 }); + await run({ maxDimension: 800, quality: 50 }); + expect(resize).toHaveBeenCalledWith( + expect.objectContaining({ width: 800, height: 800 }), + ); + expect(jpeg).toHaveBeenCalledWith({ quality: 50 }); + }); + + it('bounds sharp processing with the default timeout (whole seconds)', async () => { + probe({ width: 4096, height: 3072, frameCount: 1 }); + await run(); + expect(timeout).toHaveBeenCalledWith({ + seconds: DEFAULT_POLICY_TOOL_TIMEOUT_MS / 1000, + }); + }); + + it('threads policyTools..runtime.timeoutMs into sharp, rounded up to seconds', async () => { + probe({ width: 4096, height: 3072, frameCount: 1 }); + const configured = new OmniDownsampleImageTool({ + getOmniPolicyToolsSettings: () => ({ + [OMNI_DOWNSAMPLE_IMAGE_TOOL_NAME]: { + runtime: { timeoutMs: 90_500 }, + }, + }), + }); + const invocation = configured.build({ inputPath, outputDir } as never); + await invocation.execute(new AbortController().signal); + expect(timeout).toHaveBeenCalledWith({ seconds: 91 }); + }); + + it('omits original dimensions from the disclosure when the probe lacks them', async () => { + probe({ frameCount: 1 }); + const { result } = await run(); + expect(result.artifacts?.[0]?.metadata?.['omniDisclosure']).toBe( + '原 2MB → 1568×1176/300KB，质量 75，细节与文字锐度受损', + ); + }); + + it('refuses animated images instead of silently keeping one frame', async () => { + probe({ width: 640, height: 480, frameCount: 12 }); + const { result } = await run(); + expect(result.error?.message).toMatch( + /animated image \(12 frames\) is not supported/, + ); + expect(mocks.sharpCreate).not.toHaveBeenCalled(); + expect(result.artifacts).toBeUndefined(); + }); + + it('refuses animated images the probe missed via sharp page count', async () => { + // ffprobe reports no frame count (animated WebP/APNG headers carry + // none) — sharp's metadata() is the independent second gate. + probe({ width: 640, height: 480 }); + mocks.sharpCreate.mockReturnValue({ + metadata: vi.fn().mockResolvedValue({ pages: 12 }), + }); + const { result } = await run(); + expect(result.error?.message).toMatch( + /animated image \(12 frames\) is not supported/, + ); + expect(result.artifacts).toBeUndefined(); + }); + + it('returns an error result when the input file is missing', async () => { + await fs.rm(inputPath); + const { result } = await run(); + expect(result.error?.message).toMatch(/input file not found/); + }); + + it.each([ + ['relative inputPath', { inputPath: 'rel.png' }], + ['unknown property', { extra: true }], + ['quality out of range', { quality: 150 }], + ])('build rejects %s', (_label, overrides) => { + expect(() => + tool.build({ inputPath, outputDir, ...overrides } as never), + ).toThrow(); + }); + + it('returns an error result when sharp cannot be loaded (D9)', async () => { + vi.resetModules(); + vi.doMock('sharp', () => { + throw new Error("Cannot find module 'sharp'"); + }); + try { + const { OmniDownsampleImageTool: FreshTool } = await import( + './downsample-image.js' + ); + probe({ width: 100, height: 100, frameCount: 1 }); + const invocation = new FreshTool().build({ inputPath, outputDir }); + const result = await invocation.execute(new AbortController().signal); + expect(result.error?.message).toMatch( + /"sharp" image module could not be loaded/, + ); + expect(mocks.sharpCreate).not.toHaveBeenCalled(); + } finally { + vi.doUnmock('sharp'); + vi.resetModules(); + } + }); +}); diff --git a/packages/core/src/omni/policy/tools/downsample-image.ts b/packages/core/src/omni/policy/tools/downsample-image.ts new file mode 100644 index 00000000000..596662c97e7 --- /dev/null +++ b/packages/core/src/omni/policy/tools/downsample-image.ts @@ -0,0 +1,231 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import path from 'node:path'; +import type { + MediaPolicyToolDescriptor, + ToolInvocation, + ToolResult, +} from '../../../tools/tools.js'; +import { Kind } from '../../../tools/tools.js'; +import { ToolNames } from '../../../tools/tool-names.js'; +import { probeMediaMetadata } from '../../ffmpeg.js'; +import { + assertMediaPolicyIo, + BaseMediaPolicyTool, + BaseMediaPolicyToolInvocation, + formatBytesShort, + MEDIA_POLICY_IO_SCHEMA_PROPERTIES, + mediaPolicyToolError, + mediaPolicyToolFailure, + mediaPolicyToolSuccess, + resolvePolicyToolTimeoutMs, + sharpTimeoutSeconds, + type MediaPolicyIoParams, + type MediaPolicyToolConfigView, +} from './media-policy-tool.js'; +import { loadSharp, type SharpModule } from './sharp-module.js'; + +export const OMNI_DOWNSAMPLE_IMAGE_TOOL_NAME = ToolNames.OMNI_DOWNSAMPLE_IMAGE; + +/** Fixed-call default parameters (mapping doc §6). */ +export const DOWNSAMPLE_IMAGE_DEFAULTS = { + maxDimension: 1568, + quality: 75, +} as const; + +const OUTPUT_FILE_NAME = 'downsampled.jpg'; + +export interface DownsampleImageParams extends MediaPolicyIoParams { + /** Longest-edge ceiling in pixels; aspect ratio is preserved. */ + maxDimension?: number; + /** JPEG quality factor of the re-encode (1-100). */ + quality?: number; +} + +const TUNABLE_SCHEMA_PROPERTIES = { + maxDimension: { + type: 'number', + description: + 'Longest-edge ceiling in pixels (aspect ratio preserved). Default 1568.', + minimum: 1, + }, + quality: { + type: 'number', + description: 'JPEG quality factor of the re-encode (1-100). Default 75.', + minimum: 1, + maximum: 100, + }, +} as const; + +const DESCRIPTOR: MediaPolicyToolDescriptor = { + kind: 'media_policy', + version: '1', + inputMediaTypes: ['image'], + outputs: [ + { + kind: 'media', + mimeTypes: ['image/jpeg'], + required: true, + lossy: true, + }, + { kind: 'text', role: 'disclosure', required: true }, + ], + settingsSchema: { + type: 'object', + properties: TUNABLE_SCHEMA_PROPERTIES, + additionalProperties: false, + }, +}; + +class DownsampleImageInvocation extends BaseMediaPolicyToolInvocation { + constructor( + params: DownsampleImageParams, + private readonly timeoutMs: number, + ) { + super(params); + } + + getDescription(): string { + const maxDimension = + this.params.maxDimension ?? DOWNSAMPLE_IMAGE_DEFAULTS.maxDimension; + return `Downsample ${path.basename(this.params.inputPath)} to fit ${maxDimension}px`; + } + + async execute(signal: AbortSignal): Promise { + const maxDimension = + this.params.maxDimension ?? DOWNSAMPLE_IMAGE_DEFAULTS.maxDimension; + const quality = this.params.quality ?? DOWNSAMPLE_IMAGE_DEFAULTS.quality; + try { + const { inputSizeBytes } = await assertMediaPolicyIo(this.params); + + // Probe BEFORE decoding: the original dimensions feed the disclosure, + // and animated inputs must be refused outright — sharp would silently + // re-encode only the first frame, destroying the animation without + // any disclosure of that loss (decision D9: animated images are + // excluded from the image policy; the guard handles them). + const probe = await probeMediaMetadata( + this.params.inputPath, + 'image', + signal, + ); + if ((probe.frameCount ?? 1) > 1) { + return mediaPolicyToolError( + `animated image (${probe.frameCount} frames) is not supported by ${OMNI_DOWNSAMPLE_IMAGE_TOOL_NAME}`, + ); + } + + let sharp: SharpModule; + try { + sharp = await loadSharp(); + } catch { + return mediaPolicyToolError( + 'the "sharp" image module could not be loaded; image downsampling is unavailable', + ); + } + if (signal.aborted) { + return mediaPolicyToolError('image downsampling aborted'); + } + + // Second, independent animated-input gate: ffprobe cannot always + // report a frame count (and the counting fallback is best-effort), + // while sharp's own metadata decodes the page count directly. Both + // must agree the input is single-frame before the first-frame-only + // re-encode below is lossless-in-frames. + const pages = (await sharp(this.params.inputPath).metadata()).pages; + if (pages !== undefined && pages > 1) { + return mediaPolicyToolError( + `animated image (${pages} frames) is not supported by ${OMNI_DOWNSAMPLE_IMAGE_TOOL_NAME}`, + ); + } + + const outputPath = path.join(this.params.outputDir, OUTPUT_FILE_NAME); + // `rotate()` bakes in the EXIF orientation so the resized pixels + // match what the user saw; `fit: 'inside'` preserves aspect ratio; + // `withoutEnlargement` keeps already-small originals at native size. + // PNG and other lossless inputs are re-encoded to JPEG too — the + // whole point of the policy is a smaller transport payload. + const info = await sharp(this.params.inputPath, { + failOn: 'error', + limitInputPixels: true, + }) + .timeout({ seconds: sharpTimeoutSeconds(this.timeoutMs) }) + .rotate() + .resize({ + width: maxDimension, + height: maxDimension, + fit: 'inside', + withoutEnlargement: true, + }) + .jpeg({ quality }) + .toFile(outputPath); + if (signal.aborted) { + return mediaPolicyToolError('image downsampling aborted'); + } + + // Disclosure (decision D8): dimensions/bytes plus the OUTPUT quality + // parameter only — the original's JPEG quality factor is not stored + // in the bitstream, so no claim is made about it. + const original = + probe.width !== undefined && probe.height !== undefined + ? `${probe.width}×${probe.height}/${formatBytesShort(inputSizeBytes)}` + : formatBytesShort(inputSizeBytes); + const disclosure = `原 ${original} → ${info.width}×${info.height}/${formatBytesShort(info.size)}，质量 ${quality}，细节与文字锐度受损`; + + return mediaPolicyToolSuccess({ + outputDir: this.params.outputDir, + outputFileName: OUTPUT_FILE_NAME, + artifactKind: 'image', + title: 'Downsampled image', + mimeType: 'image/jpeg', + sizeBytes: info.size, + disclosure, + }); + } catch (error) { + return mediaPolicyToolFailure(error); + } + } +} + +/** + * `omni_downsample_image` — lossy image degradation (sharp): scale to fit + * `maxDimension` and re-encode as JPEG at `quality` (mapping doc §6). + * Registered as a media-policy tool: fixed-policy-only unless modelAccess + * opens it up. + */ +export class OmniDownsampleImageTool extends BaseMediaPolicyTool { + constructor(config: MediaPolicyToolConfigView = {}) { + super( + OMNI_DOWNSAMPLE_IMAGE_TOOL_NAME, + 'DownsampleImage', + 'Downsamples an image to fit a maximum dimension and re-encodes it as JPEG, producing a smaller lossy derivative with a disclosure of the degradation.', + Kind.Other, + { + type: 'object', + properties: { + ...MEDIA_POLICY_IO_SCHEMA_PROPERTIES, + ...TUNABLE_SCHEMA_PROPERTIES, + }, + required: ['inputPath', 'outputDir'], + additionalProperties: false, + }, + config, + ); + } + + override get mediaPolicyDescriptor(): MediaPolicyToolDescriptor { + return DESCRIPTOR; + } + + protected createInvocation( + params: DownsampleImageParams, + ): ToolInvocation { + return new DownsampleImageInvocation( + params, + resolvePolicyToolTimeoutMs(this.configView, this.name), + ); + } +} diff --git a/packages/core/src/omni/policy/tools/downscale-video.test.ts b/packages/core/src/omni/policy/tools/downscale-video.test.ts new file mode 100644 index 00000000000..e079fa56970 --- /dev/null +++ b/packages/core/src/omni/policy/tools/downscale-video.test.ts @@ -0,0 +1,316 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import type { MediaProbeResult } from '../../ffmpeg.js'; +import type { ToolResult } from '../../../tools/tools.js'; +import { DEFAULT_POLICY_TOOL_TIMEOUT_MS } from './media-policy-tool.js'; +import { + DOWNSCALE_VIDEO_DEFAULTS, + OMNI_DOWNSCALE_VIDEO_TOOL_NAME, + OmniDownscaleVideoTool, +} from './downscale-video.js'; + +const mocks = vi.hoisted(() => ({ + probeMediaMetadata: vi.fn(), + runFfmpeg: vi.fn(), +})); + +vi.mock('../../ffmpeg.js', () => ({ + probeMediaMetadata: mocks.probeMediaMetadata, + runFfmpeg: mocks.runFfmpeg, +})); + +const INPUT_SIZE = 2 * 1024 ** 2; // "2MB" +const OUTPUT_SIZE = 300 * 1024; // "300KB" + +describe('OmniDownscaleVideoTool', () => { + let root: string; + let inputPath: string; + let outputDir: string; + + const tool = new OmniDownscaleVideoTool({}); + + const probe = (result: Partial): void => { + mocks.probeMediaMetadata.mockResolvedValue(result as MediaProbeResult); + }; + + /** ffmpeg success: writes the output file (last arg) and exits 0. */ + const ffmpegSucceeds = (): void => { + mocks.runFfmpeg.mockImplementation(async (args: string[]) => { + await fs.writeFile(args[args.length - 1], Buffer.alloc(OUTPUT_SIZE)); + return { code: 0, stderr: '' }; + }); + }; + + const run = async ( + params: Record = {}, + toolInstance: OmniDownscaleVideoTool = tool, + ): Promise<{ result: ToolResult; signal: AbortSignal }> => { + const invocation = toolInstance.build({ + inputPath, + outputDir, + ...params, + } as never); + const signal = new AbortController().signal; + return { result: await invocation.execute(signal), signal }; + }; + + beforeEach(async () => { + vi.clearAllMocks(); + root = await fs.mkdtemp(path.join(os.tmpdir(), 'omni-vid-')); + inputPath = path.join(root, 'clip.mov'); + await fs.writeFile(inputPath, Buffer.alloc(INPUT_SIZE)); + outputDir = path.join(root, 'staging'); + await fs.mkdir(outputDir); + ffmpegSucceeds(); + }); + + afterEach(async () => { + await fs.rm(root, { recursive: true, force: true }); + }); + + it('declares the media-policy descriptor and defaults', () => { + expect(tool.name).toBe(OMNI_DOWNSCALE_VIDEO_TOOL_NAME); + expect(tool.mediaPolicyDescriptor).toEqual({ + kind: 'media_policy', + version: '1', + inputMediaTypes: ['video'], + outputs: [ + { + kind: 'media', + mimeTypes: ['video/mp4'], + required: true, + lossy: true, + }, + { kind: 'text', role: 'disclosure', required: true }, + ], + settingsSchema: expect.objectContaining({ type: 'object' }), + }); + expect(DOWNSCALE_VIDEO_DEFAULTS).toEqual({ + maxHeight: 480, + fps: 10, + crf: 28, + preset: 'veryfast', + }); + }); + + it("defaults to 'ask' permission: a model-origin call writes files and must confirm outside yolo", async () => { + const invocation = tool.build({ inputPath, outputDir } as never); + await expect(invocation.getDefaultPermission()).resolves.toBe('ask'); + }); + + it('downscales with the fixed-call defaults, audio stream-copied', async () => { + probe({ height: 1080, frameRate: 30 }); + const { result, signal } = await run(); + + expect(mocks.probeMediaMetadata).toHaveBeenCalledWith( + inputPath, + 'video', + signal, + ); + const outputPath = path.join(outputDir, 'downscaled.mp4'); + expect(mocks.runFfmpeg).toHaveBeenCalledTimes(1); + expect(mocks.runFfmpeg).toHaveBeenCalledWith( + [ + '-y', + '-i', + inputPath, + '-vf', + 'scale=-2:480,fps=10', + '-c:v', + 'libx264', + '-crf', + '28', + '-preset', + 'veryfast', + '-c:a', + 'copy', + outputPath, + ], + { signal, timeoutMs: DEFAULT_POLICY_TOOL_TIMEOUT_MS }, + ); + + expect(result.error).toBeUndefined(); + expect(result.artifacts).toEqual([ + { + kind: 'video', + storage: 'workspace', + title: 'Downscaled video', + workspacePath: 'downscaled.mp4', + mimeType: 'video/mp4', + sizeBytes: OUTPUT_SIZE, + metadata: { + omniDisclosure: + '原 1080p30/2MB → 480p10/300KB，分辨率与帧率下降，细节受损', + }, + }, + ]); + }); + + it('never upscales and rounds the target height down to even', async () => { + probe({ height: 359, frameRate: 24 }); + const { result } = await run(); + const args = mocks.runFfmpeg.mock.calls[0][0] as string[]; + expect(args).toContain('scale=-2:358,fps=10'); + expect(result.artifacts?.[0]?.metadata?.['omniDisclosure']).toBe( + '原 359p24/2MB → 358p10/300KB，分辨率与帧率下降，细节受损', + ); + }); + + it('discloses only the dimensions that actually dropped', async () => { + // 360p@8fps against the 480p/10fps defaults: neither the height nor + // the frame rate goes down — the loss clause must not claim it did. + probe({ height: 360, frameRate: 8 }); + const { result } = await run(); + expect(result.artifacts?.[0]?.metadata?.['omniDisclosure']).toBe( + '原 360p8/2MB → 360p10/300KB，重新编码压缩，细节受损', + ); + + // Height drops, frame rate does not. + probe({ height: 720, frameRate: 8 }); + const heightOnly = await run(); + expect( + heightOnly.result.artifacts?.[0]?.metadata?.['omniDisclosure'], + ).toBe('原 720p8/2MB → 480p10/300KB，分辨率下降，细节受损'); + + // Frame rate drops, height does not. + probe({ height: 360, frameRate: 30 }); + const rateOnly = await run(); + expect(rateOnly.result.artifacts?.[0]?.metadata?.['omniDisclosure']).toBe( + '原 360p30/2MB → 360p10/300KB，帧率下降，细节受损', + ); + }); + + it('falls back to AAC 64k when audio stream copy fails', async () => { + probe({ height: 720, frameRate: 25 }); + mocks.runFfmpeg + .mockResolvedValueOnce({ code: 1, stderr: 'pcm in mp4 unsupported' }) + .mockImplementationOnce(async (args: string[]) => { + await fs.writeFile(args[args.length - 1], Buffer.alloc(OUTPUT_SIZE)); + return { code: 0, stderr: '' }; + }); + + const { result } = await run(); + expect(result.error).toBeUndefined(); + expect(mocks.runFfmpeg).toHaveBeenCalledTimes(2); + const firstArgs = mocks.runFfmpeg.mock.calls[0][0] as string[]; + const secondArgs = mocks.runFfmpeg.mock.calls[1][0] as string[]; + expect(firstArgs).toContain('copy'); + expect(secondArgs).not.toContain('copy'); + expect(secondArgs.join(' ')).toContain('-c:a aac -b:a 64k'); + }); + + it('charges the AAC fallback against the SAME wall-clock budget (no timeout doubling)', async () => { + probe({ height: 720, frameRate: 25 }); + mocks.runFfmpeg + .mockImplementationOnce(async () => { + // Burn measurable wall-clock time in the failing copy pass. + await new Promise((r) => setTimeout(r, 50)); + return { code: 1, stderr: 'pcm in mp4 unsupported' }; + }) + .mockImplementationOnce(async (args: string[]) => { + await fs.writeFile(args[args.length - 1], Buffer.alloc(OUTPUT_SIZE)); + return { code: 0, stderr: '' }; + }); + + const { result } = await run(); + expect(result.error).toBeUndefined(); + expect(mocks.runFfmpeg).toHaveBeenCalledTimes(2); + const firstTimeout = ( + mocks.runFfmpeg.mock.calls[0][1] as { timeoutMs: number } + ).timeoutMs; + const secondTimeout = ( + mocks.runFfmpeg.mock.calls[1][1] as { timeoutMs: number } + ).timeoutMs; + expect(firstTimeout).toBe(DEFAULT_POLICY_TOOL_TIMEOUT_MS); + // The fallback gets only what the copy pass left, never a fresh + // full budget (timers never fire early, so ≥40ms must be gone). + expect(secondTimeout).toBeLessThanOrEqual( + DEFAULT_POLICY_TOOL_TIMEOUT_MS - 40, + ); + expect(secondTimeout).toBeGreaterThan(0); + }); + + it('reports the ffmpeg error when both attempts fail', async () => { + probe({ height: 720, frameRate: 25 }); + mocks.runFfmpeg.mockResolvedValue({ + code: 187, + stderr: 'Conversion failed!', + }); + const { result } = await run(); + expect(mocks.runFfmpeg).toHaveBeenCalledTimes(2); + expect(result.error?.message).toMatch(/ffmpeg failed \(exit 187\)/); + expect(result.error?.message).toContain('Conversion failed!'); + expect(result.artifacts).toBeUndefined(); + }); + + it('threads policyTools..runtime.timeoutMs into runFfmpeg', async () => { + probe({ height: 720, frameRate: 25 }); + const configured = new OmniDownscaleVideoTool({ + getOmniPolicyToolsSettings: () => ({ + [OMNI_DOWNSCALE_VIDEO_TOOL_NAME]: { + runtime: { timeoutMs: 120_000 }, + }, + }), + }); + await run({}, configured); + expect(mocks.runFfmpeg).toHaveBeenCalledWith( + expect.any(Array), + expect.objectContaining({ timeoutMs: 120_000 }), + ); + }); + + it('threads tunable overrides into the ffmpeg args', async () => { + probe({ height: 2160, frameRate: 60 }); + await run({ maxHeight: 720, fps: 15, crf: 32, preset: 'fast' }); + const args = mocks.runFfmpeg.mock.calls[0][0] as string[]; + expect(args).toContain('scale=-2:720,fps=15'); + expect(args.join(' ')).toContain('-crf 32 -preset fast'); + }); + + it('reports an aborted run without attempting the audio fallback', async () => { + probe({ height: 720, frameRate: 25 }); + const controller = new AbortController(); + mocks.runFfmpeg.mockImplementation(async () => { + controller.abort(); + return { code: 1, stderr: '' }; + }); + const invocation = tool.build({ inputPath, outputDir }); + const result = await invocation.execute(controller.signal); + expect(result.error?.message).toBe('video downscaling aborted'); + expect(mocks.runFfmpeg).toHaveBeenCalledTimes(1); + }); + + it('errors when the probe cannot determine the video height', async () => { + probe({ frameRate: 25 }); + const { result } = await run(); + expect(result.error?.message).toMatch(/could not determine video height/); + expect(mocks.runFfmpeg).not.toHaveBeenCalled(); + }); + + it('renders an unknown original frame rate as "?"', async () => { + probe({ height: 480 }); + const { result } = await run(); + expect(result.artifacts?.[0]?.metadata?.['omniDisclosure']).toContain( + '原 480p?/', + ); + }); + + it.each([ + ['relative outputDir', { outputDir: 'staging' }], + ['unknown property', { extra: 1 }], + ['invalid preset', { preset: 'warp-speed' }], + ['crf out of range', { crf: 99 }], + ])('build rejects %s', (_label, overrides) => { + expect(() => + tool.build({ inputPath, outputDir, ...overrides } as never), + ).toThrow(); + }); +}); diff --git a/packages/core/src/omni/policy/tools/downscale-video.ts b/packages/core/src/omni/policy/tools/downscale-video.ts new file mode 100644 index 00000000000..b6c2f09df11 --- /dev/null +++ b/packages/core/src/omni/policy/tools/downscale-video.ts @@ -0,0 +1,278 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import fs from 'node:fs/promises'; +import path from 'node:path'; +import type { + MediaPolicyToolDescriptor, + ToolInvocation, + ToolResult, +} from '../../../tools/tools.js'; +import { Kind } from '../../../tools/tools.js'; +import { ToolNames } from '../../../tools/tool-names.js'; +import { probeMediaMetadata, runFfmpeg } from '../../ffmpeg.js'; +import { + assertMediaPolicyIo, + BaseMediaPolicyTool, + ffmpegFailureMessage, + BaseMediaPolicyToolInvocation, + formatBytesShort, + MEDIA_POLICY_IO_SCHEMA_PROPERTIES, + mediaPolicyToolError, + mediaPolicyToolFailure, + mediaPolicyToolSuccess, + resolvePolicyToolTimeoutMs, + createPolicyToolTimeoutBudget, + type MediaPolicyIoParams, + type MediaPolicyToolConfigView, +} from './media-policy-tool.js'; + +export const OMNI_DOWNSCALE_VIDEO_TOOL_NAME = ToolNames.OMNI_DOWNSCALE_VIDEO; + +/** Fixed-call default parameters (mapping doc §6). */ +export const DOWNSCALE_VIDEO_DEFAULTS = { + maxHeight: 480, + fps: 10, + crf: 28, + preset: 'veryfast', +} as const; + +const OUTPUT_FILE_NAME = 'downscaled.mp4'; + +/** x264 presets accepted by the `preset` tunable. */ +const X264_PRESETS = [ + 'ultrafast', + 'superfast', + 'veryfast', + 'faster', + 'fast', + 'medium', + 'slow', + 'slower', + 'veryslow', +] as const; + +export interface DownscaleVideoParams extends MediaPolicyIoParams { + /** Output height ceiling in pixels (width follows aspect ratio). */ + maxHeight?: number; + /** Output frame rate. */ + fps?: number; + /** x264 constant rate factor (higher = smaller/lossier). */ + crf?: number; + /** x264 encoding preset. */ + preset?: string; +} + +const TUNABLE_SCHEMA_PROPERTIES = { + maxHeight: { + type: 'number', + description: + 'Output height ceiling in pixels (width follows aspect ratio). Default 480.', + minimum: 2, + }, + fps: { + type: 'number', + description: + 'Output frame rate. Fractional rates (e.g. 0.5 = one frame every 2s) are supported. Default 10.', + // Fractional floor: the server-side billing of omni video is per + // SAMPLED FRAME, so sub-1fps rates are the effective degradation + // lever for long clips (reactive server-limit fallback ladder). + minimum: 0.01, + }, + crf: { + type: 'number', + description: + 'x264 constant rate factor, 0-51 (higher = smaller/lossier). Default 28.', + minimum: 0, + maximum: 51, + }, + preset: { + type: 'string', + description: 'x264 encoding preset. Default "veryfast".', + enum: [...X264_PRESETS], + }, +} as const; + +const DESCRIPTOR: MediaPolicyToolDescriptor = { + kind: 'media_policy', + version: '1', + inputMediaTypes: ['video'], + outputs: [ + { + kind: 'media', + mimeTypes: ['video/mp4'], + required: true, + lossy: true, + }, + { kind: 'text', role: 'disclosure', required: true }, + ], + settingsSchema: { + type: 'object', + properties: TUNABLE_SCHEMA_PROPERTIES, + additionalProperties: false, + }, +}; + +class DownscaleVideoInvocation extends BaseMediaPolicyToolInvocation { + constructor( + params: DownscaleVideoParams, + private readonly timeoutMs: number, + ) { + super(params); + } + + getDescription(): string { + const maxHeight = + this.params.maxHeight ?? DOWNSCALE_VIDEO_DEFAULTS.maxHeight; + return `Downscale ${path.basename(this.params.inputPath)} to ${maxHeight}p`; + } + + async execute(signal: AbortSignal): Promise { + const maxHeight = + this.params.maxHeight ?? DOWNSCALE_VIDEO_DEFAULTS.maxHeight; + const fps = this.params.fps ?? DOWNSCALE_VIDEO_DEFAULTS.fps; + const crf = this.params.crf ?? DOWNSCALE_VIDEO_DEFAULTS.crf; + const preset = this.params.preset ?? DOWNSCALE_VIDEO_DEFAULTS.preset; + try { + const { inputSizeBytes } = await assertMediaPolicyIo(this.params); + const probe = await probeMediaMetadata( + this.params.inputPath, + 'video', + signal, + ); + if (probe.height === undefined) { + return mediaPolicyToolError( + `could not determine video height of ${path.basename(this.params.inputPath)}`, + ); + } + + // Target height computed in JS from the probe (not an ffmpeg scale + // expression — expression commas need filtergraph escaping and are + // easy to get subtly wrong): never upscale, and round down to even + // because libx264 requires even dimensions. `scale=-2:h` rounds the + // width to even automatically. + const targetHeight = Math.max( + 2, + Math.floor(Math.min(maxHeight, probe.height) / 2) * 2, + ); + const outputPath = path.join(this.params.outputDir, OUTPUT_FILE_NAME); + const argsFor = (audio: string[]): string[] => [ + '-y', + '-i', + this.params.inputPath, + '-vf', + `scale=-2:${targetHeight},fps=${fps}`, + '-c:v', + 'libx264', + '-crf', + String(crf), + '-preset', + preset, + ...audio, + outputPath, + ]; + + // Audio: try stream copy first (free); if the source codec cannot be + // muxed into mp4 (e.g. pcm, vorbis) ffmpeg fails fast, and the + // fallback re-encodes to AAC 64k (mapping doc §6: copy→aac 兜底). + // Both passes share ONE wall-clock budget: the fallback gets only + // what the failed copy pass left, keeping the invocation within the + // configured timeout instead of doubling it. + const remainingTimeoutMs = createPolicyToolTimeoutBudget(this.timeoutMs); + let run = await runFfmpeg(argsFor(['-c:a', 'copy']), { + signal, + timeoutMs: remainingTimeoutMs(), + }); + if (signal.aborted) { + return mediaPolicyToolError('video downscaling aborted'); + } + if (run.code !== 0) { + run = await runFfmpeg(argsFor(['-c:a', 'aac', '-b:a', '64k']), { + signal, + timeoutMs: remainingTimeoutMs(), + }); + if (signal.aborted) { + return mediaPolicyToolError('video downscaling aborted'); + } + if (run.code !== 0) { + return mediaPolicyToolError( + ffmpegFailureMessage(run, 'downscaling', this.params.inputPath), + ); + } + } + + const outputSizeBytes = (await fs.stat(outputPath)).size; + const originalRate = + probe.frameRate !== undefined ? Math.round(probe.frameRate) : '?'; + // The loss clause must match the numbers shown next to it: a 360p@8 + // input downscaled for size against the 480p/10fps defaults lowers + // neither dimension — claiming 分辨率与帧率下降 would contradict the + // user-visible before/after figures (D8). + const drops = [ + ...(targetHeight < probe.height ? ['分辨率下降'] : []), + ...(probe.frameRate !== undefined && fps < probe.frameRate + ? ['帧率下降'] + : []), + ]; + const lossClause = + drops.length === 2 + ? '分辨率与帧率下降' + : (drops[0] ?? '重新编码压缩'); + const disclosure = `原 ${probe.height}p${originalRate}/${formatBytesShort(inputSizeBytes)} → ${targetHeight}p${fps}/${formatBytesShort(outputSizeBytes)}，${lossClause}，细节受损`; + + return mediaPolicyToolSuccess({ + outputDir: this.params.outputDir, + outputFileName: OUTPUT_FILE_NAME, + artifactKind: 'video', + title: 'Downscaled video', + mimeType: 'video/mp4', + sizeBytes: outputSizeBytes, + disclosure, + }); + } catch (error) { + return mediaPolicyToolFailure(error); + } + } +} + +/** + * `omni_downscale_video` — lossy video degradation (ffmpeg): scale to a + * height ceiling, drop the frame rate, re-encode with x264 at a fixed CRF; + * audio is stream-copied with an AAC 64k fallback (mapping doc §6). + */ +export class OmniDownscaleVideoTool extends BaseMediaPolicyTool { + constructor(config: MediaPolicyToolConfigView) { + super( + OMNI_DOWNSCALE_VIDEO_TOOL_NAME, + 'DownscaleVideo', + 'Downscales a video to a maximum height and frame rate and re-encodes it, producing a smaller lossy derivative with a disclosure of the degradation.', + Kind.Other, + { + type: 'object', + properties: { + ...MEDIA_POLICY_IO_SCHEMA_PROPERTIES, + ...TUNABLE_SCHEMA_PROPERTIES, + }, + required: ['inputPath', 'outputDir'], + additionalProperties: false, + }, + config, + ); + } + + override get mediaPolicyDescriptor(): MediaPolicyToolDescriptor { + return DESCRIPTOR; + } + + protected createInvocation( + params: DownscaleVideoParams, + ): ToolInvocation { + return new DownscaleVideoInvocation( + params, + resolvePolicyToolTimeoutMs(this.configView, this.name), + ); + } +} diff --git a/packages/core/src/omni/policy/tools/extract-audio.test.ts b/packages/core/src/omni/policy/tools/extract-audio.test.ts new file mode 100644 index 00000000000..4d320c33b59 --- /dev/null +++ b/packages/core/src/omni/policy/tools/extract-audio.test.ts @@ -0,0 +1,231 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import type { MediaProbeResult } from '../../ffmpeg.js'; +import type { ToolResult } from '../../../tools/tools.js'; +import { DEFAULT_POLICY_TOOL_TIMEOUT_MS } from './media-policy-tool.js'; +import { + EXTRACT_AUDIO_DEFAULTS, + OMNI_EXTRACT_AUDIO_TOOL_NAME, + OmniExtractAudioTool, +} from './extract-audio.js'; + +const mocks = vi.hoisted(() => ({ + probeMediaMetadata: vi.fn(), + runFfmpeg: vi.fn(), +})); + +vi.mock('../../ffmpeg.js', () => ({ + probeMediaMetadata: mocks.probeMediaMetadata, + runFfmpeg: mocks.runFfmpeg, +})); + +const INPUT_SIZE = 2 * 1024 ** 2; // "2MB" +const OUTPUT_SIZE = 500 * 1024; // "500KB" + +describe('OmniExtractAudioTool', () => { + let root: string; + let inputPath: string; + let outputDir: string; + + const tool = new OmniExtractAudioTool({}); + + const probe = (result: Partial): void => { + mocks.probeMediaMetadata.mockResolvedValue(result as MediaProbeResult); + }; + + const ffmpegSucceeds = (): void => { + mocks.runFfmpeg.mockImplementation(async (args: string[]) => { + await fs.writeFile(args[args.length - 1], Buffer.alloc(OUTPUT_SIZE)); + return { code: 0, stderr: '' }; + }); + }; + + const run = async ( + params: Record = {}, + ): Promise<{ result: ToolResult; signal: AbortSignal }> => { + const invocation = tool.build({ + inputPath, + outputDir, + ...params, + } as never); + const signal = new AbortController().signal; + return { result: await invocation.execute(signal), signal }; + }; + + beforeEach(async () => { + vi.clearAllMocks(); + root = await fs.mkdtemp(path.join(os.tmpdir(), 'omni-xa-')); + inputPath = path.join(root, 'clip.mp4'); + await fs.writeFile(inputPath, Buffer.alloc(INPUT_SIZE)); + outputDir = path.join(root, 'staging'); + await fs.mkdir(outputDir); + ffmpegSucceeds(); + }); + + afterEach(async () => { + await fs.rm(root, { recursive: true, force: true }); + }); + + it('declares the media-policy descriptor and defaults', () => { + expect(tool.name).toBe(OMNI_EXTRACT_AUDIO_TOOL_NAME); + expect(tool.mediaPolicyDescriptor).toEqual({ + kind: 'media_policy', + version: '1', + inputMediaTypes: ['video'], + outputs: [ + { + kind: 'media', + mimeTypes: ['audio/wav', 'audio/mpeg', 'audio/mp4'], + required: true, + lossy: true, + }, + { kind: 'text', role: 'disclosure', required: true }, + ], + settingsSchema: expect.objectContaining({ type: 'object' }), + }); + expect(EXTRACT_AUDIO_DEFAULTS).toEqual({ + format: 'wav', + sampleRateHz: 16_000, + channels: 1, + bitrateKbps: 64, + }); + }); + + it('extracts a 16kHz mono WAV by default (ASR-recommended shape)', async () => { + probe({ durationMs: 63_000 }); + const { result, signal } = await run(); + + expect(mocks.probeMediaMetadata).toHaveBeenCalledWith( + inputPath, + 'video', + signal, + ); + const outputPath = path.join(outputDir, 'extracted.wav'); + expect(mocks.runFfmpeg).toHaveBeenCalledWith( + [ + '-y', + '-i', + inputPath, + '-vn', + '-c:a', + 'pcm_s16le', + '-ar', + '16000', + '-ac', + '1', + outputPath, + ], + { signal, timeoutMs: DEFAULT_POLICY_TOOL_TIMEOUT_MS }, + ); + + expect(result.error).toBeUndefined(); + expect(result.artifacts).toEqual([ + { + kind: 'audio', + storage: 'workspace', + title: 'Extracted audio track', + workspacePath: 'extracted.wav', + mimeType: 'audio/wav', + sizeBytes: OUTPUT_SIZE, + metadata: { + omniDisclosure: + '原视频 63s/2MB → 音轨 WAV/16kHz 单声道，视觉信息全部丢弃', + }, + }, + ]); + }); + + it('encodes MP3 with the bit rate when format=mp3', async () => { + probe({ durationMs: 63_000 }); + const { result } = await run({ format: 'mp3', bitrateKbps: 128 }); + const args = mocks.runFfmpeg.mock.calls[0][0] as string[]; + expect(args.join(' ')).toContain('-c:a libmp3lame -b:a 128k'); + expect(result.artifacts?.[0]).toMatchObject({ + workspacePath: 'extracted.mp3', + mimeType: 'audio/mpeg', + }); + expect(result.artifacts?.[0]?.metadata?.['omniDisclosure']).toContain( + '音轨 MP3/', + ); + }); + + it('encodes AAC in m4a when format=m4a', async () => { + probe({ durationMs: 63_000 }); + const { result } = await run({ format: 'm4a' }); + const args = mocks.runFfmpeg.mock.calls[0][0] as string[]; + expect(args.join(' ')).toContain('-c:a aac -b:a 64k'); + expect(result.artifacts?.[0]).toMatchObject({ + workspacePath: 'extracted.m4a', + mimeType: 'audio/mp4', + }); + }); + + it('omits the duration from the disclosure when the probe lacks it', async () => { + probe({}); + const { result } = await run(); + expect(result.artifacts?.[0]?.metadata?.['omniDisclosure']).toBe( + '原视频 2MB → 音轨 WAV/16kHz 单声道，视觉信息全部丢弃', + ); + }); + + it('reports the ffmpeg error (e.g. a video without an audio stream)', async () => { + probe({ durationMs: 63_000 }); + mocks.runFfmpeg.mockResolvedValue({ + code: 1, + stderr: 'Output file #0 does not contain any stream', + }); + const { result } = await run(); + expect(result.error?.message).toMatch(/ffmpeg failed \(exit 1\)/); + expect(result.error?.message).toContain('does not contain any stream'); + expect(result.artifacts).toBeUndefined(); + }); + + it('reports an aborted run', async () => { + probe({ durationMs: 63_000 }); + const controller = new AbortController(); + mocks.runFfmpeg.mockImplementation(async () => { + controller.abort(); + return { code: 1, stderr: '' }; + }); + const invocation = tool.build({ inputPath, outputDir }); + const result = await invocation.execute(controller.signal); + expect(result.error?.message).toBe('audio extraction aborted'); + }); + + it('threads policyTools..runtime.timeoutMs into runFfmpeg', async () => { + probe({ durationMs: 63_000 }); + const configured = new OmniExtractAudioTool({ + getOmniPolicyToolsSettings: () => ({ + [OMNI_EXTRACT_AUDIO_TOOL_NAME]: { + runtime: { timeoutMs: 45_000 }, + }, + }), + }); + const invocation = configured.build({ inputPath, outputDir }); + await invocation.execute(new AbortController().signal); + expect(mocks.runFfmpeg).toHaveBeenCalledWith( + expect.any(Array), + expect.objectContaining({ timeoutMs: 45_000 }), + ); + }); + + it.each([ + ['relative outputDir', { outputDir: 'staging' }], + ['unknown property', { extra: 1 }], + ['unknown format', { format: 'flac' }], + ['sample rate below floor', { sampleRateHz: 4000 }], + ['too many channels', { channels: 6 }], + ])('build rejects %s', (_label, overrides) => { + expect(() => + tool.build({ inputPath, outputDir, ...overrides } as never), + ).toThrow(); + }); +}); diff --git a/packages/core/src/omni/policy/tools/extract-audio.ts b/packages/core/src/omni/policy/tools/extract-audio.ts new file mode 100644 index 00000000000..b60f6a7cf2d --- /dev/null +++ b/packages/core/src/omni/policy/tools/extract-audio.ts @@ -0,0 +1,252 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import fs from 'node:fs/promises'; +import path from 'node:path'; +import type { + MediaPolicyToolDescriptor, + ToolInvocation, + ToolResult, +} from '../../../tools/tools.js'; +import { Kind } from '../../../tools/tools.js'; +import { ToolNames } from '../../../tools/tool-names.js'; +import { probeMediaMetadata, runFfmpeg } from '../../ffmpeg.js'; +import { + assertMediaPolicyIo, + BaseMediaPolicyTool, + ffmpegFailureMessage, + BaseMediaPolicyToolInvocation, + describeChannels, + formatBytesShort, + MEDIA_POLICY_IO_SCHEMA_PROPERTIES, + mediaPolicyToolError, + mediaPolicyToolFailure, + mediaPolicyToolSuccess, + resolvePolicyToolTimeoutMs, + type MediaPolicyIoParams, + type MediaPolicyToolConfigView, +} from './media-policy-tool.js'; + +export const OMNI_EXTRACT_AUDIO_TOOL_NAME = ToolNames.OMNI_EXTRACT_AUDIO; + +/** Fixed-call default parameters (mapping doc §6.1): 16 kHz mono WAV is + * the ASR-recommended input shape, chaining into omni_transcribe_audio. */ +export const EXTRACT_AUDIO_DEFAULTS = { + format: 'wav', + sampleRateHz: 16_000, + channels: 1, + bitrateKbps: 64, +} as const; + +interface OutputFormat { + fileName: string; + mimeType: string; + label: string; + /** Codec args; lossy formats consume the bit rate. */ + codecArgs(bitrateKbps: number): string[]; +} + +const OUTPUT_FORMATS: Record = { + wav: { + fileName: 'extracted.wav', + mimeType: 'audio/wav', + label: 'WAV', + codecArgs: () => ['-c:a', 'pcm_s16le'], + }, + mp3: { + fileName: 'extracted.mp3', + mimeType: 'audio/mpeg', + label: 'MP3', + codecArgs: (kbps) => ['-c:a', 'libmp3lame', '-b:a', `${kbps}k`], + }, + m4a: { + fileName: 'extracted.m4a', + mimeType: 'audio/mp4', + label: 'M4A', + codecArgs: (kbps) => ['-c:a', 'aac', '-b:a', `${kbps}k`], + }, +}; + +export interface ExtractAudioParams extends MediaPolicyIoParams { + /** Output container/codec: wav (PCM), mp3, or m4a (AAC). */ + format?: 'wav' | 'mp3' | 'm4a'; + /** Output sample rate in Hz. */ + sampleRateHz?: number; + /** Output channel count. */ + channels?: number; + /** Output bit rate in kbit/s (mp3/m4a only; wav is PCM). */ + bitrateKbps?: number; +} + +const TUNABLE_SCHEMA_PROPERTIES = { + format: { + type: 'string', + enum: ['wav', 'mp3', 'm4a'], + description: + "Output format: 'wav' (16-bit PCM), 'mp3', or 'm4a' (AAC). Default 'wav'.", + }, + sampleRateHz: { + type: 'number', + description: 'Output sample rate in Hz. Default 16000.', + minimum: 8000, + }, + channels: { + type: 'number', + description: 'Output channel count. Default 1 (mono).', + minimum: 1, + maximum: 2, + }, + bitrateKbps: { + type: 'number', + description: + 'Output bit rate in kbit/s for mp3/m4a (ignored for wav). Default 64.', + minimum: 8, + }, +} as const; + +const DESCRIPTOR: MediaPolicyToolDescriptor = { + kind: 'media_policy', + version: '1', + inputMediaTypes: ['video'], + outputs: [ + { + kind: 'media', + // One spec, three possible containers: the orchestrator matches the + // recognized mime against this list (mapping doc §6.1). + mimeTypes: ['audio/wav', 'audio/mpeg', 'audio/mp4'], + required: true, + lossy: true, + }, + { kind: 'text', role: 'disclosure', required: true }, + ], + settingsSchema: { + type: 'object', + properties: TUNABLE_SCHEMA_PROPERTIES, + additionalProperties: false, + }, +}; + +class ExtractAudioInvocation extends BaseMediaPolicyToolInvocation { + constructor( + params: ExtractAudioParams, + private readonly timeoutMs: number, + ) { + super(params); + } + + getDescription(): string { + const format = this.params.format ?? EXTRACT_AUDIO_DEFAULTS.format; + return `Extract ${format.toUpperCase()} audio track from ${path.basename(this.params.inputPath)}`; + } + + async execute(signal: AbortSignal): Promise { + const format = this.params.format ?? EXTRACT_AUDIO_DEFAULTS.format; + const sampleRateHz = + this.params.sampleRateHz ?? EXTRACT_AUDIO_DEFAULTS.sampleRateHz; + const channels = this.params.channels ?? EXTRACT_AUDIO_DEFAULTS.channels; + const bitrateKbps = + this.params.bitrateKbps ?? EXTRACT_AUDIO_DEFAULTS.bitrateKbps; + const output = OUTPUT_FORMATS[format]; + try { + const { inputSizeBytes } = await assertMediaPolicyIo(this.params); + const probe = await probeMediaMetadata( + this.params.inputPath, + 'video', + signal, + ); + + const outputPath = path.join(this.params.outputDir, output.fileName); + const run = await runFfmpeg( + [ + '-y', + '-i', + this.params.inputPath, + // Drop the video stream entirely — the audio track is the output. + '-vn', + ...output.codecArgs(bitrateKbps), + '-ar', + String(sampleRateHz), + '-ac', + String(channels), + outputPath, + ], + { signal, timeoutMs: this.timeoutMs }, + ); + if (signal.aborted) { + return mediaPolicyToolError('audio extraction aborted'); + } + if (run.code !== 0) { + return mediaPolicyToolError( + ffmpegFailureMessage( + run, + 'extracting audio from', + this.params.inputPath, + ), + ); + } + + const outputSizeBytes = (await fs.stat(outputPath)).size; + const originalDuration = + probe.durationMs !== undefined + ? `${Math.round(probe.durationMs / 1000)}s/` + : ''; + const disclosure = `原视频 ${originalDuration}${formatBytesShort(inputSizeBytes)} → 音轨 ${output.label}/${Math.round(sampleRateHz / 1000)}kHz${describeChannels(channels)}，视觉信息全部丢弃`; + + return mediaPolicyToolSuccess({ + outputDir: this.params.outputDir, + outputFileName: output.fileName, + artifactKind: 'audio', + title: 'Extracted audio track', + mimeType: output.mimeType, + sizeBytes: outputSizeBytes, + disclosure, + }); + } catch (error) { + return mediaPolicyToolFailure(error); + } + } +} + +/** + * `omni_extract_audio` — audio-track extraction from video (ffmpeg): + * drop the video stream and re-encode the audio as WAV/MP3/M4A (mapping + * doc §6.1). The representation change — the visual channel is discarded + * — makes this lossy by definition, so the disclosure obligation applies + * even to the technically-lossless WAV output. + */ +export class OmniExtractAudioTool extends BaseMediaPolicyTool { + constructor(config: MediaPolicyToolConfigView) { + super( + OMNI_EXTRACT_AUDIO_TOOL_NAME, + 'ExtractAudio', + 'Extracts the audio track from a video into WAV/MP3/M4A, discarding the visual stream, with a disclosure of the loss.', + Kind.Other, + { + type: 'object', + properties: { + ...MEDIA_POLICY_IO_SCHEMA_PROPERTIES, + ...TUNABLE_SCHEMA_PROPERTIES, + }, + required: ['inputPath', 'outputDir'], + additionalProperties: false, + }, + config, + ); + } + + override get mediaPolicyDescriptor(): MediaPolicyToolDescriptor { + return DESCRIPTOR; + } + + protected createInvocation( + params: ExtractAudioParams, + ): ToolInvocation { + return new ExtractAudioInvocation( + params, + resolvePolicyToolTimeoutMs(this.configView, this.name), + ); + } +} diff --git a/packages/core/src/omni/policy/tools/extract-keyframes.test.ts b/packages/core/src/omni/policy/tools/extract-keyframes.test.ts new file mode 100644 index 00000000000..8e62ef3b8a1 --- /dev/null +++ b/packages/core/src/omni/policy/tools/extract-keyframes.test.ts @@ -0,0 +1,481 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import type { MediaProbeResult } from '../../ffmpeg.js'; +import type { ToolResult } from '../../../tools/tools.js'; +import { DEFAULT_POLICY_TOOL_TIMEOUT_MS } from './media-policy-tool.js'; +import { + EXTRACT_KEYFRAMES_DEFAULTS, + OMNI_EXTRACT_KEYFRAMES_TOOL_NAME, + OmniExtractKeyframesTool, + parseShowinfoTimestamps, +} from './extract-keyframes.js'; + +const mocks = vi.hoisted(() => ({ + probeMediaMetadata: vi.fn(), + runFfmpeg: vi.fn(), +})); + +vi.mock('../../ffmpeg.js', () => ({ + probeMediaMetadata: mocks.probeMediaMetadata, + runFfmpeg: mocks.runFfmpeg, +})); + +const FRAME_SIZE = 42 * 1024; + +/** Realistic showinfo stderr lines, one per kept frame. */ +const showinfoStderr = (times: number[]): string => + times + .map( + (t, i) => + `[Parsed_showinfo_2 @ 0x600] n:${String(i).padStart(4, ' ')} pts: ${Math.round(t * 12800)} pts_time:${t} pos: 99 fmt:yuvj420p`, + ) + .join('\n'); + +describe('OmniExtractKeyframesTool', () => { + let root: string; + let inputPath: string; + let outputDir: string; + + const tool = new OmniExtractKeyframesTool({}); + + const probe = (result: Partial): void => { + mocks.probeMediaMetadata.mockResolvedValue(result as MediaProbeResult); + }; + + /** One ffmpeg run: writes `count` frames from the output arg (a + * `%04d` pattern on the legacy path, a literal filename on the + * bucketed path — replace() is a no-op there) and emits matching + * showinfo stderr. */ + const framesRun = + (count: number, times: number[]) => + async (args: string[]): Promise<{ code: number; stderr: string }> => { + const pattern = args[args.length - 1]; + for (let i = 1; i <= count; i++) { + await fs.writeFile( + pattern.replace('%04d', String(i).padStart(4, '0')), + Buffer.alloc(FRAME_SIZE), + ); + } + return { code: 0, stderr: showinfoStderr(times) }; + }; + + /** One ffmpeg run that exits cleanly but produces NO output file — + * a bucket window with no scene change above the threshold. */ + const noFrameRun = async (): Promise<{ code: number; stderr: string }> => ({ + code: 0, + stderr: '', + }); + + const run = async ( + params: Record = {}, + ): Promise<{ result: ToolResult; signal: AbortSignal }> => { + const invocation = tool.build({ + inputPath, + outputDir, + ...params, + } as never); + const signal = new AbortController().signal; + return { result: await invocation.execute(signal), signal }; + }; + + beforeEach(async () => { + vi.clearAllMocks(); + root = await fs.mkdtemp(path.join(os.tmpdir(), 'omni-kf-')); + inputPath = path.join(root, 'clip.mp4'); + await fs.writeFile(inputPath, Buffer.alloc(1024)); + outputDir = path.join(root, 'staging'); + await fs.mkdir(outputDir); + }); + + afterEach(async () => { + await fs.rm(root, { recursive: true, force: true }); + }); + + it('declares the media-policy descriptor and defaults', () => { + expect(tool.name).toBe(OMNI_EXTRACT_KEYFRAMES_TOOL_NAME); + expect(tool.mediaPolicyDescriptor).toEqual({ + kind: 'media_policy', + version: '1', + inputMediaTypes: ['video'], + outputs: [ + { + kind: 'media', + mimeTypes: ['image/jpeg'], + required: true, + lossy: true, + }, + { kind: 'text', role: 'disclosure', required: true }, + ], + settingsSchema: expect.objectContaining({ type: 'object' }), + }); + expect(EXTRACT_KEYFRAMES_DEFAULTS).toEqual({ + maxFrames: 8, + sceneThreshold: 0.2, + maxDimension: 768, + }); + }); + + describe('bucketed extraction (known duration, maxFrames > 1)', () => { + it('spreads one frame per equal bucket across the FULL duration', async () => { + probe({ durationMs: 80_000, width: 1920, height: 1080 }); + // Every bucket has a scene change 3.5s into its window. + mocks.runFfmpeg.mockImplementation(framesRun(1, [3.5])); + const { result, signal } = await run({ maxFrames: 4 }); + + // One scene attempt per bucket, no fallbacks needed. + expect(mocks.runFfmpeg).toHaveBeenCalledTimes(4); + expect(mocks.runFfmpeg).toHaveBeenNthCalledWith( + 1, + [ + '-y', + '-ss', + '0.000', + '-t', + '20.000', + '-i', + inputPath, + '-vf', + "select='gt(scene,0.2)'," + + "scale='min(768,iw)':'min(768,ih)':force_original_aspect_ratio=decrease," + + 'showinfo', + '-vsync', + 'vfr', + '-frames:v', + '1', + '-q:v', + '4', + '-update', + '1', + path.join(outputDir, 'keyframe_0001.jpg'), + ], + { signal, timeoutMs: DEFAULT_POLICY_TOOL_TIMEOUT_MS }, + ); + // Buckets seek to 20s, 40s, 60s — coverage reaches the last + // quarter of the video instead of stopping at the first scenes. + const seeks = mocks.runFfmpeg.mock.calls.map( + (call) => (call[0] as string[])[2], + ); + expect(seeks).toEqual(['0.000', '20.000', '40.000', '60.000']); + + expect(result.error).toBeUndefined(); + expect(result.artifacts).toHaveLength(4); + // Absolute timestamp = bucket start + showinfo pts_time (input + // seeking resets pts to ~0 within each window). + expect(result.artifacts?.[3]).toEqual({ + kind: 'image', + storage: 'workspace', + title: 'Keyframe 4/4', + workspacePath: 'keyframe_0004.jpg', + mimeType: 'image/jpeg', + sizeBytes: FRAME_SIZE, + metadata: { + omniDisclosure: + '原视频 80s/1920×1080 → 关键帧 4/4 @ 63.5s，静态抽帧（全片分桶采样），时间连续性丢失', + }, + }); + for (const artifact of result.artifacts ?? []) { + expect(artifact.metadata?.['omniDisclosure']).toContain('全片分桶采样'); + } + }); + + it('caps the per-bucket scene search window at 30s on long videos', async () => { + probe({ durationMs: 4_882_000, width: 1920, height: 804 }); + mocks.runFfmpeg.mockImplementation(framesRun(1, [1])); + await run({ maxFrames: 2 }); + + const first = mocks.runFfmpeg.mock.calls[0][0] as string[]; + const second = mocks.runFfmpeg.mock.calls[1][0] as string[]; + // Bucket = 2441s, but the scene search only decodes 30s of it. + expect(first.slice(1, 5)).toEqual(['-ss', '0.000', '-t', '30.000']); + expect(second.slice(1, 5)).toEqual(['-ss', '2441.000', '-t', '30.000']); + }); + + it('falls back to the bucket midpoint when the window has no scene change', async () => { + probe({ durationMs: 40_000, width: 640, height: 360 }); + mocks.runFfmpeg + .mockImplementationOnce(noFrameRun) // bucket 1: scene attempt → nothing + .mockImplementationOnce(framesRun(1, [])) // bucket 1: midpoint frame + .mockImplementationOnce(framesRun(1, [2])); // bucket 2: scene hit + const { result, signal } = await run({ maxFrames: 2 }); + + expect(mocks.runFfmpeg).toHaveBeenCalledTimes(3); + // Midpoint fallback: plain seek to bucketStart + bucket/2, no + // select filter, same literal output file. + expect(mocks.runFfmpeg).toHaveBeenNthCalledWith( + 2, + [ + '-y', + '-ss', + '10.000', + '-i', + inputPath, + '-vf', + "scale='min(768,iw)':'min(768,ih)':force_original_aspect_ratio=decrease", + '-frames:v', + '1', + '-q:v', + '4', + '-update', + '1', + path.join(outputDir, 'keyframe_0001.jpg'), + ], + { signal, timeoutMs: expect.any(Number) }, + ); + expect(result.error).toBeUndefined(); + expect(result.artifacts).toHaveLength(2); + expect(result.artifacts?.[0]?.metadata?.['omniDisclosure']).toContain( + '关键帧 1/2 @ 10s', + ); + expect(result.artifacts?.[1]?.metadata?.['omniDisclosure']).toContain( + '关键帧 2/2 @ 22s', + ); + }); + + it('tolerates individual bucket failures and keeps the surviving frames', async () => { + probe({ durationMs: 40_000 }); + mocks.runFfmpeg + .mockResolvedValueOnce({ code: 187, stderr: 'scene boom' }) // bucket 1 scene + .mockResolvedValueOnce({ code: 187, stderr: 'midpoint boom' }) // bucket 1 midpoint + .mockImplementationOnce(framesRun(1, [5])); // bucket 2 scene + const { result } = await run({ maxFrames: 2 }); + + expect(result.error).toBeUndefined(); + expect(result.artifacts).toHaveLength(1); + expect(result.artifacts?.[0]?.title).toBe('Keyframe 1/1'); + expect(result.artifacts?.[0]?.metadata?.['omniDisclosure']).toContain( + '@ 25s', + ); + }); + + it('discloses partial bucket coverage when some buckets yield no frame (D8)', async () => { + probe({ durationMs: 40_000, width: 640, height: 360 }); + mocks.runFfmpeg + .mockImplementationOnce(framesRun(1, [2])) // bucket 1: scene hit + .mockImplementationOnce(noFrameRun) // bucket 2: scene attempt → nothing + .mockImplementationOnce(noFrameRun); // bucket 2: midpoint → nothing + const { result } = await run({ maxFrames: 2 }); + + expect(result.error).toBeUndefined(); + expect(result.artifacts).toHaveLength(1); + // The blanket 全片分桶采样 claim would be false here — bucket 2 was + // never sampled, so the note must disclose the actual coverage. + expect(result.artifacts?.[0]?.metadata?.['omniDisclosure']).toContain( + '静态抽帧（全片分桶采样，仅覆盖 1/2 个分桶，其余时段未采样）', + ); + }); + + it('surfaces the last ffmpeg failure when every bucket failed', async () => { + probe({ durationMs: 20_000 }); + mocks.runFfmpeg.mockResolvedValue({ code: 187, stderr: 'boom' }); + const { result } = await run({ maxFrames: 2 }); + expect(result.error?.message).toMatch(/ffmpeg failed \(exit 187\)/); + expect(result.error?.message).toContain('boom'); + expect(result.artifacts).toBeUndefined(); + }); + + it('errors generically when no bucket produced a frame without any ffmpeg failure', async () => { + probe({ durationMs: 20_000 }); + mocks.runFfmpeg.mockImplementation(noFrameRun); + const { result } = await run({ maxFrames: 2 }); + // 2 buckets × (scene attempt + midpoint fallback) + expect(mocks.runFfmpeg).toHaveBeenCalledTimes(4); + expect(result.error?.message).toMatch(/no keyframes could be extracted/); + }); + + it('charges every bucket run against the SAME wall-clock budget', async () => { + probe({ durationMs: 20_000 }); + mocks.runFfmpeg + .mockImplementationOnce(async (args: string[]) => { + // Burn measurable wall-clock time in the first bucket. + await new Promise((r) => setTimeout(r, 50)); + return framesRun(1, [0])(args); + }) + .mockImplementationOnce(framesRun(1, [1])); + const { result } = await run({ maxFrames: 2 }); + + expect(result.error).toBeUndefined(); + expect(mocks.runFfmpeg).toHaveBeenCalledTimes(2); + const firstTimeout = ( + mocks.runFfmpeg.mock.calls[0][1] as { timeoutMs: number } + ).timeoutMs; + const secondTimeout = ( + mocks.runFfmpeg.mock.calls[1][1] as { timeoutMs: number } + ).timeoutMs; + expect(firstTimeout).toBe(DEFAULT_POLICY_TOOL_TIMEOUT_MS); + // The second bucket gets only what the first one left, never a + // fresh full budget (timers never fire early, so ≥40ms is gone). + expect(secondTimeout).toBeLessThanOrEqual( + DEFAULT_POLICY_TOOL_TIMEOUT_MS - 40, + ); + expect(secondTimeout).toBeGreaterThan(0); + }); + + it('stops looping and returns the frames gathered so far when the budget runs out', async () => { + probe({ durationMs: 80_000 }); + const configured = new OmniExtractKeyframesTool({ + getOmniPolicyToolsSettings: () => ({ + [OMNI_EXTRACT_KEYFRAMES_TOOL_NAME]: { + runtime: { timeoutMs: 60 }, + }, + }), + }); + mocks.runFfmpeg.mockImplementation(async (args: string[]) => { + // Outlive the whole 60ms budget inside the first bucket. + await new Promise((r) => setTimeout(r, 90)); + return framesRun(1, [0])(args); + }); + const invocation = configured.build({ inputPath, outputDir }); + const result = await invocation.execute(new AbortController().signal); + + // Buckets 2..8 were never attempted. + expect(mocks.runFfmpeg).toHaveBeenCalledTimes(1); + expect(result.error).toBeUndefined(); + expect(result.artifacts).toHaveLength(1); + }); + + it('threads tunable overrides into every bucket scene attempt', async () => { + probe({ durationMs: 64_000 }); + mocks.runFfmpeg.mockImplementation(framesRun(1, [0])); + await run({ maxFrames: 16, sceneThreshold: 0.5, maxDimension: 512 }); + expect(mocks.runFfmpeg).toHaveBeenCalledTimes(16); + const args = mocks.runFfmpeg.mock.calls[0][0] as string[]; + expect(args.join(' ')).toContain('gt(scene,0.5)'); + expect(args.join(' ')).toContain("'min(512,iw)':'min(512,ih)'"); + }); + + it('threads policyTools..runtime.timeoutMs into runFfmpeg', async () => { + probe({ durationMs: 63_000 }); + mocks.runFfmpeg.mockImplementation(framesRun(1, [0])); + const configured = new OmniExtractKeyframesTool({ + getOmniPolicyToolsSettings: () => ({ + [OMNI_EXTRACT_KEYFRAMES_TOOL_NAME]: { + runtime: { timeoutMs: 90_000 }, + }, + }), + }); + const invocation = configured.build({ inputPath, outputDir }); + await invocation.execute(new AbortController().signal); + expect(mocks.runFfmpeg).toHaveBeenNthCalledWith( + 1, + expect.any(Array), + expect.objectContaining({ timeoutMs: 90_000 }), + ); + }); + + it('reports an aborted run', async () => { + probe({ durationMs: 63_000 }); + const controller = new AbortController(); + mocks.runFfmpeg.mockImplementation(async () => { + controller.abort(); + return { code: 0, stderr: '' }; + }); + const invocation = tool.build({ inputPath, outputDir }); + const result = await invocation.execute(controller.signal); + expect(result.error?.message).toBe('keyframe extraction aborted'); + }); + }); + + describe('single-pass extraction (unknown duration or maxFrames 1)', () => { + it('extracts scene-detected frames in one pass when the duration is unknown', async () => { + probe({ width: 1920, height: 1080 }); + mocks.runFfmpeg.mockImplementation(framesRun(3, [0, 12.4, 47])); + const { result, signal } = await run(); + + expect(mocks.runFfmpeg).toHaveBeenCalledTimes(1); + expect(mocks.runFfmpeg).toHaveBeenCalledWith( + [ + '-y', + '-i', + inputPath, + '-vf', + "select='eq(n,0)+gt(scene,0.2)'," + + "scale='min(768,iw)':'min(768,ih)':force_original_aspect_ratio=decrease," + + 'showinfo', + '-vsync', + 'vfr', + '-frames:v', + '8', + '-q:v', + '4', + path.join(outputDir, 'keyframe_%04d.jpg'), + ], + { signal, timeoutMs: DEFAULT_POLICY_TOOL_TIMEOUT_MS }, + ); + + expect(result.error).toBeUndefined(); + expect(result.artifacts).toHaveLength(3); + const disclosure = result.artifacts?.[1]?.metadata?.['omniDisclosure']; + expect(disclosure).toContain('关键帧 2/3 @ 12.4s'); + expect(disclosure).toContain('静态抽帧，时间连续性丢失'); + expect(disclosure).not.toContain('全片分桶采样'); + }); + + it('uses a single pass when a single frame is all that was asked for', async () => { + probe({ durationMs: 10_000 }); + mocks.runFfmpeg.mockImplementation(framesRun(1, [0])); + const { result } = await run({ maxFrames: 1 }); + expect(mocks.runFfmpeg).toHaveBeenCalledTimes(1); + const args = mocks.runFfmpeg.mock.calls[0][0] as string[]; + expect(args[args.length - 1]).toBe( + path.join(outputDir, 'keyframe_%04d.jpg'), + ); + expect(result.artifacts).toHaveLength(1); + }); + + it('errors when no frames could be extracted at all', async () => { + probe({}); + mocks.runFfmpeg.mockResolvedValue({ code: 0, stderr: '' }); + const { result } = await run(); + expect(result.error?.message).toMatch(/no keyframes could be extracted/); + expect(result.artifacts).toBeUndefined(); + }); + + it('reports the ffmpeg error on a failed scene pass', async () => { + probe({}); + mocks.runFfmpeg.mockResolvedValue({ code: 187, stderr: 'boom' }); + const { result } = await run(); + expect(result.error?.message).toMatch(/ffmpeg failed \(exit 187\)/); + expect(result.error?.message).toContain('boom'); + }); + }); + + it.each([ + ['relative outputDir', { outputDir: 'staging' }], + ['unknown property', { extra: 1 }], + ['maxFrames above cap', { maxFrames: 200 }], + ['sceneThreshold out of range', { sceneThreshold: 1.5 }], + ])('build rejects %s', (_label, overrides) => { + expect(() => + tool.build({ inputPath, outputDir, ...overrides } as never), + ).toThrow(); + }); +}); + +describe('parseShowinfoTimestamps', () => { + it('parses pts_time per frame in output order', () => { + expect(parseShowinfoTimestamps(showinfoStderr([0, 12.4, 47]))).toEqual([ + 0, 12.4, 47, + ]); + }); + + it('ignores unrelated stderr noise', () => { + const stderr = [ + 'frame= 3 fps=0.0 q=4.0 size=N/A', + showinfoStderr([1.5]), + '[out#0/image2 @ 0x600] video:126KiB', + ].join('\n'); + expect(parseShowinfoTimestamps(stderr)).toEqual([1.5]); + }); + + it('returns an empty array when showinfo produced nothing', () => { + expect(parseShowinfoTimestamps('Conversion failed!')).toEqual([]); + }); +}); diff --git a/packages/core/src/omni/policy/tools/extract-keyframes.ts b/packages/core/src/omni/policy/tools/extract-keyframes.ts new file mode 100644 index 00000000000..e3f33dcf293 --- /dev/null +++ b/packages/core/src/omni/policy/tools/extract-keyframes.ts @@ -0,0 +1,523 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import fs from 'node:fs/promises'; +import path from 'node:path'; +import type { + MediaPolicyToolDescriptor, + ToolArtifact, + ToolInvocation, + ToolResult, +} from '../../../tools/tools.js'; +import { Kind } from '../../../tools/tools.js'; +import { ToolNames } from '../../../tools/tool-names.js'; +import { + probeMediaMetadata, + runFfmpeg, + type FfmpegRunResult, +} from '../../ffmpeg.js'; +import { + assertMediaPolicyIo, + BaseMediaPolicyTool, + ffmpegFailureMessage, + BaseMediaPolicyToolInvocation, + formatBytesShort, + MEDIA_POLICY_IO_SCHEMA_PROPERTIES, + mediaPolicyToolError, + mediaPolicyToolFailure, + resolvePolicyToolTimeoutMs, + createPolicyToolTimeoutBudget, + type MediaPolicyIoParams, + type MediaPolicyToolConfigView, +} from './media-policy-tool.js'; + +export const OMNI_EXTRACT_KEYFRAMES_TOOL_NAME = + ToolNames.OMNI_EXTRACT_KEYFRAMES; + +/** Fixed-call default parameters (mapping doc §6.1). */ +export const EXTRACT_KEYFRAMES_DEFAULTS = { + maxFrames: 8, + sceneThreshold: 0.2, + maxDimension: 768, +} as const; + +const FRAME_FILE_PATTERN = /^keyframe_(\d{4})\.jpg$/; + +export interface ExtractKeyframesParams extends MediaPolicyIoParams { + /** Maximum number of frames to extract. */ + maxFrames?: number; + /** Scene-change threshold (0-1) for the scene-detection engine. */ + sceneThreshold?: number; + /** Longest-edge ceiling in pixels for the extracted frames. */ + maxDimension?: number; +} + +const TUNABLE_SCHEMA_PROPERTIES = { + maxFrames: { + type: 'number', + description: 'Maximum number of frames to extract. Default 8.', + minimum: 1, + maximum: 64, + }, + sceneThreshold: { + type: 'number', + description: + 'Scene-change threshold (0-1) for keyframe selection. Default 0.2.', + minimum: 0, + maximum: 1, + }, + maxDimension: { + type: 'number', + description: + 'Longest-edge ceiling in pixels for the extracted frames ' + + '(aspect ratio preserved, never enlarged). Default 768.', + minimum: 16, + }, +} as const; + +const DESCRIPTOR: MediaPolicyToolDescriptor = { + kind: 'media_policy', + version: '1', + inputMediaTypes: ['video'], + outputs: [ + { + kind: 'media', + mimeTypes: ['image/jpeg'], + required: true, + lossy: true, + }, + { kind: 'text', role: 'disclosure', required: true }, + ], + settingsSchema: { + type: 'object', + properties: TUNABLE_SCHEMA_PROPERTIES, + additionalProperties: false, + }, +}; + +/** + * Fit-inside scale expression: longest edge capped at `maxDimension`, + * aspect ratio preserved, small inputs never enlarged (the min() box is + * the input's own size when it is already within the ceiling). + */ +function scaleFilter(maxDimension: number): string { + return ( + `scale='min(${maxDimension},iw)':'min(${maxDimension},ih)'` + + `:force_original_aspect_ratio=decrease` + ); +} + +/** + * Parse per-frame presentation timestamps from ffmpeg's showinfo stderr + * lines (`[Parsed_showinfo…] n: 3 … pts_time:12.4 …`), in output order. + */ +export function parseShowinfoTimestamps(stderr: string): number[] { + const timestamps: number[] = []; + const pattern = /\bn:\s*\d+.*?\bpts_time:(-?[\d.]+)/g; + for (const match of stderr.matchAll(pattern)) { + const t = Number(match[1]); + timestamps.push(Number.isFinite(t) && t >= 0 ? t : NaN); + } + return timestamps; +} + +/** List produced keyframe files in frame order. */ +async function listFrameFiles(outputDir: string): Promise { + const entries = await fs.readdir(outputDir); + return entries.filter((name) => FRAME_FILE_PATTERN.test(name)).sort(); +} + +/** + * Per-bucket scene search window cap in seconds. Bounding the search + * keeps the worst case (no scene changes anywhere — every window decoded + * to its end) at `maxFrames × window` seconds of decoded video instead + * of the full duration. + */ +const SCENE_SEARCH_WINDOW_SECONDS = 30; + +/** One extracted frame with its absolute position on the timeline. */ +interface ExtractedFrame { + fileName: string; + /** Absolute timestamp in seconds (undefined only on the legacy path + * when showinfo produced no usable pts). */ + timeSeconds?: number; +} + +/** Seconds formatted for ffmpeg `-ss`/`-t` args: fixed-point, never + * scientific notation, millisecond precision. */ +function formatSeconds(seconds: number): string { + return seconds.toFixed(3); +} + +async function fileExists(filePath: string): Promise { + try { + await fs.access(filePath); + return true; + } catch { + return false; + } +} + +/** Shared knobs threaded through one extraction run. */ +interface ExtractionContext { + maxFrames: number; + sceneThreshold: number; + maxDimension: number; + remainingTimeoutMs: () => number; + signal: AbortSignal; +} + +class ExtractKeyframesInvocation extends BaseMediaPolicyToolInvocation { + constructor( + params: ExtractKeyframesParams, + private readonly timeoutMs: number, + ) { + super(params); + } + + getDescription(): string { + const maxFrames = + this.params.maxFrames ?? EXTRACT_KEYFRAMES_DEFAULTS.maxFrames; + return `Extract up to ${maxFrames} keyframes from ${path.basename(this.params.inputPath)}`; + } + + async execute(signal: AbortSignal): Promise { + const maxFrames = Math.floor( + this.params.maxFrames ?? EXTRACT_KEYFRAMES_DEFAULTS.maxFrames, + ); + const sceneThreshold = + this.params.sceneThreshold ?? EXTRACT_KEYFRAMES_DEFAULTS.sceneThreshold; + const maxDimension = + this.params.maxDimension ?? EXTRACT_KEYFRAMES_DEFAULTS.maxDimension; + try { + const { inputSizeBytes } = await assertMediaPolicyIo(this.params); + const probe = await probeMediaMetadata( + this.params.inputPath, + 'video', + signal, + ); + const durationSeconds = + probe.durationMs !== undefined && probe.durationMs > 0 + ? probe.durationMs / 1000 + : undefined; + + // ALL ffmpeg passes share ONE wall-clock budget, keeping the + // invocation within the configured timeout no matter how many + // per-bucket runs it takes. + const remainingTimeoutMs = createPolicyToolTimeoutBudget(this.timeoutMs); + const context: ExtractionContext = { + maxFrames, + sceneThreshold, + maxDimension, + remainingTimeoutMs, + signal, + }; + + // Bucketed extraction is the primary path: it spreads the frames + // across the FULL duration instead of stopping at the first + // maxFrames scene changes (which front-loads every frame into the + // opening minutes of a long video). The single-pass path remains + // for unknown duration (no way to place buckets) and single-frame + // requests (one bucket ≡ one pass). + const bucketed = durationSeconds !== undefined && maxFrames > 1; + const extraction = bucketed + ? await this.extractBucketed(context, durationSeconds) + : await this.extractSinglePass(context); + if (!Array.isArray(extraction)) { + return extraction; + } + const frames = extraction; + + const originalDuration = + durationSeconds !== undefined + ? `${Math.round(durationSeconds)}s` + : formatBytesShort(inputSizeBytes); + const originalResolution = + probe.width !== undefined && probe.height !== undefined + ? `/${probe.width}×${probe.height}` + : ''; + // D8: full-duration coverage may only be claimed when every bucket + // actually yielded a frame — an early budget stop or a failed + // bucket leaves unsampled spans, and the model must be told so it + // does not answer questions about footage it never saw. + const samplingNote = bucketed + ? frames.length < maxFrames + ? `静态抽帧（全片分桶采样，仅覆盖 ${frames.length}/${maxFrames} 个分桶，其余时段未采样）` + : '静态抽帧（全片分桶采样）' + : '静态抽帧'; + + const artifacts: ToolArtifact[] = []; + for (const [index, frame] of frames.entries()) { + const sizeBytes = ( + await fs.stat(path.join(this.params.outputDir, frame.fileName)) + ).size; + const t = frame.timeSeconds; + const atTime = + t !== undefined && Number.isFinite(t) + ? ` @ ${Math.round(t * 10) / 10}s` + : ''; + artifacts.push({ + kind: 'image', + storage: 'workspace', + title: `Keyframe ${index + 1}/${frames.length}`, + workspacePath: frame.fileName, + mimeType: 'image/jpeg', + sizeBytes, + metadata: { + omniDisclosure: `原视频 ${originalDuration}${originalResolution} → 关键帧 ${index + 1}/${frames.length}${atTime}，${samplingNote}，时间连续性丢失`, + }, + }); + } + + const summary = `Extracted ${frames.length} keyframe(s) from ${path.basename(this.params.inputPath)} (${originalDuration}${originalResolution})`; + // Not mediaPolicyToolSuccess: that helper encodes the common + // one-artifact contract, while this is the multi-artifact tool — + // every frame is its own artifact with its own disclosure. + return { + llmContent: summary, + returnDisplay: summary, + artifacts, + }; + } catch (error) { + return mediaPolicyToolFailure(error); + } + } + + /** + * Full-duration coverage: split the timeline into `maxFrames` equal + * buckets and extract one frame per bucket — a scene change from the + * bucket's opening window when one exists, the bucket midpoint + * otherwise. Input seeking (`-ss` before `-i`) jumps straight to each + * bucket without decoding the preceding footage; it also resets + * pts to ~0, so the absolute timestamp is bucketStart + showinfo + * pts_time. Individual bucket failures are tolerated (the last one is + * kept for the zero-frames diagnostic); the loop stops early when the + * shared budget is exhausted. + */ + private async extractBucketed( + context: ExtractionContext, + durationSeconds: number, + ): Promise { + const { + maxFrames, + sceneThreshold, + maxDimension, + remainingTimeoutMs, + signal, + } = context; + const bucket = durationSeconds / maxFrames; + const window = Math.min(bucket, SCENE_SEARCH_WINDOW_SECONDS); + const frames: ExtractedFrame[] = []; + let lastFailure: FfmpegRunResult | undefined; + + for (let i = 0; i < maxFrames; i++) { + if (remainingTimeoutMs() <= 1) { + break; + } + const bucketStart = i * bucket; + const fileName = `keyframe_${String(i + 1).padStart(4, '0')}.jpg`; + const outputPath = path.join(this.params.outputDir, fileName); + + // Scene attempt: first scene change within the bucket's opening + // window. `-update 1` lets ffmpeg write a literal (non-pattern) + // image filename; `-frames:v 1` stops the decode at the first hit. + const sceneRun = await runFfmpeg( + [ + '-y', + '-ss', + formatSeconds(bucketStart), + '-t', + formatSeconds(window), + '-i', + this.params.inputPath, + '-vf', + `select='gt(scene,${sceneThreshold})',${scaleFilter(maxDimension)},showinfo`, + '-vsync', + 'vfr', + '-frames:v', + '1', + '-q:v', + '4', + '-update', + '1', + outputPath, + ], + { signal, timeoutMs: remainingTimeoutMs() }, + ); + if (signal.aborted) { + return mediaPolicyToolError('keyframe extraction aborted'); + } + if (sceneRun.code === 0 && (await fileExists(outputPath))) { + const pts = parseShowinfoTimestamps(sceneRun.stderr)[0]; + frames.push({ + fileName, + timeSeconds: + bucketStart + + (pts !== undefined && Number.isFinite(pts) ? pts : window / 2), + }); + continue; + } + if (sceneRun.code !== 0) { + lastFailure = sceneRun; + } + + // Midpoint fallback: no scene change in the window (static or + // slow footage) — take the bucket's midpoint frame instead so the + // bucket still contributes coverage. + const midpoint = bucketStart + bucket / 2; + const midpointRun = await runFfmpeg( + [ + '-y', + '-ss', + formatSeconds(midpoint), + '-i', + this.params.inputPath, + '-vf', + scaleFilter(maxDimension), + '-frames:v', + '1', + '-q:v', + '4', + '-update', + '1', + outputPath, + ], + { signal, timeoutMs: remainingTimeoutMs() }, + ); + if (signal.aborted) { + return mediaPolicyToolError('keyframe extraction aborted'); + } + if (midpointRun.code === 0 && (await fileExists(outputPath))) { + frames.push({ fileName, timeSeconds: midpoint }); + } else if (midpointRun.code !== 0) { + lastFailure = midpointRun; + } + } + + if (frames.length === 0) { + return mediaPolicyToolError( + lastFailure !== undefined + ? ffmpegFailureMessage( + lastFailure, + 'extracting keyframes from', + this.params.inputPath, + ) + : `no keyframes could be extracted from ${path.basename(this.params.inputPath)}`, + ); + } + return frames; + } + + /** + * Single-pass scene detection (legacy path): frame 0 always selected, + * then every frame whose scene score exceeds the threshold, capped at + * maxFrames. Only used when the duration is unknown (buckets cannot + * be placed) or a single frame was requested. showinfo (after select) + * logs one stderr line per KEPT frame with its pts_time — the + * timestamps feed the per-frame disclosures. + */ + private async extractSinglePass( + context: ExtractionContext, + ): Promise { + const { + maxFrames, + sceneThreshold, + maxDimension, + remainingTimeoutMs, + signal, + } = context; + const outputPattern = path.join(this.params.outputDir, 'keyframe_%04d.jpg'); + const scenePass = await runFfmpeg( + [ + '-y', + '-i', + this.params.inputPath, + '-vf', + `select='eq(n,0)+gt(scene,${sceneThreshold})',${scaleFilter(maxDimension)},showinfo`, + '-vsync', + 'vfr', + '-frames:v', + String(maxFrames), + '-q:v', + '4', + outputPattern, + ], + { signal, timeoutMs: remainingTimeoutMs() }, + ); + if (signal.aborted) { + return mediaPolicyToolError('keyframe extraction aborted'); + } + if (scenePass.code !== 0) { + return mediaPolicyToolError( + ffmpegFailureMessage( + scenePass, + 'extracting keyframes from', + this.params.inputPath, + ), + ); + } + + const frameFiles = await listFrameFiles(this.params.outputDir); + if (frameFiles.length === 0) { + return mediaPolicyToolError( + `no keyframes could be extracted from ${path.basename(this.params.inputPath)}`, + ); + } + const timestamps = parseShowinfoTimestamps(scenePass.stderr); + return frameFiles.map((fileName, index) => { + const t = timestamps[index]; + return { + fileName, + timeSeconds: t !== undefined && Number.isFinite(t) ? t : undefined, + }; + }); + } +} + +/** + * `omni_extract_keyframes` — still frames covering the FULL video + * duration (ffmpeg): the timeline is split into `maxFrames` equal + * buckets and each bucket contributes one frame — a scene change from + * its opening window when one exists, its midpoint otherwise — scaled to + * fit `maxDimension`, as JPEG artifacts with per-frame timestamps in the + * disclosure (mapping doc §6.1). Single-pass scene detection remains for + * unknown duration or single-frame requests. This is the multi-artifact + * policy tool — every frame is promoted in one atomic invocation + * transaction. + */ +export class OmniExtractKeyframesTool extends BaseMediaPolicyTool { + constructor(config: MediaPolicyToolConfigView) { + super( + OMNI_EXTRACT_KEYFRAMES_TOOL_NAME, + 'ExtractKeyframes', + 'Extracts representative still frames spread across the full video duration (per-segment scene detection with midpoint fallback), producing JPEG keyframes with per-frame timestamps and a disclosure of the temporal loss.', + Kind.Other, + { + type: 'object', + properties: { + ...MEDIA_POLICY_IO_SCHEMA_PROPERTIES, + ...TUNABLE_SCHEMA_PROPERTIES, + }, + required: ['inputPath', 'outputDir'], + additionalProperties: false, + }, + config, + ); + } + + override get mediaPolicyDescriptor(): MediaPolicyToolDescriptor { + return DESCRIPTOR; + } + + protected createInvocation( + params: ExtractKeyframesParams, + ): ToolInvocation { + return new ExtractKeyframesInvocation( + params, + resolvePolicyToolTimeoutMs(this.configView, this.name), + ); + } +} diff --git a/packages/core/src/omni/policy/tools/media-policy-tool.test.ts b/packages/core/src/omni/policy/tools/media-policy-tool.test.ts new file mode 100644 index 00000000000..a8992397537 --- /dev/null +++ b/packages/core/src/omni/policy/tools/media-policy-tool.test.ts @@ -0,0 +1,330 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import type { MediaPolicyToolDescriptor } from '../../../tools/tools.js'; +import { Kind, type ToolResult } from '../../../tools/tools.js'; +import { + assertMediaPolicyIo, + BaseMediaPolicyTool, + createPolicyToolTimeoutBudget, + DEFAULT_POLICY_TOOL_TIMEOUT_MS, + formatBytesShort, + resolvePolicyToolTimeoutMs, + validateMediaPolicyIoParams, + type MediaPolicyToolConfigView, +} from './media-policy-tool.js'; +import { BaseToolInvocation } from '../../../tools/tools.js'; + +describe('formatBytesShort', () => { + it.each([ + [512, '512B'], + [8_200_000, '7.8MB'], + [943_718, '921.6KB'], + [2 * 1024 ** 3, '2GB'], + [180 * 1024 ** 2, '180MB'], + [1024, '1KB'], + ])('%d → %s', (bytes, expected) => { + expect(formatBytesShort(bytes)).toBe(expected); + }); +}); + +describe('resolvePolicyToolTimeoutMs', () => { + it('defaults to 600s when unset', () => { + expect(resolvePolicyToolTimeoutMs({}, 'omni_downscale_video')).toBe( + DEFAULT_POLICY_TOOL_TIMEOUT_MS, + ); + expect(DEFAULT_POLICY_TOOL_TIMEOUT_MS).toBe(600_000); + }); + + it('reads policyTools..runtime.timeoutMs', () => { + const config = { + getOmniPolicyToolsSettings: () => ({ + omni_downscale_video: { runtime: { timeoutMs: 120_000 } }, + }), + }; + expect(resolvePolicyToolTimeoutMs(config, 'omni_downscale_video')).toBe( + 120_000, + ); + }); + + it.each([ + ['tombstone entry', null], + ['malformed runtime', { runtime: 'fast' }], + ['non-numeric timeout', { runtime: { timeoutMs: 'soon' } }], + ['non-positive timeout', { runtime: { timeoutMs: 0 } }], + ['non-finite timeout', { runtime: { timeoutMs: Infinity } }], + ])('falls back to the default on %s', (_label, entry) => { + const config = { + getOmniPolicyToolsSettings: () => ({ + omni_downscale_video: entry as never, + }), + }; + expect(resolvePolicyToolTimeoutMs(config, 'omni_downscale_video')).toBe( + DEFAULT_POLICY_TOOL_TIMEOUT_MS, + ); + }); +}); + +describe('createPolicyToolTimeoutBudget', () => { + beforeEach(() => { + vi.useFakeTimers(); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it('gives the first pass the full budget and later passes only the remainder', () => { + const remaining = createPolicyToolTimeoutBudget(10_000); + expect(remaining()).toBe(10_000); + vi.advanceTimersByTime(4_000); + expect(remaining()).toBe(6_000); + }); + + it('starts the clock at the first call, not at creation', () => { + const remaining = createPolicyToolTimeoutBudget(10_000); + vi.advanceTimersByTime(5_000); // setup time before the first pass + expect(remaining()).toBe(10_000); + }); + + it('floors an exhausted budget at 1ms so a follow-up pass fails fast', () => { + const remaining = createPolicyToolTimeoutBudget(10_000); + expect(remaining()).toBe(10_000); + vi.advanceTimersByTime(60_000); + expect(remaining()).toBe(1); + }); +}); + +describe('validateMediaPolicyIoParams', () => { + it('accepts absolute paths', () => { + expect( + validateMediaPolicyIoParams({ + inputPath: '/a/in.mp4', + outputDir: '/b/staging', + }), + ).toBeNull(); + }); + + it.each([ + ['relative inputPath', 'in.mp4', '/b', /inputPath must be an absolute/], + ['relative outputDir', '/a/in.mp4', 'out', /outputDir must be an absolute/], + ])('rejects %s', (_label, inputPath, outputDir, pattern) => { + expect(validateMediaPolicyIoParams({ inputPath, outputDir })).toMatch( + pattern, + ); + }); +}); + +describe('assertMediaPolicyIo', () => { + let root: string; + + beforeEach(async () => { + root = await fs.mkdtemp(path.join(os.tmpdir(), 'omni-mp-io-')); + }); + + afterEach(async () => { + await fs.rm(root, { recursive: true, force: true }); + }); + + it('returns the input size for a valid pair', async () => { + const inputPath = path.join(root, 'in.bin'); + await fs.writeFile(inputPath, Buffer.alloc(1234)); + const outputDir = path.join(root, 'staging'); + await fs.mkdir(outputDir); + await expect( + assertMediaPolicyIo({ inputPath, outputDir }), + ).resolves.toEqual({ inputSizeBytes: 1234 }); + }); + + it('rejects a missing input file', async () => { + const outputDir = path.join(root, 'staging'); + await fs.mkdir(outputDir); + await expect( + assertMediaPolicyIo({ inputPath: path.join(root, 'nope'), outputDir }), + ).rejects.toThrow(/input file not found/); + }); + + it('rejects a symlinked input (never reads through a link)', async () => { + const real = path.join(root, 'real.bin'); + await fs.writeFile(real, 'x'); + const link = path.join(root, 'link.bin'); + await fs.symlink(real, link); + const outputDir = path.join(root, 'staging'); + await fs.mkdir(outputDir); + await expect( + assertMediaPolicyIo({ inputPath: link, outputDir }), + ).rejects.toThrow(/not a regular file/); + }); + + it('rejects a missing output directory', async () => { + const inputPath = path.join(root, 'in.bin'); + await fs.writeFile(inputPath, 'x'); + await expect( + assertMediaPolicyIo({ inputPath, outputDir: path.join(root, 'nope') }), + ).rejects.toThrow(/output directory not found/); + }); + + it('rejects a symlinked output directory', async () => { + const inputPath = path.join(root, 'in.bin'); + await fs.writeFile(inputPath, 'x'); + const realDir = path.join(root, 'real-dir'); + await fs.mkdir(realDir); + const linkDir = path.join(root, 'link-dir'); + await fs.symlink(realDir, linkDir); + await expect( + assertMediaPolicyIo({ inputPath, outputDir: linkDir }), + ).rejects.toThrow(/not a real directory/); + }); +}); + +describe('BaseMediaPolicyTool validation', () => { + interface TestParams { + inputPath: string; + outputDir: string; + level?: number; + } + + class NoopInvocation extends BaseToolInvocation { + getDescription(): string { + return 'noop'; + } + async execute(): Promise { + return { llmContent: 'ok', returnDisplay: 'ok' }; + } + } + + class TestPolicyTool extends BaseMediaPolicyTool { + constructor(view: MediaPolicyToolConfigView = {}) { + super( + 'test_policy_tool', + 'TestPolicyTool', + 'test', + Kind.Other, + { + type: 'object', + properties: { + inputPath: { type: 'string' }, + outputDir: { type: 'string' }, + level: { type: 'number', minimum: 1 }, + }, + required: ['inputPath', 'outputDir'], + additionalProperties: false, + }, + view, + ); + } + override get mediaPolicyDescriptor(): MediaPolicyToolDescriptor { + return { + kind: 'media_policy', + inputMediaTypes: ['image'], + outputs: [{ kind: 'media', required: true, lossy: true }], + }; + } + protected override validateToolParamValues( + params: TestParams, + ): string | null { + return validateMediaPolicyIoParams(params); + } + protected createInvocation(params: TestParams): NoopInvocation { + return new NoopInvocation(params); + } + } + + const tool = new TestPolicyTool(); + + it('validates against the NATIVE parameter schema', () => { + expect( + tool.validateToolParams({ + inputPath: '/a/in.png', + outputDir: '/b/staging', + level: 3, + }), + ).toBeNull(); + }); + + it('rejects schema violations (unknown property, missing required)', () => { + expect( + tool.validateToolParams({ + inputPath: '/a/in.png', + outputDir: '/b/staging', + extra: true, + } as never), + ).not.toBeNull(); + expect( + tool.validateToolParams({ inputPath: '/a/in.png' } as never), + ).not.toBeNull(); + }); + + it('runs value validation after schema validation', () => { + expect( + tool.validateToolParams({ inputPath: 'rel.png', outputDir: '/b' }), + ).toMatch(/absolute/); + }); + + it('build throws on invalid params', () => { + expect(() => tool.build({ inputPath: 'rel.png', outputDir: '/b' })).toThrow( + /absolute/, + ); + }); + + describe('model-visible schema projection (decision D6)', () => { + it('declares the native schema unchanged without modelAccess settings', () => { + expect(tool.schema).toEqual({ + name: 'test_policy_tool', + description: 'test', + parametersJsonSchema: { + type: 'object', + properties: { + inputPath: { type: 'string' }, + outputDir: { type: 'string' }, + level: { type: 'number', minimum: 1 }, + }, + required: ['inputPath', 'outputDir'], + additionalProperties: false, + }, + }); + }); + + it('projects the declaration while validation keeps the native schema', () => { + const configured = new TestPolicyTool({ + getOmniPolicyToolsSettings: () => ({ + test_policy_tool: { + modelAccess: { + enabled: true, + description: 'Model-facing description.', + lockedArguments: { inputPath: '/x', outputDir: '/y' }, + parameterSchema: { properties: { level: { maximum: 9 } } }, + }, + }, + }), + }); + // The model sees ONLY the tunable, with the override merged in. + expect(configured.schema).toEqual({ + name: 'test_policy_tool', + description: 'Model-facing description.', + parametersJsonSchema: { + type: 'object', + properties: { level: { type: 'number', minimum: 1, maximum: 9 } }, + required: [], + additionalProperties: false, + }, + }); + // …but the harness-injected io arguments the projection hides must + // remain valid: validation runs on the NATIVE schema (§9.4). + expect( + configured.validateToolParams({ + inputPath: '/a/in.png', + outputDir: '/b/staging', + level: 3, + }), + ).toBeNull(); + }); + }); +}); diff --git a/packages/core/src/omni/policy/tools/media-policy-tool.ts b/packages/core/src/omni/policy/tools/media-policy-tool.ts new file mode 100644 index 00000000000..eca35e67002 --- /dev/null +++ b/packages/core/src/omni/policy/tools/media-policy-tool.ts @@ -0,0 +1,366 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import fs from 'node:fs/promises'; +import path from 'node:path'; +import type { FunctionDeclaration } from '@google/genai'; +import type { + MediaPolicyToolDescriptor, + ToolArtifact, + ToolArtifactKind, + ToolResult, +} from '../../../tools/tools.js'; +import type { Kind } from '../../../tools/tools.js'; +import { + BaseDeclarativeTool, + BaseToolInvocation, +} from '../../../tools/tools.js'; +import type { PermissionDecision } from '../../../permissions/types.js'; +import { ToolErrorType } from '../../../tools/tool-error.js'; +import { getErrorMessage } from '../../../utils/errors.js'; +import { SchemaValidator } from '../../../utils/schemaValidator.js'; +import { projectMediaPolicyToolDeclaration } from '../model-access.js'; +import { isPlainRecord } from '../types.js'; +import type { MediaPolicyToolConfigView } from '../types.js'; + +/** Re-exported for the many tool modules that already import the config + * view from here; the definition lives in types.ts. */ +export type { MediaPolicyToolConfigView }; + +/** Default transcode timeout when `policyTools..runtime.timeoutMs` + * is not configured (mapping doc §6). */ +export const DEFAULT_POLICY_TOOL_TIMEOUT_MS = 600_000; + +/** Parameters every media-policy degradation tool shares: one input file, + * one harness-injected output directory (the invocation's staging dir — + * the tool's ONLY permitted output location). */ +export interface MediaPolicyIoParams { + /** Absolute path of the source media file. */ + inputPath: string; + /** Absolute path of the directory the tool must write into. */ + outputDir: string; +} + +/** JSON-schema fragments for the shared io parameters. */ +export const MEDIA_POLICY_IO_SCHEMA_PROPERTIES = { + inputPath: { + type: 'string', + description: 'Absolute path of the source media file.', + }, + outputDir: { + type: 'string', + description: + 'Absolute path of the directory the output file is written into.', + }, +} as const; + +/** Read `omni.processing.policyTools..runtime.timeoutMs` + * leniently; anything absent or malformed resolves to the default. */ +export function resolvePolicyToolTimeoutMs( + config: MediaPolicyToolConfigView, + toolName: string, +): number { + const entry = config.getOmniPolicyToolsSettings?.()?.[toolName]; + const runtime = + isPlainRecord(entry) && isPlainRecord(entry['runtime']) + ? entry['runtime'] + : undefined; + const timeoutMs = runtime?.['timeoutMs']; + return typeof timeoutMs === 'number' && + Number.isFinite(timeoutMs) && + timeoutMs > 0 + ? timeoutMs + : DEFAULT_POLICY_TOOL_TIMEOUT_MS; +} + +/** + * Read `omni.processing.policyTools..settings` leniently. The + * map is raw settings input (values may be null tombstones or malformed), + * so anything non-conforming reads as "no defaults" rather than throwing + * mid-run. + */ +export function resolvePolicyToolSettings( + config: MediaPolicyToolConfigView, + toolName: string, +): Record { + const entry = config.getOmniPolicyToolsSettings?.()?.[toolName]; + const settings = isPlainRecord(entry) ? entry['settings'] : undefined; + return isPlainRecord(settings) ? settings : {}; +} + +/** + * Shared wall-clock budget for tools that may run MORE than one ffmpeg + * pass (copy→aac audio fallback, scene→uniform sampling fallback): each + * pass receives the time REMAINING, so the invocation's total transcode + * time stays within the configured `runtime.timeoutMs` instead of + * timeoutMs × passes. The first call returns the full budget + * (deterministic — the clock starts at that call, not at construction); + * later calls return what is left, floored at 1ms so runFfmpeg still + * receives a positive timeout and the exhausted pass fails fast. + */ +export function createPolicyToolTimeoutBudget(totalMs: number): () => number { + let deadline: number | undefined; + return () => { + const now = Date.now(); + deadline ??= now + totalMs; + return Math.max(1, deadline - now); + }; +} + +/** + * Convert a runtime.timeoutMs into sharp's whole-second `timeout()` unit, + * rounding up and flooring at 1s so a small-but-positive budget still + * bounds libvips instead of disabling the timeout (sharp treats 0 as + * "no limit"). The ffmpeg tools get the same bound via runFfmpeg's + * process timeout. + */ +export function sharpTimeoutSeconds(timeoutMs: number): number { + return Math.max(1, Math.ceil(timeoutMs / 1000)); +} + +/** + * Base invocation for media-policy tools. These invocations spawn + * ffmpeg/sharp and WRITE files (with overwrite) at the caller-chosen + * `outputDir`, so they are side-effecting: a model-origin call must be + * confirmation-gated like Write/Edit rather than inherit the read-only + * `'allow'` default. The fixed-policy path is unaffected — the scheduler + * skips the permission flow entirely for `fixed_policy` origin, and the + * orchestrator pins `outputDir` to the invocation's staging directory. + */ +export abstract class BaseMediaPolicyToolInvocation< + TParams extends object, +> extends BaseToolInvocation { + override getDefaultPermission(): Promise { + return Promise.resolve('ask'); + } +} + +/** + * Base class for omni media-policy tools (real DeclarativeTools — the + * orchestrator executes them through the ordinary scheduler path, and + * Stage B's modelAccess can open them to the model). + * + * `mediaPolicyDescriptor` is abstract: every subclass MUST declare its + * descriptor — that code-level fact is what the modelAccess gate and the + * orchestrator key off. + */ +export abstract class BaseMediaPolicyTool< + TParams extends MediaPolicyIoParams, +> extends BaseDeclarativeTool { + constructor( + name: string, + displayName: string, + description: string, + kind: Kind, + parameterSchema: unknown, + /** Config view feeding the modelAccess declaration projection and the + * subclasses' timeout/settings resolution; tools constructed without + * one (tests, embedders) declare their native schema unchanged and use + * built-in defaults. */ + protected readonly configView: MediaPolicyToolConfigView = {}, + ) { + super(name, displayName, description, kind, parameterSchema); + } + + abstract override get mediaPolicyDescriptor(): MediaPolicyToolDescriptor; + + /** Memoized projection result, keyed on the settings-object identity it + * was computed from ({@link schema}). */ + private projectedSchema?: { + settings: unknown; + declaration: FunctionDeclaration; + }; + + /** + * Model-visible declaration (decision D6): the single projection point + * every declaration surface reads — the native schema minus + * `modelAccess.lockedArguments` keys, narrowed to + * `modelAccess.parameterSchema` when configured, with the optional + * description override applied. Validation deliberately does NOT use + * this projection (see {@link validateToolParams}). + * + * The projection is pure over (native schema, settings object), and the + * config stores one normalized settings object per initialize() — so the + * result is memoized on that object's identity, and a re-initialize + * (which swaps the object) recomputes naturally. + */ + override get schema(): FunctionDeclaration { + const settings = this.configView.getOmniPolicyToolsSettings?.(); + if (!this.projectedSchema || this.projectedSchema.settings !== settings) { + this.projectedSchema = { + settings, + declaration: projectMediaPolicyToolDeclaration(this.configView, { + name: this.name, + description: this.description, + parametersJsonSchema: this.parameterSchema, + operatorOnlyParams: this.mediaPolicyDescriptor.operatorOnlyParams, + }), + }; + } + return this.projectedSchema.declaration; + } + + /** + * Validate against the tool's NATIVE parameter schema, never the + * model-visible `schema` getter: Stage B's modelAccess projection makes + * `schema` a narrowed view (lockedArguments removed), while validation + * must keep accepting the harness-injected arguments the projection + * hides (policy design §9.4). + */ + override validateToolParams(params: TParams): string | null { + const errors = SchemaValidator.validate(this.parameterSchema, params); + if (errors) { + return errors; + } + return this.validateToolParamValues(params); + } + + /** Every media-policy tool shares the io params; tools with extra + * value-level rules override this and layer them on top. */ + protected override validateToolParamValues(params: TParams): string | null { + return validateMediaPolicyIoParams(params); + } + + /** + * The other half of the Write/Edit permission posture these tools adopt: + * without this override, the AUTO-mode classifier sees the empty-string + * sentinel (`Arguments: {}`) and its path-based block rules can never + * fire on a model-origin call. Project exactly the fields those rules + * key on — the two filesystem paths carry no secrets. + */ + override toAutoClassifierInput(params: TParams): Record { + return { inputPath: params.inputPath, outputDir: params.outputDir }; + } +} + +/** Shared structural validation for the io params (schema has already + * checked types/required-ness). Returns an error message or null. */ +export function validateMediaPolicyIoParams( + params: MediaPolicyIoParams, +): string | null { + if (!path.isAbsolute(params.inputPath)) { + return `inputPath must be an absolute path (got ${JSON.stringify(params.inputPath)})`; + } + if (!path.isAbsolute(params.outputDir)) { + return `outputDir must be an absolute path (got ${JSON.stringify(params.outputDir)})`; + } + return null; +} + +/** + * Execution-time io checks (validateToolParams is synchronous, so + * filesystem state is asserted here): the input must be an existing + * REGULAR file (lstat — a symlink is refused, the tool must never read + * through a link planted in its input position) and the output directory + * an existing real directory. Returns the input size in bytes. + */ +export async function assertMediaPolicyIo( + params: MediaPolicyIoParams, +): Promise<{ inputSizeBytes: number }> { + let inputStat; + try { + inputStat = await fs.lstat(params.inputPath); + } catch { + throw new Error(`input file not found: ${params.inputPath}`); + } + if (!inputStat.isFile()) { + throw new Error(`input is not a regular file: ${params.inputPath}`); + } + let outStat; + try { + outStat = await fs.lstat(params.outputDir); + } catch { + throw new Error(`output directory not found: ${params.outputDir}`); + } + if (!outStat.isDirectory()) { + throw new Error(`output path is not a real directory: ${params.outputDir}`); + } + return { inputSizeBytes: inputStat.size }; +} + +/** Compact human-readable byte count for disclosure texts ("8.2MB", + * "0.9MB", "2GB", "180MB", "512KB"). */ +export function formatBytesShort(bytes: number): string { + const trim = (n: number): string => + (Math.round(n * 10) / 10).toString().replace(/\.0$/, ''); + if (bytes >= 1024 ** 3) return `${trim(bytes / 1024 ** 3)}GB`; + if (bytes >= 1024 ** 2) return `${trim(bytes / 1024 ** 2)}MB`; + if (bytes >= 1024) return `${trim(bytes / 1024)}KB`; + return `${bytes}B`; +} + +/** "立体声" / "单声道" / "N声道" for disclosure texts (leading space so + * an unknown channel count renders as nothing). */ +export function describeChannels(channels: number | undefined): string { + if (channels === undefined) return ''; + if (channels === 1) return ' 单声道'; + if (channels === 2) return ' 立体声'; + return ` ${channels}声道`; +} + +/** Uniform error ToolResult for a failed policy-tool execution. */ +export function mediaPolicyToolError(message: string): ToolResult { + return { + llmContent: `Error: ${message}`, + returnDisplay: message, + error: { message, type: ToolErrorType.EXECUTION_FAILED }, + }; +} + +/** Shared catch-tail: turn whatever a policy tool threw into the uniform + * error ToolResult. */ +export function mediaPolicyToolFailure(error: unknown): ToolResult { + return mediaPolicyToolError(getErrorMessage(error)); +} + +/** Uniform "ffmpeg failed" message: exit code, the action underway, the + * input's basename (never its full path), and the stderr tail. */ +export function ffmpegFailureMessage( + run: { code: number | null; stderr: string }, + action: string, + inputPath: string, +): string { + return `ffmpeg failed (exit ${run.code}) ${action} ${path.basename(inputPath)}: ${run.stderr.slice(-500)}`; +} + +/** + * Successful policy-tool result: a one-line summary for the model-facing + * channel and exactly one lossy media artifact whose + * `metadata.omniDisclosure` carries the disclosure text the orchestrator + * validates and delivers adjacent to the media (decision D8). + */ +export function mediaPolicyToolSuccess(args: { + outputDir: string; + outputFileName: string; + artifactKind: ToolArtifactKind; + title: string; + mimeType: string; + sizeBytes: number; + disclosure: string; + /** `metadata.omniRole` label (e.g. 'transcript' for the §6.2 transcript + * protocol); omitted for plain media derivatives. */ + role?: string; +}): ToolResult { + const artifact: ToolArtifact = { + kind: args.artifactKind, + storage: 'workspace', + title: args.title, + // Relative to the invocation's staging directory — the orchestrator + // resolves and re-validates containment before promotion. + workspacePath: args.outputFileName, + mimeType: args.mimeType, + sizeBytes: args.sizeBytes, + metadata: + args.role === undefined + ? { omniDisclosure: args.disclosure } + : { omniDisclosure: args.disclosure, omniRole: args.role }, + }; + return { + llmContent: `${args.title}: ${args.disclosure}`, + returnDisplay: args.disclosure, + artifacts: [artifact], + }; +} diff --git a/packages/core/src/omni/policy/tools/sharp-module.ts b/packages/core/src/omni/policy/tools/sharp-module.ts new file mode 100644 index 00000000000..9b9a137af1d --- /dev/null +++ b/packages/core/src/omni/policy/tools/sharp-module.ts @@ -0,0 +1,45 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Minimal slice of the sharp module the omni policy tools use, shared by + * `omni_downsample_image` and `omni_convert_image`. + */ +export type SharpModule = (input: string, options?: object) => SharpPipeline; + +export interface SharpPipeline { + rotate(): SharpPipeline; + timeout(options: { seconds: number }): SharpPipeline; + resize(options: { + width: number; + height: number; + fit: 'inside'; + withoutEnlargement: boolean; + }): SharpPipeline; + jpeg(options: { quality: number }): SharpPipeline; + png(): SharpPipeline; + webp(options: { quality: number }): SharpPipeline; + /** Header-derived metadata; `pages` is the frame/page count of + * multi-frame containers (animated GIF/WebP/APNG) — the tools' second, + * ffprobe-independent animated-input gate. */ + metadata(): Promise<{ pages?: number }>; + toFile( + outputPath: string, + ): Promise<{ width: number; height: number; size: number }>; +} + +/** + * Load sharp lazily (decision D9: soft dependency, mirroring the + * image-view.ts convention). A load failure is an EXECUTION failure of + * the calling invocation — onFailure semantics take over — never a + * startup gate. + */ +export async function loadSharp(): Promise { + // sharp is a CJS `export =` module, so the callable is on `.default` + // at runtime even though NodeNext types collapse that namespace away. + return ((await import('sharp')) as unknown as { default: SharpModule }) + .default; +} diff --git a/packages/core/src/omni/policy/tools/transcribe-audio.test.ts b/packages/core/src/omni/policy/tools/transcribe-audio.test.ts new file mode 100644 index 00000000000..18aa9540c3d --- /dev/null +++ b/packages/core/src/omni/policy/tools/transcribe-audio.test.ts @@ -0,0 +1,649 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import type { MediaProbeResult } from '../../ffmpeg.js'; +import type { ToolResult } from '../../../tools/tools.js'; +import type { MediaPolicyToolConfigView } from './media-policy-tool.js'; +import { + OMNI_TRANSCRIBE_AUDIO_TOOL_NAME, + OmniTranscribeAudioTool, + TRANSCRIBE_AUDIO_DEFAULTS, + collapseRepetitionDegeneration, + parseSseTranscript, +} from './transcribe-audio.js'; + +const mocks = vi.hoisted(() => ({ + probeMediaMetadata: vi.fn(), + runFfmpeg: vi.fn(), +})); + +vi.mock('../../ffmpeg.js', () => ({ + probeMediaMetadata: mocks.probeMediaMetadata, + runFfmpeg: mocks.runFfmpeg, +})); + +/** Minimal RIFF/WAVE header so recognition sniffs audio/wav. */ +const WAV_BYTES = Buffer.concat([ + Buffer.from('RIFF'), + Buffer.alloc(4), + Buffer.from('WAVE'), + Buffer.alloc(1024), +]); + +const sse = (...contents: string[]): string => + contents + .map( + (c) => + `data: ${JSON.stringify({ choices: [{ delta: { content: c } }] })}`, + ) + .concat(['data: [DONE]']) + .join('\n\n') + '\n'; + +describe('parseSseTranscript', () => { + it('concatenates delta content across data lines and stops at [DONE]', () => { + const body = + 'data: {"choices":[{"delta":{"content":"你好"}}]}\r\n' + + '\r\n' + + ': keep-alive comment\n' + + 'event: message\n' + + 'data: {"choices":[{"delta":{"content":"，世界"}}]}\n' + + 'data: not-json\n' + + 'data: {"choices":[{"delta":{"content":null}}]}\n' + + 'data: [DONE]\n' + + 'data: {"choices":[{"delta":{"content":"after-done"}}]}\n'; + expect(parseSseTranscript(body)).toBe('你好，世界'); + }); + + it('returns empty string for a body with no content frames', () => { + expect(parseSseTranscript('data: [DONE]\n')).toBe(''); + }); +}); + +describe('collapseRepetitionDegeneration', () => { + it('collapses a degenerated tail to a single copy of the unit', () => { + const { text, degenerated } = collapseRepetitionDegeneration( + 'Vi ses. ' + 'Hej! '.repeat(48), + ); + expect(degenerated).toBe(true); + expect(text).toBe('Vi ses. Hej!'); + }); + + it('collapses multi-char CJK units', () => { + const { text, degenerated } = collapseRepetitionDegeneration( + '你好。' + '再见！'.repeat(20), + ); + expect(degenerated).toBe(true); + expect(text).toBe('你好。再见！'); + }); + + it('leaves normal prose untouched', () => { + const input = '今天天气很好，我们一起去潜水吧。水下三十米，一切都很安静。'; + expect(collapseRepetitionDegeneration(input)).toEqual({ + text: input, + degenerated: false, + }); + }); + + it('does not treat a whitespace run as degeneration', () => { + const input = '转写结束' + ' '.repeat(100); + expect(collapseRepetitionDegeneration(input)).toEqual({ + text: input, + degenerated: false, + }); + }); + + it('requires at least 8 repetitions', () => { + // 6-char unit so 7 reps (42 chars) already clear the 24-char span + // floor — this pins the rep threshold itself, not the span floor. + const input = '好的，收到。'.repeat(7); + expect(collapseRepetitionDegeneration(input).degenerated).toBe(false); + const { text, degenerated } = collapseRepetitionDegeneration( + '好的，收到。'.repeat(8), + ); + expect(degenerated).toBe(true); + expect(text).toBe('好的，收到。'); + }); + + it('requires the repeated span to reach 24 chars', () => { + // 8 reps but only a 16-char span — too short to count as a loop. + const input = 'ab'.repeat(8); + expect(collapseRepetitionDegeneration(input).degenerated).toBe(false); + }); + + it('collapses single-char loops once the span is long enough', () => { + const { text, degenerated } = collapseRepetitionDegeneration( + '嗯' + '啊'.repeat(30), + ); + expect(degenerated).toBe(true); + expect(text).toBe('嗯啊'); + }); +}); + +describe('OmniTranscribeAudioTool', () => { + let root: string; + let inputPath: string; + let outputDir: string; + let fetchMock: ReturnType; + + const tool = new OmniTranscribeAudioTool({}); + + const probe = (result: Partial): void => { + mocks.probeMediaMetadata.mockResolvedValue(result as MediaProbeResult); + }; + + const fetchReturnsSse = (...contents: string[]): void => { + fetchMock.mockResolvedValue({ + ok: true, + status: 200, + text: async () => sse(...contents), + }); + }; + + const run = async ( + params: Record = {}, + subject: OmniTranscribeAudioTool = tool, + ): Promise<{ result: ToolResult; signal: AbortSignal }> => { + const invocation = subject.build({ + inputPath, + outputDir, + ...params, + } as never); + const signal = new AbortController().signal; + return { result: await invocation.execute(signal), signal }; + }; + + beforeEach(async () => { + vi.clearAllMocks(); + root = await fs.mkdtemp(path.join(os.tmpdir(), 'omni-ta-')); + inputPath = path.join(root, 'speech.wav'); + await fs.writeFile(inputPath, WAV_BYTES); + outputDir = path.join(root, 'staging'); + await fs.mkdir(outputDir); + probe({ durationMs: 63_000 }); + fetchMock = vi.fn(); + vi.stubGlobal('fetch', fetchMock); + vi.stubEnv('DASHSCOPE_API_KEY', 'test-key-123'); + fetchReturnsSse('你好', '，世界'); + }); + + afterEach(async () => { + vi.unstubAllGlobals(); + vi.unstubAllEnvs(); + await fs.rm(root, { recursive: true, force: true }); + }); + + it('declares the media-policy descriptor and backend defaults', () => { + expect(tool.name).toBe(OMNI_TRANSCRIBE_AUDIO_TOOL_NAME); + expect(tool.mediaPolicyDescriptor).toEqual({ + kind: 'media_policy', + version: '1', + inputMediaTypes: ['audio'], + outputs: [ + { + kind: 'file', + role: 'transcript', + mimeTypes: ['text/plain'], + required: true, + lossy: true, + }, + { kind: 'text', role: 'disclosure', required: true }, + ], + settingsSchema: expect.objectContaining({ type: 'object' }), + // Endpoint + credential selection is operator-only: a gated caller + // choosing both would let injected content exfiltrate arbitrary env + // secrets to an attacker host. + operatorOnlyParams: ['baseUrl', 'apiKeyEnv'], + }); + expect(TRANSCRIBE_AUDIO_DEFAULTS).toEqual({ + model: 'qwen3.5-omni-plus', + baseUrl: 'https://dashscope.aliyuncs.com/compatible-mode/v1', + apiKeyEnv: 'DASHSCOPE_API_KEY', + maxInputBytes: 10 * 1024 * 1024, + chunkSeconds: 180, + }); + }); + + it('transcribes via a streaming chat.completions call and emits the transcript-protocol artifact', async () => { + const { result, signal } = await run(); + + expect(mocks.probeMediaMetadata).toHaveBeenCalledWith( + inputPath, + 'audio', + signal, + ); + expect(fetchMock).toHaveBeenCalledTimes(1); + const [url, init] = fetchMock.mock.calls[0] as [string, RequestInit]; + expect(url).toBe( + 'https://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions', + ); + expect(init.method).toBe('POST'); + expect(init.headers).toEqual({ + Authorization: 'Bearer test-key-123', + 'Content-Type': 'application/json', + }); + const body = JSON.parse(init.body as string); + expect(body).toEqual({ + model: 'qwen3.5-omni-plus', + modalities: ['text'], + stream: true, + messages: [ + { + role: 'user', + content: [ + { + type: 'input_audio', + input_audio: { + data: `data:audio/wav;base64,${WAV_BYTES.toString('base64')}`, + format: 'wav', + }, + }, + { + type: 'text', + text: '请逐字转写这段音频的内容，只输出转写文本，不要添加任何解释。', + }, + ], + }, + ], + }); + + expect(result.error).toBeUndefined(); + const disclosure = + '原 63s 音频 → 转写文本 5 字，语气/音色/非语音信息丢失，识别可能有误'; + expect(result.artifacts).toEqual([ + { + kind: 'file', + storage: 'workspace', + title: 'Audio transcript', + workspacePath: 'transcript.txt', + mimeType: 'text/plain', + sizeBytes: Buffer.byteLength('你好，世界', 'utf-8'), + metadata: { omniDisclosure: disclosure, omniRole: 'transcript' }, + }, + ]); + await expect( + fs.readFile(path.join(outputDir, 'transcript.txt'), 'utf-8'), + ).resolves.toBe('你好，世界'); + }); + + it('appends the language hint to the prompt when provided', async () => { + await run({ language: 'zh' }); + const body = JSON.parse( + (fetchMock.mock.calls[0] as [string, RequestInit])[1].body as string, + ); + expect(body.messages[0].content[1].text).toBe( + '请逐字转写这段音频的内容，只输出转写文本，不要添加任何解释。音频语言：zh。', + ); + }); + + it('falls back to policyTools settings for backend values, with params overriding', async () => { + vi.stubEnv('MY_ASR_KEY', 'settings-key'); + const view: MediaPolicyToolConfigView = { + getOmniPolicyToolsSettings: () => ({ + [OMNI_TRANSCRIBE_AUDIO_TOOL_NAME]: { + settings: { + model: 'settings-model', + baseUrl: 'https://example.com/v1/', + apiKeyEnv: 'MY_ASR_KEY', + maxInputBytes: 5000, + }, + }, + }), + }; + const configured = new OmniTranscribeAudioTool(view); + + await run({}, configured); + let [url, init] = fetchMock.mock.calls[0] as [string, RequestInit]; + // Trailing slash on baseUrl is normalized away. + expect(url).toBe('https://example.com/v1/chat/completions'); + expect((init.headers as Record)['Authorization']).toBe( + 'Bearer settings-key', + ); + expect(JSON.parse(init.body as string).model).toBe('settings-model'); + + fetchMock.mockClear(); + fetchReturnsSse('嗯'); + await run({ model: 'param-model' }, configured); + [url, init] = fetchMock.mock.calls[0] as [string, RequestInit]; + expect(JSON.parse(init.body as string).model).toBe('param-model'); + }); + + it('fails without a network call when the API key env is unset', async () => { + vi.stubEnv('DASHSCOPE_API_KEY', ''); + const { result } = await run(); + expect(result.error?.message).toBe( + 'environment variable DASHSCOPE_API_KEY is not set; transcription is unavailable', + ); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('fails without a network call when the input exceeds maxInputBytes', async () => { + const { result } = await run({ maxInputBytes: 10 }); + expect(result.error?.message).toBe( + `input audio is ${WAV_BYTES.length} bytes, over the 10-byte transcription limit`, + ); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('reports only the HTTP status on a non-2xx response (no body leak)', async () => { + fetchMock.mockResolvedValue({ + ok: false, + status: 429, + text: async () => '{"error":{"message":"secret internal detail"}}', + }); + const { result } = await run(); + expect(result.error?.message).toBe( + 'transcription request failed: HTTP 429', + ); + }); + + it('fails when the stream yields an empty transcript', async () => { + fetchMock.mockResolvedValue({ + ok: true, + status: 200, + text: async () => 'data: [DONE]\n', + }); + const { result } = await run(); + expect(result.error?.message).toBe('transcription returned empty text'); + }); + + it('refuses input whose bytes are not audio', async () => { + // PNG magic — sniffs as image, expected audio. + await fs.writeFile( + inputPath, + Buffer.concat([ + Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), + Buffer.alloc(64), + ]), + ); + const { result } = await run(); + expect(result.error?.message).toContain('sniffs as image'); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('maps an AbortSignal.timeout expiry to a timeout error', async () => { + const view: MediaPolicyToolConfigView = { + getOmniPolicyToolsSettings: () => ({ + [OMNI_TRANSCRIBE_AUDIO_TOOL_NAME]: { + runtime: { timeoutMs: 123 }, + }, + }), + }; + fetchMock.mockRejectedValue( + Object.assign(new Error('The operation timed out'), { + name: 'TimeoutError', + }), + ); + const { result } = await run({}, new OmniTranscribeAudioTool(view)); + expect(result.error?.message).toBe('transcription timed out after 123ms'); + }); + + it('collapses single-shot repetition degeneration and discloses it', async () => { + fetchReturnsSse('你好。', '再见！'.repeat(20)); + const { result } = await run(); + + expect(result.error).toBeUndefined(); + await expect( + fs.readFile(path.join(outputDir, 'transcript.txt'), 'utf-8'), + ).resolves.toBe('你好。再见！'); + expect(result.artifacts?.[0]?.metadata?.['omniDisclosure']).toBe( + '原 63s 音频 → 转写文本 6 字，检测到重复退化已截断，语气/音色/非语音信息丢失，识别可能有误', + ); + }); + + describe('chunked transcription (duration > chunkSeconds)', () => { + /** ffmpeg cut mock: writes the chunk file whose CONTENT is the run's + * `-ss` value, so the fetch mock can tell segments apart by decoding + * the base64 payload it receives. */ + const cutWritesSeekTag = (): void => { + mocks.runFfmpeg.mockImplementation(async (args: string[]) => { + await fs.writeFile(args[args.length - 1], Buffer.from(args[2])); + return { code: 0, stderr: '' }; + }); + }; + + /** Decode the `-ss` seek tag back out of a fetch call's audio payload. */ + const seekTagOf = (init: RequestInit): string => { + const body = JSON.parse(init.body as string); + const dataUri = body.messages[0].content[0].input_audio.data as string; + return Buffer.from(dataUri.split(',')[1], 'base64').toString(); + }; + + beforeEach(() => { + // 400s of audio at the default 180s chunk → 3 segments of 133.333s. + probe({ durationMs: 400_000 }); + cutWritesSeekTag(); + }); + + it('cuts equal 16kHz-mono-AAC segments, transcribes each, and assembles time-labeled lines', async () => { + fetchMock.mockImplementation(async (_url: string, init: RequestInit) => ({ + ok: true, + status: 200, + text: async () => sse(`片段@${seekTagOf(init)}`), + })); + const { result, signal } = await run(); + + // One cut per segment, seeking to the segment start. + expect(mocks.runFfmpeg).toHaveBeenCalledTimes(3); + expect(mocks.runFfmpeg).toHaveBeenNthCalledWith( + 1, + [ + '-y', + '-ss', + '0.000', + '-t', + '133.333', + '-i', + inputPath, + '-vn', + '-c:a', + 'aac', + '-b:a', + '32k', + '-ar', + '16000', + '-ac', + '1', + path.join(outputDir, 'chunk_0001.m4a'), + ], + { signal, timeoutMs: expect.any(Number) }, + ); + const seeks = mocks.runFfmpeg.mock.calls.map( + (call) => (call[0] as string[])[2], + ); + expect(seeks).toEqual(['0.000', '133.333', '266.667']); + + // Every chunk request carries the re-encoded m4a payload. + expect(fetchMock).toHaveBeenCalledTimes(3); + for (const call of fetchMock.mock.calls) { + const body = JSON.parse((call[1] as RequestInit).body as string); + expect(body.messages[0].content[0].input_audio.format).toBe('m4a'); + expect(body.model).toBe('qwen3.5-omni-plus'); + } + + expect(result.error).toBeUndefined(); + await expect( + fs.readFile(path.join(outputDir, 'transcript.txt'), 'utf-8'), + ).resolves.toBe( + '[00:00-02:13] 片段@0.000\n' + + '[02:13-04:27] 片段@133.333\n' + + '[04:27-06:40] 片段@266.667', + ); + const disclosure = result.artifacts?.[0]?.metadata?.['omniDisclosure']; + expect(disclosure).toContain('原 400s 音频 → 分 3 段转写文本'); + expect(disclosure).not.toContain('段失败'); + + // Temporary chunk cuts are cleaned up; only the transcript remains. + await expect(fs.readdir(outputDir)).resolves.toEqual(['transcript.txt']); + }); + + it('uses H:MM:SS ranges for audio of an hour or longer', async () => { + probe({ durationMs: 4_882_000 }); // 81:22 film → 28 segments + fetchReturnsSse('对白'); + const { result } = await run(); + + expect(result.error).toBeUndefined(); + const transcript = await fs.readFile( + path.join(outputDir, 'transcript.txt'), + 'utf-8', + ); + expect(transcript).toContain('[0:00:00-0:02:54] 对白'); + expect(transcript).toContain('[1:18:28-1:21:22] 对白'); + expect(result.artifacts?.[0]?.metadata?.['omniDisclosure']).toContain( + '分 28 段转写文本', + ); + }); + + it('fails closed on an implausible container duration (segment-count ceiling)', async () => { + // 10⁸ claimed seconds → ~555556 segments at 180s. The duration is + // attacker-influenced metadata: without the ceiling this would fan + // out into hundreds of thousands of ffmpeg cuts + API calls. + probe({ durationMs: 100_000_000_000 }); + const { result } = await run(); + + expect(result.error?.message).toMatch(/over the 512-segment ceiling/); + expect(result.error?.message).toMatch(/implausible/); + expect(mocks.runFfmpeg).not.toHaveBeenCalled(); + expect(fetchMock).not.toHaveBeenCalled(); + expect(result.artifacts).toBeUndefined(); + }); + + it('marks individual failed segments inline instead of failing the run', async () => { + fetchMock.mockImplementation(async (_url: string, init: RequestInit) => + seekTagOf(init).startsWith('133') + ? { ok: false, status: 500, text: async () => 'secret detail' } + : { ok: true, status: 200, text: async () => sse('还行') }, + ); + const { result } = await run(); + + expect(result.error).toBeUndefined(); + const transcript = await fs.readFile( + path.join(outputDir, 'transcript.txt'), + 'utf-8', + ); + expect(transcript).toContain('[02:13-04:27] （该段转写失败：HTTP 500）'); + expect(transcript).not.toContain('secret detail'); + expect(result.artifacts?.[0]?.metadata?.['omniDisclosure']).toContain( + '（1 段失败）', + ); + }); + + it('errors only when EVERY segment failed', async () => { + fetchMock.mockResolvedValue({ + ok: false, + status: 500, + text: async () => 'boom', + }); + const { result } = await run(); + expect(result.error?.message).toBe( + 'transcription failed for all 3 segments (last: HTTP 500)', + ); + }); + + it('collapses per-segment repetition degeneration and counts it in the disclosure', async () => { + fetchReturnsSse('大家好。', '再见！'.repeat(20)); + const { result } = await run(); + + expect(result.error).toBeUndefined(); + const transcript = await fs.readFile( + path.join(outputDir, 'transcript.txt'), + 'utf-8', + ); + expect(transcript).toContain('大家好。再见！'); + expect(transcript).not.toContain('再见！再见！'); + expect(result.artifacts?.[0]?.metadata?.['omniDisclosure']).toContain( + '3 段检测到重复退化已截断', + ); + }); + + it('marks segments beyond an exhausted budget instead of starting them', async () => { + probe({ durationMs: 720_000 }); // 4 segments — one more than the pool + const view: MediaPolicyToolConfigView = { + getOmniPolicyToolsSettings: () => ({ + [OMNI_TRANSCRIBE_AUDIO_TOOL_NAME]: { + runtime: { timeoutMs: 60 }, + }, + }), + }; + mocks.runFfmpeg.mockImplementation(async (args: string[]) => { + // Outlive the whole 60ms budget inside the first wave of cuts. + await new Promise((r) => setTimeout(r, 90)); + await fs.writeFile(args[args.length - 1], Buffer.from('audio')); + return { code: 0, stderr: '' }; + }); + fetchReturnsSse('还行'); + const { result } = await run({}, new OmniTranscribeAudioTool(view)); + + // Segment 4 was never cut — its budget was gone before it started. + expect(mocks.runFfmpeg).toHaveBeenCalledTimes(3); + expect(result.error).toBeUndefined(); + const transcript = await fs.readFile( + path.join(outputDir, 'transcript.txt'), + 'utf-8', + ); + expect(transcript).toContain( + '[09:00-12:00] （该段转写失败：时间预算耗尽）', + ); + expect(result.artifacts?.[0]?.metadata?.['omniDisclosure']).toContain( + '（1 段失败）', + ); + }); + + it('marks a failed cut with a short message (no ffmpeg stderr leak)', async () => { + mocks.runFfmpeg + .mockResolvedValueOnce({ code: 187, stderr: 'very long stderr dump' }) + .mockImplementation(async (args: string[]) => { + await fs.writeFile(args[args.length - 1], Buffer.from('audio')); + return { code: 0, stderr: '' }; + }); + fetchReturnsSse('还行'); + const { result } = await run(); + + expect(result.error).toBeUndefined(); + const transcript = await fs.readFile( + path.join(outputDir, 'transcript.txt'), + 'utf-8', + ); + expect(transcript).toContain( + '[00:00-02:13] （该段转写失败：切片失败（ffmpeg exit 187））', + ); + expect(transcript).not.toContain('very long stderr dump'); + }); + }); + + it.each([ + [ + 'relative inputPath', + { inputPath: 'rel/a.wav', outputDir: '/tmp/x' }, + /absolute/, + ], + [ + 'relative outputDir', + { inputPath: '/tmp/a.wav', outputDir: 'staging' }, + /absolute/, + ], + [ + 'unknown parameter', + { inputPath: '/tmp/a.wav', outputDir: '/tmp/x', volume: 2 }, + /additional properties|not allowed/i, + ], + [ + 'maxInputBytes below minimum', + { inputPath: '/tmp/a.wav', outputDir: '/tmp/x', maxInputBytes: 0 }, + /minimum|>= 1/i, + ], + [ + 'chunkSeconds below minimum', + { inputPath: '/tmp/a.wav', outputDir: '/tmp/x', chunkSeconds: 10 }, + /minimum|>= 30/i, + ], + ])('build rejects %s', (_name, params, message) => { + expect(() => tool.build(params as never)).toThrow(message); + }); +}); diff --git a/packages/core/src/omni/policy/tools/transcribe-audio.ts b/packages/core/src/omni/policy/tools/transcribe-audio.ts new file mode 100644 index 00000000000..5cae78d493a --- /dev/null +++ b/packages/core/src/omni/policy/tools/transcribe-audio.ts @@ -0,0 +1,705 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import fs from 'node:fs/promises'; +import path from 'node:path'; +import type { + MediaPolicyToolDescriptor, + ToolInvocation, + ToolResult, +} from '../../../tools/tools.js'; +import { Kind } from '../../../tools/tools.js'; +import { ToolNames } from '../../../tools/tool-names.js'; +import { recognizeMediaFile } from '../../recognition.js'; +import { runFfmpeg } from '../../ffmpeg.js'; +import { + assertMediaPolicyIo, + BaseMediaPolicyTool, + BaseMediaPolicyToolInvocation, + createPolicyToolTimeoutBudget, + MEDIA_POLICY_IO_SCHEMA_PROPERTIES, + mediaPolicyToolError, + mediaPolicyToolFailure, + mediaPolicyToolSuccess, + resolvePolicyToolSettings, + resolvePolicyToolTimeoutMs, + type MediaPolicyIoParams, + type MediaPolicyToolConfigView, +} from './media-policy-tool.js'; + +export const OMNI_TRANSCRIBE_AUDIO_TOOL_NAME = ToolNames.OMNI_TRANSCRIBE_AUDIO; + +/** + * Backend defaults (mapping doc §6.1): the qwen3.5-omni ASR backend over + * the DashScope OpenAI-compatible endpoint. Every value is overridable — + * per call via tool arguments, per deployment via + * `policyTools.omni_transcribe_audio.settings` (the orchestrator merges + * settings underneath fixed-policy arguments; model-origin calls fall + * back to the same settings here). + */ +export const TRANSCRIBE_AUDIO_DEFAULTS = { + model: 'qwen3.5-omni-plus', + baseUrl: 'https://dashscope.aliyuncs.com/compatible-mode/v1', + apiKeyEnv: 'DASHSCOPE_API_KEY', + maxInputBytes: 10 * 1024 * 1024, + chunkSeconds: 180, +} as const; + +/** Output file the transcript artifact is written to (staging-relative). */ +const OUTPUT_FILE_NAME = 'transcript.txt'; + +/** How many chunk transcription requests run concurrently. */ +const CHUNK_CONCURRENCY = 3; + +/** Hard ceiling on chunked-transcription segments. The claimed duration + * comes from container metadata, which a crafted file controls freely: an + * absurd duration must not translate into millions of outcome slots and + * queued ffmpeg cuts. 512 × the 30s chunkSeconds floor ≈ 4h16m — far past + * anything the 10MiB default input cap plausibly holds. */ +const MAX_SEGMENT_COUNT = 512; + +/** Chunk re-encode target: 16kHz mono AAC — small enough that a chunk's + * base64 payload stays far under request limits, and speech-sufficient. */ +const CHUNK_AUDIO_ARGS = [ + '-vn', + '-c:a', + 'aac', + '-b:a', + '32k', + '-ar', + '16000', + '-ac', + '1', +] as const; + +/** Detected MIME → the `input_audio.format` token DashScope expects. */ +const INPUT_AUDIO_FORMATS: Record = { + 'audio/wav': 'wav', + 'audio/mpeg': 'mp3', + 'audio/aac': 'aac', + 'audio/flac': 'flac', + 'audio/ogg': 'ogg', + 'audio/mp4': 'm4a', +}; + +export interface TranscribeAudioParams extends MediaPolicyIoParams { + /** Optional language hint passed to the ASR model (e.g. "zh", "en"). */ + language?: string; + /** ASR model id. */ + model?: string; + /** OpenAI-compatible endpoint base URL. */ + baseUrl?: string; + /** Name of the environment variable holding the API key. */ + apiKeyEnv?: string; + /** Maximum input audio size in bytes. */ + maxInputBytes?: number; + /** Segment length in seconds for chunked transcription of long audio. */ + chunkSeconds?: number; +} + +const TUNABLE_SCHEMA_PROPERTIES = { + language: { + type: 'string', + description: + 'Optional language hint for the transcription (e.g. "zh", "en").', + }, + model: { + type: 'string', + description: "ASR model id. Default 'qwen3.5-omni-plus'.", + }, + baseUrl: { + type: 'string', + description: + 'OpenAI-compatible endpoint base URL the transcription request is sent to. Defaults to the DashScope compatible-mode endpoint.', + }, + apiKeyEnv: { + type: 'string', + description: + "Environment variable holding the API key for the endpoint. Default 'DASHSCOPE_API_KEY'.", + }, + maxInputBytes: { + type: 'number', + description: 'Maximum input audio size in bytes. Default 10485760 (10MiB).', + minimum: 1, + }, + chunkSeconds: { + type: 'number', + description: + 'Audio longer than this is split into segments of this length and each segment is transcribed separately (per-segment time ranges are prefixed to the text). Default 180.', + minimum: 30, + maximum: 1800, + }, +} as const; + +const DESCRIPTOR: MediaPolicyToolDescriptor = { + kind: 'media_policy', + version: '1', + inputMediaTypes: ['audio'], + outputs: [ + { + // Transcript protocol (policy design §6.2): a non-media file + // artifact — strict UTF-8 text/plain with + // `metadata.omniRole: 'transcript'` — delivered as a text Part. + kind: 'file', + role: 'transcript', + mimeTypes: ['text/plain'], + required: true, + // Uniform lossy declaration (mapping doc §6.1): tone, timbre and + // non-speech information are lost, and recognition may err. + lossy: true, + }, + { kind: 'text', role: 'disclosure', required: true }, + ], + settingsSchema: { + type: 'object', + properties: TUNABLE_SCHEMA_PROPERTIES, + additionalProperties: false, + }, + // Endpoint + credential selection must stay operator-controlled: a + // gated caller choosing both `apiKeyEnv` and `baseUrl` could point any + // environment secret (e.g. OPENAI_API_KEY) at an attacker-controlled + // host. They remain configurable via policyTools settings and + // modelAccess default/lockedArguments (operator surfaces), and stay in + // the params schema because fixed-policy `arguments` and settings + // defaults are merged into tool args under + // `additionalProperties: false`. + operatorOnlyParams: ['baseUrl', 'apiKeyEnv'], +}; + +const readString = ( + settings: Record, + key: string, +): string | undefined => { + const value = settings[key]; + return typeof value === 'string' && value.length > 0 ? value : undefined; +}; + +const readNumber = ( + settings: Record, + key: string, +): number | undefined => { + const value = settings[key]; + return typeof value === 'number' && Number.isFinite(value) && value > 0 + ? value + : undefined; +}; + +/** One SSE `data:` chunk of an OpenAI-compatible streaming response. */ +interface StreamChunk { + choices?: Array<{ delta?: { content?: string | null } }>; +} + +/** Concatenate `choices[0].delta.content` across SSE lines. Exported for + * tests. */ +export function parseSseTranscript(body: string): string { + let transcript = ''; + for (const rawLine of body.split(/\r?\n/)) { + const line = rawLine.trim(); + if (!line.startsWith('data:')) continue; + const payload = line.slice('data:'.length).trim(); + if (payload === '[DONE]') break; + let chunk: StreamChunk; + try { + chunk = JSON.parse(payload) as StreamChunk; + } catch { + continue; // tolerate keep-alive/malformed frames + } + const content = chunk.choices?.[0]?.delta?.content; + if (typeof content === 'string') transcript += content; + } + return transcript; +} + +/** Repetition-degeneration thresholds: a transcript tail counts as + * degenerated when one unit (≤64 chars, non-whitespace) repeats at least + * 8 consecutive times spanning at least 24 characters. */ +const REPETITION_MIN_REPS = 8; +const REPETITION_MIN_SPAN = 24; +const REPETITION_MAX_UNIT = 64; + +/** + * Detect and collapse ASR repetition degeneration: long inputs make the + * autoregressive decoder fall into a loop that pads the transcript tail + * with one endlessly repeated token ("Hej! Hej! Hej! …"). When the text + * ends in ≥8 consecutive copies of the same unit spanning ≥24 chars, all + * but the first copy are dropped and the collapse is reported so the + * caller can disclose it. Exported for tests. + */ +export function collapseRepetitionDegeneration(text: string): { + text: string; + degenerated: boolean; +} { + let best: { unitLen: number; reps: number } | undefined; + for (let unitLen = 1; unitLen <= REPETITION_MAX_UNIT; unitLen++) { + if (unitLen * REPETITION_MIN_REPS > text.length) break; + const unit = text.slice(text.length - unitLen); + if (unit.trim().length === 0) continue; // whitespace runs are not loops + let reps = 1; + while ( + (reps + 1) * unitLen <= text.length && + text.startsWith(unit, text.length - (reps + 1) * unitLen) + ) { + reps++; + } + if ( + reps >= REPETITION_MIN_REPS && + reps * unitLen >= REPETITION_MIN_SPAN && + (best === undefined || reps * unitLen > best.reps * best.unitLen) + ) { + best = { unitLen, reps }; + } + } + if (best === undefined) { + return { text, degenerated: false }; + } + return { + text: text.slice(0, text.length - best.unitLen * (best.reps - 1)).trimEnd(), + degenerated: true, + }; +} + +/** `MM:SS` (or `H:MM:SS` when `withHours`) clock label for segment + * ranges in the assembled transcript. */ +function formatClock(totalSeconds: number, withHours: boolean): string { + const s = Math.round(totalSeconds); + const h = Math.floor(s / 3600); + const m = Math.floor((s % 3600) / 60); + const mm = String(m).padStart(2, '0'); + const ss = String(s % 60).padStart(2, '0'); + return withHours ? `${h}:${mm}:${ss}` : `${mm}:${ss}`; +} + +/** Outcome of transcribing one audio segment. */ +interface ChunkOutcome { + text?: string; + failure?: string; + degenerated: boolean; +} + +class TranscribeAudioInvocation extends BaseMediaPolicyToolInvocation { + constructor( + params: TranscribeAudioParams, + private readonly settingsDefaults: Record, + private readonly timeoutMs: number, + ) { + super(params); + } + + getDescription(): string { + return `Transcribe ${path.basename(this.params.inputPath)} to text`; + } + + async execute(signal: AbortSignal): Promise { + const settings = this.settingsDefaults; + const model = + this.params.model ?? + readString(settings, 'model') ?? + TRANSCRIBE_AUDIO_DEFAULTS.model; + const baseUrl = + this.params.baseUrl ?? + readString(settings, 'baseUrl') ?? + TRANSCRIBE_AUDIO_DEFAULTS.baseUrl; + const apiKeyEnv = + this.params.apiKeyEnv ?? + readString(settings, 'apiKeyEnv') ?? + TRANSCRIBE_AUDIO_DEFAULTS.apiKeyEnv; + const maxInputBytes = + this.params.maxInputBytes ?? + readNumber(settings, 'maxInputBytes') ?? + TRANSCRIBE_AUDIO_DEFAULTS.maxInputBytes; + const chunkSeconds = + this.params.chunkSeconds ?? + readNumber(settings, 'chunkSeconds') ?? + TRANSCRIBE_AUDIO_DEFAULTS.chunkSeconds; + const language = this.params.language ?? readString(settings, 'language'); + const prompt = + '请逐字转写这段音频的内容，只输出转写文本，不要添加任何解释。' + + (language ? `音频语言：${language}。` : ''); + + try { + const { inputSizeBytes } = await assertMediaPolicyIo(this.params); + if (inputSizeBytes > maxInputBytes) { + return mediaPolicyToolError( + `input audio is ${inputSizeBytes} bytes, over the ${maxInputBytes}-byte transcription limit`, + ); + } + + const apiKey = process.env[apiKeyEnv]; + if (!apiKey) { + return mediaPolicyToolError( + `environment variable ${apiKeyEnv} is not set; transcription is unavailable`, + ); + } + + // Content recognition (sniff + probe): the detected MIME feeds the + // request's audio format and the probed duration feeds the + // disclosure and the chunking decision. Non-audio input is refused + // here. + const recognized = await recognizeMediaFile(this.params.inputPath, { + expectedModality: 'audio', + signal, + }); + const format = INPUT_AUDIO_FORMATS[recognized.detectedMimeType]; + if (!format) { + return mediaPolicyToolError( + `audio container ${recognized.detectedMimeType} is not supported by ${OMNI_TRANSCRIBE_AUDIO_TOOL_NAME}`, + ); + } + const durationSeconds = + recognized.metadata.durationMs !== undefined && + recognized.metadata.durationMs > 0 + ? recognized.metadata.durationMs / 1000 + : undefined; + + const backend = { model, baseUrl, apiKey, prompt }; + // Long audio degrades single-request ASR twice over: the decoder + // truncates well before the end and falls into repetition loops + // ("Hej!" × 48). Segment count > 1 → chunked transcription: split + // the timeline evenly, transcribe every segment independently, and + // label each with its time range. + const segmentCount = + durationSeconds !== undefined + ? Math.ceil(durationSeconds / chunkSeconds) + : 1; + if (segmentCount > MAX_SEGMENT_COUNT) { + // Fail closed: the duration is attacker-influenced metadata, and + // the size gate above already bounds what REAL audio can be here. + return mediaPolicyToolError( + `container claims ${Math.round(durationSeconds ?? 0)}s of audio (${segmentCount} segments of ${chunkSeconds}s, over the ${MAX_SEGMENT_COUNT}-segment ceiling) — implausible for a ${inputSizeBytes}-byte input`, + ); + } + + let transcript: string; + let degeneratedSegments = 0; + let failedSegments = 0; + + if (durationSeconds !== undefined && segmentCount > 1) { + const chunked = await this.transcribeChunked({ + backend, + durationSeconds, + segmentCount, + signal, + }); + if (!Array.isArray(chunked)) { + return chunked; + } + // Match formatClock's Math.round: a 3599.6s duration rounds to + // 3600 inside the clock label, which without the hours field + // would render as "00:00" instead of "1:00:00". + const withHours = Math.round(durationSeconds) >= 3600; + const lines: string[] = []; + const segmentLength = durationSeconds / segmentCount; + for (const [index, outcome] of chunked.entries()) { + const range = `[${formatClock(index * segmentLength, withHours)}-${formatClock(Math.min((index + 1) * segmentLength, durationSeconds), withHours)}]`; + if (outcome.text !== undefined) { + lines.push(`${range} ${outcome.text}`); + if (outcome.degenerated) degeneratedSegments++; + } else { + lines.push(`${range} （该段转写失败：${outcome.failure}）`); + failedSegments++; + } + } + transcript = lines.join('\n'); + } else { + const bytes = await fs.readFile(this.params.inputPath); + const dataUri = `data:${recognized.detectedMimeType};base64,${bytes.toString('base64')}`; + const response = await this.requestTranscription({ + ...backend, + dataUri, + format, + timeoutMs: this.timeoutMs, + signal, + }); + if (!response.ok) { + return mediaPolicyToolError( + `transcription request failed: ${response.error}`, + ); + } + const collapsed = collapseRepetitionDegeneration(response.text); + transcript = collapsed.text; + if (collapsed.degenerated) degeneratedSegments = 1; + if (!transcript) { + return mediaPolicyToolError('transcription returned empty text'); + } + } + + const outputPath = path.join(this.params.outputDir, OUTPUT_FILE_NAME); + const encoded = Buffer.from(transcript, 'utf-8'); + await fs.writeFile(outputPath, encoded); + + const durationPart = + durationSeconds !== undefined ? `${Math.round(durationSeconds)}s ` : ''; + const segmentPart = + segmentCount > 1 ? `分 ${segmentCount} 段转写文本` : '转写文本'; + const failurePart = + failedSegments > 0 ? `（${failedSegments} 段失败）` : ''; + const degenerationPart = + degeneratedSegments > 0 + ? `，${segmentCount > 1 ? `${degeneratedSegments} 段` : ''}检测到重复退化已截断` + : ''; + const disclosure = `原 ${durationPart}音频 → ${segmentPart} ${[...transcript].length} 字${failurePart}${degenerationPart}，语气/音色/非语音信息丢失，识别可能有误`; + + return mediaPolicyToolSuccess({ + outputDir: this.params.outputDir, + outputFileName: OUTPUT_FILE_NAME, + artifactKind: 'file', + title: 'Audio transcript', + mimeType: 'text/plain', + sizeBytes: encoded.length, + disclosure, + role: 'transcript', + }); + } catch (error) { + if ( + error instanceof Error && + error.name === 'TimeoutError' && + !signal.aborted + ) { + return mediaPolicyToolError( + `transcription timed out after ${this.timeoutMs}ms`, + ); + } + return mediaPolicyToolFailure(error); + } + } + + /** + * Chunked transcription: cut the audio into `segmentCount` equal + * segments (16kHz mono AAC — small payloads, speech-sufficient) and + * transcribe them with bounded concurrency. Individual segment + * failures become inline markers instead of failing the whole run; the + * run only errors when EVERY segment failed. All cuts and requests + * share one wall-clock budget. + */ + private async transcribeChunked(options: { + backend: { model: string; baseUrl: string; apiKey: string; prompt: string }; + durationSeconds: number; + segmentCount: number; + signal: AbortSignal; + }): Promise { + const { backend, durationSeconds, segmentCount, signal } = options; + const segmentLength = durationSeconds / segmentCount; + const remainingTimeoutMs = createPolicyToolTimeoutBudget(this.timeoutMs); + const outcomes: ChunkOutcome[] = new Array(segmentCount); + + let nextIndex = 0; + const worker = async (): Promise => { + while (!signal.aborted) { + const index = nextIndex++; + if (index >= segmentCount) return; + if (remainingTimeoutMs() <= 1) { + outcomes[index] = { failure: '时间预算耗尽', degenerated: false }; + continue; + } + outcomes[index] = await this.transcribeChunk({ + backend, + index, + startSeconds: index * segmentLength, + lengthSeconds: segmentLength, + remainingTimeoutMs, + signal, + }); + } + }; + await Promise.all( + Array.from({ length: Math.min(CHUNK_CONCURRENCY, segmentCount) }, worker), + ); + if (signal.aborted) { + return mediaPolicyToolError('transcription aborted'); + } + + if (outcomes.every((o) => o.text === undefined)) { + const lastFailure = outcomes[outcomes.length - 1]?.failure ?? 'unknown'; + return mediaPolicyToolError( + `transcription failed for all ${segmentCount} segments (last: ${lastFailure})`, + ); + } + return outcomes; + } + + /** Cut one segment with ffmpeg, transcribe it, collapse repetition + * degeneration, and clean the temporary cut up. Never throws for + * per-segment problems — they come back as `failure`. */ + private async transcribeChunk(options: { + backend: { model: string; baseUrl: string; apiKey: string; prompt: string }; + index: number; + startSeconds: number; + lengthSeconds: number; + remainingTimeoutMs: () => number; + signal: AbortSignal; + }): Promise { + const { + backend, + index, + startSeconds, + lengthSeconds, + remainingTimeoutMs, + signal, + } = options; + const chunkPath = path.join( + this.params.outputDir, + `chunk_${String(index + 1).padStart(4, '0')}.m4a`, + ); + try { + const cut = await runFfmpeg( + [ + '-y', + '-ss', + startSeconds.toFixed(3), + '-t', + lengthSeconds.toFixed(3), + '-i', + this.params.inputPath, + ...CHUNK_AUDIO_ARGS, + chunkPath, + ], + { signal, timeoutMs: remainingTimeoutMs() }, + ); + if (signal.aborted) { + return { failure: 'aborted', degenerated: false }; + } + if (cut.code !== 0) { + return { + failure: `切片失败（ffmpeg exit ${cut.code}）`, + degenerated: false, + }; + } + + const bytes = await fs.readFile(chunkPath); + const dataUri = `data:audio/mp4;base64,${bytes.toString('base64')}`; + const response = await this.requestTranscription({ + ...backend, + dataUri, + format: 'm4a', + timeoutMs: remainingTimeoutMs(), + signal, + }); + if (!response.ok) { + return { failure: response.error, degenerated: false }; + } + const collapsed = collapseRepetitionDegeneration(response.text); + if (!collapsed.text) { + return { failure: '返回空文本', degenerated: false }; + } + return { text: collapsed.text, degenerated: collapsed.degenerated }; + } catch (error) { + if (signal.aborted) { + return { failure: 'aborted', degenerated: false }; + } + if (error instanceof Error && error.name === 'TimeoutError') { + return { failure: '请求超时', degenerated: false }; + } + return { + failure: error instanceof Error ? error.message : String(error), + degenerated: false, + }; + } finally { + await fs.rm(chunkPath, { force: true }).catch(() => {}); + } + } + + /** One streaming chat.completions ASR request. DashScope + * compatible-mode omni models only support streaming — stream:true and + * SSE assembly of delta.content (mapping doc §6.1). Non-2xx statuses + * come back as `HTTP ` only: raw upstream bodies must not + * reach model-visible content. */ + private async requestTranscription(options: { + model: string; + baseUrl: string; + apiKey: string; + prompt: string; + dataUri: string; + format: string; + timeoutMs: number; + signal: AbortSignal; + }): Promise<{ ok: true; text: string } | { ok: false; error: string }> { + const requestSignal = AbortSignal.any([ + options.signal, + AbortSignal.timeout(options.timeoutMs), + ]); + const response = await fetch( + `${options.baseUrl.replace(/\/+$/, '')}/chat/completions`, + { + method: 'POST', + headers: { + Authorization: `Bearer ${options.apiKey}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + model: options.model, + modalities: ['text'], + stream: true, + messages: [ + { + role: 'user', + content: [ + { + type: 'input_audio', + input_audio: { + data: options.dataUri, + format: options.format, + }, + }, + { type: 'text', text: options.prompt }, + ], + }, + ], + }), + signal: requestSignal, + }, + ); + if (!response.ok) { + return { ok: false, error: `HTTP ${response.status}` }; + } + return { ok: true, text: parseSseTranscript(await response.text()).trim() }; + } +} + +/** + * `omni_transcribe_audio` — speech-to-text over the qwen3.5-omni ASR + * backend (mapping doc §6.1): OpenAI-compatible chat.completions with an + * `input_audio` content part (base64 data URI), streamed SSE response. + * Audio longer than `chunkSeconds` is split into equal segments that are + * transcribed independently and labeled with their time ranges — a + * single request over long audio truncates early and degenerates into + * repetition loops. Repetition degeneration is detected and collapsed in + * every (segment) transcript. Produces a transcript-protocol file + * artifact (policy design §6.2) plus the mandatory disclosure. + */ +export class OmniTranscribeAudioTool extends BaseMediaPolicyTool { + constructor(config: MediaPolicyToolConfigView = {}) { + super( + OMNI_TRANSCRIBE_AUDIO_TOOL_NAME, + 'TranscribeAudio', + 'Transcribes an audio file to text via the qwen3.5-omni ASR backend (long audio is split into time-labeled segments), discarding tone, timbre and non-speech information, with a disclosure of the loss.', + Kind.Other, + { + type: 'object', + properties: { + ...MEDIA_POLICY_IO_SCHEMA_PROPERTIES, + ...TUNABLE_SCHEMA_PROPERTIES, + }, + required: ['inputPath', 'outputDir'], + additionalProperties: false, + }, + config, + ); + } + + override get mediaPolicyDescriptor(): MediaPolicyToolDescriptor { + return DESCRIPTOR; + } + + protected createInvocation( + params: TranscribeAudioParams, + ): ToolInvocation { + return new TranscribeAudioInvocation( + params, + resolvePolicyToolSettings(this.configView, this.name), + resolvePolicyToolTimeoutMs(this.configView, this.name), + ); + } +} diff --git a/packages/core/src/omni/policy/types.ts b/packages/core/src/omni/policy/types.ts new file mode 100644 index 00000000000..d1df371644e --- /dev/null +++ b/packages/core/src/omni/policy/types.ts @@ -0,0 +1,183 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Omni policy-pipeline protocol types. + * + * The wire-facing pieces live where their consumers already are — + * `ToolExecutionOrigin` / `PolicyArtifactBatch` next to the scheduler + * protocol in core/turn.ts, `MediaPolicyToolDescriptor` next to the tool + * framework in tools/tools.ts — and are re-exported here so omni code can + * import everything policy-related from one place. + */ + +export type { + ToolExecutionOrigin, + PolicyArtifactBatch, +} from '../../core/turn.js'; +export type { + MediaPolicyToolDescriptor, + MediaPolicyToolOutputSpec, +} from '../../tools/tools.js'; +export type { + ComparisonCondition, + ComparisonOperator, + ConditionEvaluation, + ConditionOperand, + FixedPolicyCondition, + FixedPolicyConditionContext, + FixedPolicyField, +} from './conditions.js'; + +import type { FixedPolicyCondition } from './conditions.js'; +import type { OmniModality } from '../recognition.js'; + +/** Provenance labels a fixed policy can match on: `user` = user-attached + * input, `tool` = tool-result media, `policy` = a derivative produced by + * another fixed policy. */ +export type FixedPolicyOrigin = 'user' | 'tool' | 'policy'; + +/** + * One fixed policy AFTER config normalization (policy design §8): every + * field present, defaults applied, structure validated. The orchestrator + * consumes only this shape — raw settings never reach it. + */ +export interface NormalizedFixedPolicy { + /** Unique id (settings key). Ties run records, staging dirs and the + * `fixed_policy` execution origin back to their configuration. */ + id: string; + /** Bigger runs first; ties broken by id (ascending) for determinism. */ + priority: number; + /** Modalities the policy applies to. */ + mediaTypes: OmniModality[]; + /** Resource provenances the policy applies to. */ + origins: FixedPolicyOrigin[]; + /** Optional condition; absent means "always applies". */ + when?: FixedPolicyCondition; + /** What to do when `when` cannot be decided (default: skip). */ + onConditionUnavailable: 'skip' | 'run'; + /** Media-policy tool the policy invokes. */ + toolName: string; + /** Fixed tool arguments (io params are injected per invocation). */ + arguments: Record; + /** Max executions of THIS policy along one derivation chain. */ + maxRunsPerLineage: number; + /** Failure behavior: keep the source in the delivery set and move on, + * or abort the whole media delivery. */ + onFailure: 'continue' | 'abort'; + output: { + /** Whether derivatives re-enter policy matching. */ + reprocessMedia: boolean; + /** Whether the source stays in the delivery set alongside the + * derivatives (`keep`) or is replaced by them (`omit`). */ + source: 'keep' | 'omit'; + /** + * Per-artifact delivery decision (upstream P output.artifacts): + * selector → action. Selectors are `role:` (matches the + * artifact's `metadata.omniRole`), `kind:`, + * or `*`; most-specific wins (role > kind > `*`), and an artifact no + * selector matches is retained (registered but not delivered). + * Defaults to `{'*': 'include'}` when unconfigured — the historical + * "every derivative delivers" behavior. + */ + artifacts: Record; + }; + /** Pipeline stage the policy runs in. Transport-guard policies fail + * closed regardless of `onFailure`. */ + stage: 'preprocessing' | 'transport_guard'; +} + +/** Normalized `omni.processing.limits` — per-root derivation budgets + * (policy design §12.2). Every field concrete after normalization. */ +export interface NormalizedOmniProcessingLimits { + /** Media resources processed by policies in parallel per request. */ + maxConcurrentResources: number; + /** Tokens reserved for model output when computing + * `session.availableContextTokens` for when-conditions. */ + reservedOutputTokens: number; + /** Maximum derivation chain length from a root resource. */ + maxLineageDepth: number; + /** Maximum policy invocations per root within one orchestrator run. */ + maxPolicyRunsPerRoot: number; + /** Maximum derived artifacts per root within one orchestrator run. */ + maxArtifactsPerRoot: number; + /** Byte budget for derived artifacts per root within one run. */ + maxDerivedBytesPerRoot: number; + /** Maximum transport-guard passes per resource before explicit + * omission. */ + maxTransportPasses: number; +} + +/** Normalized `omni.processing` view the pipeline consumes. */ +export interface NormalizedOmniProcessingConfig { + fixedPolicies: NormalizedFixedPolicy[]; + transportGuardPolicies: NormalizedFixedPolicy[]; + limits: NormalizedOmniProcessingLimits; +} + +/** Structural Config view for the processing config accessor (optional so + * stub configs and embedders without omni settings keep working; the real + * accessor lands with config normalization). */ +export interface OmniProcessingConfigView { + getOmniProcessingConfig?: () => NormalizedOmniProcessingConfig | undefined; +} + +/** + * Raw (pre-normalization) shape of one + * `omni.processing.policyTools.` settings entry. Full semantic + * validation happens in the config-normalization pass; these types only + * capture the structure the lenient readers navigate. + */ +export interface OmniPolicyToolModelAccessSettings { + /** Whether the model (and direct client calls) may invoke the tool. + * Default: false — media-policy tools are fixed-policy-only unless + * explicitly opened up. */ + enabled?: boolean; + /** Overrides the tool description the model sees. */ + description?: string; + /** Filled in when the model omits them. */ + defaultArguments?: Record; + /** Harness-injected arguments, hidden from the model's schema; a model + * call that passes any of these keys explicitly is a parameter error. */ + lockedArguments?: Record; + /** Narrowing-only projection over the tool's native schema. */ + parameterSchema?: Record; + /** Artifact behavior for model-origin calls (Stage B). */ + output?: Record; +} + +/** Raw shape of one `omni.processing.policyTools.` entry. */ +export interface OmniPolicyToolSettings { + /** Tool-level settings validated against the descriptor's settingsSchema. */ + settings?: Record; + /** Per-tool runtime limits (timeoutMs). */ + runtime?: Record; + /** Model-callability gate and argument projection. */ + modelAccess?: OmniPolicyToolModelAccessSettings; +} + +/** Raw `omni.processing.policyTools` map as loaded from settings. Values + * may be null (scope-merge tombstones) or malformed — readers must treat + * anything non-conforming as absent (fail closed). */ +export type OmniPolicyToolsSettings = Record< + string, + OmniPolicyToolSettings | null +>; + +/** Structural "is a plain JSON object" check shared by every omni policy + * reader of raw settings input (which may be null tombstones, arrays, or + * scalars — all of which must read as "absent", never throw). */ +export const isPlainRecord = ( + value: unknown, +): value is Record => + typeof value === 'object' && value !== null && !Array.isArray(value); + +/** Minimal structural view of Config used by omni policy tools and the + * modelAccess projection. Optional so partial/stub configs (tests, + * embedders) fall back to defaults / fail closed. */ +export interface MediaPolicyToolConfigView { + getOmniPolicyToolsSettings?: () => OmniPolicyToolsSettings | undefined; +} diff --git a/packages/core/src/omni/reactive-degrade.test.ts b/packages/core/src/omni/reactive-degrade.test.ts new file mode 100644 index 00000000000..305eadbdde8 --- /dev/null +++ b/packages/core/src/omni/reactive-degrade.test.ts @@ -0,0 +1,304 @@ +/** + * @license + * Copyright 2026 Qwen Team + * SPDX-License-Identifier: Apache-2.0 + */ + +import { afterEach, describe, expect, it } from 'vitest'; +import type { Content, Part } from '@google/genai'; +import { ToolNames } from '../tools/tool-names.js'; +import type { NormalizedFixedPolicy } from './policy/types.js'; +import { + applyOssMediaReplacements, + buildLadderPolicy, + collectOssMediaRefs, + contentsHaveOssMedia, + getObservedServerInputLimit, + recordObservedServerInputLimit, + resetObservedServerInputLimitsForTests, + type OssMediaReplacement, +} from './reactive-degrade.js'; +import { OMNI_DISCLOSURE_TEXT_PREFIX } from './disclosure.js'; + +afterEach(() => { + resetObservedServerInputLimitsForTests(); +}); + +function videoGuardPolicy( + overrides?: Partial, +): NormalizedFixedPolicy { + return { + id: 'video-downscale', + priority: 0, + mediaTypes: ['video'], + origins: ['user', 'tool', 'policy'], + onConditionUnavailable: 'skip', + toolName: ToolNames.OMNI_DOWNSCALE_VIDEO, + arguments: {}, + maxRunsPerLineage: 1, + onFailure: 'continue', + output: { + reprocessMedia: false, + source: 'omit', + artifacts: { '*': 'include' }, + }, + stage: 'transport_guard', + ...overrides, + }; +} + +function ossContents(): Content[] { + return [ + { + role: 'user', + parts: [ + { text: 'analyze this' }, + { + fileData: { + fileUri: 'oss://bucket/clip', + mimeType: 'video/mp4', + displayName: 'movie.mkv', + }, + }, + { + fileData: { + fileUri: 'oss://bucket/frame1', + mimeType: 'image/jpeg', + displayName: 'frame1.jpg', + }, + }, + ], + }, + { + role: 'model', + parts: [{ text: 'ok' }], + }, + ]; +} + +/** A tool result carrying media (and a text sibling) on + * `functionResponse.parts` — qwen-code's extension to the \@google/genai + * schema (see `coreToolScheduler.createFunctionResponsePart`), which is + * why the nested array is typed as `Part[]` and cast on assembly. */ +function nestedToolResultPart(): Part { + const nestedParts: Part[] = [ + { text: 'tool output' }, + { + fileData: { + fileUri: 'oss://bucket/nested', + mimeType: 'video/mp4', + displayName: 'nested.mp4', + }, + }, + ]; + const functionResponse = { + name: 'some_tool', + response: {}, + parts: nestedParts, + }; + return { functionResponse } as Part; +} + +describe('collectOssMediaRefs', () => { + it('collects distinct oss:// media parts with display names', () => { + const refs = collectOssMediaRefs(ossContents()); + expect(refs).toEqual([ + { + fileUri: 'oss://bucket/clip', + mimeType: 'video/mp4', + displayName: 'movie.mkv', + }, + { + fileUri: 'oss://bucket/frame1', + mimeType: 'image/jpeg', + displayName: 'frame1.jpg', + }, + ]); + }); + + it('dedups repeated URIs and skips non-oss fileData', () => { + const contents: Content[] = [ + { + role: 'user', + parts: [ + { fileData: { fileUri: 'oss://bucket/a', mimeType: 'video/mp4' } }, + { fileData: { fileUri: 'oss://bucket/a', mimeType: 'video/mp4' } }, + { + fileData: { + fileUri: 'https://example.com/x.mp4', + mimeType: 'video/mp4', + }, + }, + ], + }, + ]; + const refs = collectOssMediaRefs(contents); + expect(refs).toHaveLength(1); + expect(refs[0].fileUri).toBe('oss://bucket/a'); + // No displayName on the part: falls back to the URI basename. + expect(refs[0].displayName).toBe('a'); + }); + + it('contentsHaveOssMedia mirrors the collector', () => { + expect(contentsHaveOssMedia(ossContents())).toBe(true); + expect( + contentsHaveOssMedia([{ role: 'user', parts: [{ text: 'hi' }] }]), + ).toBe(false); + }); + + it('sees media nested in functionResponse.parts (tool-result deliveries)', () => { + const contents: Content[] = [ + { role: 'user', parts: [nestedToolResultPart()] }, + ]; + expect(contentsHaveOssMedia(contents)).toBe(true); + expect(collectOssMediaRefs(contents)).toEqual([ + { + fileUri: 'oss://bucket/nested', + mimeType: 'video/mp4', + displayName: 'nested.mp4', + }, + ]); + }); +}); + +describe('buildLadderPolicy', () => { + it('merges the rung over the configured arguments for the default tool', () => { + const policy = buildLadderPolicy( + videoGuardPolicy({ arguments: { crf: 30 } }), + 'video', + 1, + ); + expect(policy.arguments).toEqual({ crf: 30, maxHeight: 360, fps: 0.5 }); + expect(policy.id).toBe('video-downscale.reactive-1'); + expect(policy.when).toBeUndefined(); + expect(policy.output.source).toBe('omit'); + expect(policy.output.reprocessMedia).toBe(false); + }); + + it('escalates fps down the rungs and clamps past the last rung', () => { + const fpsAt = (attempt: number) => + buildLadderPolicy(videoGuardPolicy(), 'video', attempt).arguments['fps']; + expect(fpsAt(0)).toBe(2); + expect(fpsAt(1)).toBe(0.5); + expect(fpsAt(2)).toBe(0.25); + expect(fpsAt(7)).toBe(0.25); // clamped: upstream no-progress check stops the loop + }); + + it('keeps a custom guard tool untouched (no foreign arguments injected)', () => { + const custom = videoGuardPolicy({ + toolName: ToolNames.OMNI_EXTRACT_KEYFRAMES, + arguments: { maxFrames: 4 }, + }); + const policy = buildLadderPolicy(custom, 'video', 1); + expect(policy.arguments).toEqual({ maxFrames: 4 }); + }); +}); + +describe('applyOssMediaReplacements', () => { + it('swaps fileUri/mimeType in place and inserts the disclosure before the media', () => { + const contents = ossContents(); + const replacements = new Map([ + [ + 'oss://bucket/clip', + { + fileUri: 'oss://bucket/clip-degraded', + mimeType: 'video/mp4', + disclosureText: `${OMNI_DISCLOSURE_TEXT_PREFIX}movie.mkv：降质重试`, + }, + ], + ]); + const replaced = applyOssMediaReplacements(contents, replacements); + expect(replaced).toBe(1); + const parts = contents[0].parts!; + // [text, disclosure, degraded clip, untouched frame] + expect(parts).toHaveLength(4); + expect(parts[1].text).toContain(OMNI_DISCLOSURE_TEXT_PREFIX); + expect(parts[2].fileData?.fileUri).toBe('oss://bucket/clip-degraded'); + expect(parts[2].fileData?.displayName).toBe('movie.mkv'); // preserved + expect(parts[3].fileData?.fileUri).toBe('oss://bucket/frame1'); // untouched + // Model content untouched. + expect(contents[1].parts).toEqual([{ text: 'ok' }]); + }); + + it('replaces every occurrence of the same URI across contents', () => { + const contents: Content[] = [ + { + role: 'user', + parts: [{ fileData: { fileUri: 'oss://bucket/a', mimeType: 'v' } }], + }, + { + role: 'user', + parts: [{ fileData: { fileUri: 'oss://bucket/a', mimeType: 'v' } }], + }, + ]; + const replaced = applyOssMediaReplacements( + contents, + new Map([ + [ + 'oss://bucket/a', + { + fileUri: 'oss://bucket/a2', + mimeType: 'video/mp4', + disclosureText: 'd', + }, + ], + ]), + ); + expect(replaced).toBe(2); + for (const content of contents) { + expect(content.parts![1].fileData?.fileUri).toBe('oss://bucket/a2'); + } + }); + + it('is a no-op when nothing matches', () => { + const contents = ossContents(); + const before = JSON.parse(JSON.stringify(contents)); + expect(applyOssMediaReplacements(contents, new Map())).toBe(0); + expect(contents).toEqual(before); + }); + + it('swaps nested tool-result media inside the SAME functionResponse.parts array (D8)', () => { + const contents: Content[] = [ + { role: 'user', parts: [nestedToolResultPart()] }, + ]; + const replaced = applyOssMediaReplacements( + contents, + new Map([ + [ + 'oss://bucket/nested', + { + fileUri: 'oss://bucket/nested-degraded', + mimeType: 'video/mp4', + disclosureText: 'd', + }, + ], + ]), + ); + expect(replaced).toBe(1); + // Top level still holds exactly the functionResponse wrapper — the + // swap must not hoist nested media out of the tool result. + expect(contents[0].parts).toHaveLength(1); + const nested = contents[0].parts![0].functionResponse?.parts as Part[]; + expect(nested).toHaveLength(3); + expect(nested[0]).toEqual({ text: 'tool output' }); + expect(nested[1]).toEqual({ text: 'd' }); // disclosure directly before the media + expect(nested[2].fileData?.fileUri).toBe('oss://bucket/nested-degraded'); + expect(nested[2].fileData?.displayName).toBe('nested.mp4'); // preserved + }); +}); + +describe('observed server input limits', () => { + it('records the tightest observed limit per model', () => { + recordObservedServerInputLimit('m', 262144); + recordObservedServerInputLimit('m', 196608); + recordObservedServerInputLimit('m', 250000); // looser: ignored + expect(getObservedServerInputLimit('m')).toBe(196608); + expect(getObservedServerInputLimit('other')).toBeUndefined(); + }); + + it('ignores invalid limits', () => { + recordObservedServerInputLimit('m', 0); + recordObservedServerInputLimit('m', Number.NaN); + expect(getObservedServerInputLimit('m')).toBeUndefined(); + }); +}); diff --git a/packages/core/src/omni/reactive-degrade.ts b/packages/core/src/omni/reactive-degrade.ts new file mode 100644 index 00000000000..8a41dd92254 --- /dev/null +++ b/packages/core/src/omni/reactive-degrade.ts @@ -0,0 +1,466 @@ +/** + * @license + * Copyright 2026 Qwen Team + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Reactive server-limit fallback (issue: server-feedback-driven transport + * guard): when the provider rejects a request as over its REAL input + * limit (e.g. DashScope `Range of input length should be [1, 196608]`), + * the session must not abort — the delivered omni media is degraded + * further and the request retried. + * + * Why this exists: the local token estimator cannot predict server-side + * media billing (measured: qwen3.5-omni-plus bills video per SAMPLED + * FRAME, resolution-normalized, ~4fps sampling cap — so the default + * guard downscale of 480p/10fps does NOT reduce billed tokens, while + * sub-1fps rates do). The server's own 400 is therefore the only + * reliable over-limit signal, and this module turns it into another + * transport-guard pass with an escalating argument ladder instead of a + * session-fatal error. + * + * Flow per rejected oss:// media part: + * oss URL → upload-cache reverse lookup (sha256) → objects/ file → + * transport-guard policy with ladder-escalated arguments → + * promoted derivative → re-upload → fileUri swap in the chat history + * (with a fresh disclosure Part, decision D8). + * + * Everything here is best-effort: any failure returns "no progress" and + * the original server error propagates through the existing fail paths. + */ + +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { createHash } from 'node:crypto'; +import type { Content, Part } from '@google/genai'; +import type { Config } from '../config/config.js'; +import { createDebugLogger } from '../utils/debugLogger.js'; +import { ToolNames } from '../tools/tool-names.js'; +import { OmniObjectStore } from './storage.js'; +import { + OmniUploadCache, + DEFAULT_UPLOAD_CACHE_TTL_HOURS, +} from './upload-cache.js'; +import { DashScopeUploader, OSS_URL_PREFIX } from './upload.js'; +import { + recognizeMediaFile, + hashFileSha256, + extensionForMime, + type OmniModality, +} from './recognition.js'; +import { runFixedPolicies } from './policy/orchestrator.js'; +import type { NormalizedFixedPolicy } from './policy/types.js'; +import { formatDisclosureText } from './disclosure.js'; +import { isOmniDeliveryActive } from './delivery-gate.js'; + +const debugLogger = createDebugLogger('omni:reactive-degrade'); + +/** + * Escalation ladders per modality: argument overrides merged over the + * configured transport-guard policy's arguments, one rung per retry + * attempt. Grounded in measured server billing (see module doc): + * + * - video: fps is the only effective lever below the ~4fps sampling cap + * (~148 tokens per sampled frame, resolution-normalized), so the + * ladder drives fps down aggressively; maxHeight mainly bounds upload + * size. + * - image: billed per normalized resolution — shrink the longest edge. + * - audio: billed per second (duration is fixed), so the ladder only + * shrinks transfer size; it cannot shrink billed tokens. + * + * Attempts beyond the last rung reuse the last rung (the no-progress + * check upstream then stops the loop). + */ +const REACTIVE_LADDERS: Record< + OmniModality, + { toolName: string; steps: ReadonlyArray> } +> = { + video: { + toolName: ToolNames.OMNI_DOWNSCALE_VIDEO, + steps: [ + { maxHeight: 480, fps: 2 }, + { maxHeight: 360, fps: 0.5 }, + { maxHeight: 360, fps: 0.25 }, + ], + }, + image: { + toolName: ToolNames.OMNI_DOWNSAMPLE_IMAGE, + steps: [ + { maxDimension: 1024, quality: 70 }, + { maxDimension: 640, quality: 60 }, + { maxDimension: 448, quality: 50 }, + ], + }, + audio: { + toolName: ToolNames.OMNI_DOWNSAMPLE_AUDIO, + steps: [ + { bitrateKbps: 32, sampleRateHz: 16000, channels: 1 }, + { bitrateKbps: 16, sampleRateHz: 16000, channels: 1 }, + { bitrateKbps: 12, sampleRateHz: 8000, channels: 1 }, + ], + }, +}; + +/** Per-model server input limits observed from real rejections this + * session. Currently informational (logs/telemetry); a future guard can + * consume it as a calibrated ceiling. */ +const observedServerInputLimits = new Map(); + +export function recordObservedServerInputLimit( + model: string, + limitTokens: number, +): void { + if (!Number.isFinite(limitTokens) || limitTokens <= 0) return; + const prior = observedServerInputLimits.get(model); + if (prior === undefined || limitTokens < prior) { + observedServerInputLimits.set(model, limitTokens); + debugLogger.info( + `observed server input limit for ${model}: ${limitTokens} tokens`, + ); + } +} + +export function getObservedServerInputLimit(model: string): number | undefined { + return observedServerInputLimits.get(model); +} + +export function resetObservedServerInputLimitsForTests(): void { + observedServerInputLimits.clear(); +} + +/** Outcome of one reactive degradation pass. */ +export interface OmniReactiveDegradeOutcome { + /** fileData parts whose fileUri was swapped to a degraded derivative. */ + replacedParts: number; + /** Distinct source objects degraded and re-uploaded. */ + degradedResources: number; +} + +interface OssMediaRef { + fileUri: string; + mimeType: string; + displayName: string; +} + +/** Omni media can sit at the top level of a content's parts OR nested + * inside a tool result's `functionResponse.parts` (the tool-result + * funnel converts media in place there — see processToolResultOmniMedia). + * Every helper below must see both levels, or nested deliveries would be + * invisible to the reactive fallback and the retry loop would stall on + * an unchanged request. */ +function* iterateParts(content: Content): Generator { + for (const part of content.parts ?? []) { + yield part; + const nested = part.functionResponse?.parts; + if (Array.isArray(nested)) { + yield* nested as Part[]; + } + } +} + +/** Collect the distinct oss:// media deliveries present in `contents`. */ +export function collectOssMediaRefs(contents: Content[]): OssMediaRef[] { + const byUri = new Map(); + for (const content of contents) { + for (const part of iterateParts(content)) { + const fileData = part.fileData; + if (!fileData?.fileUri?.startsWith(OSS_URL_PREFIX)) continue; + if (byUri.has(fileData.fileUri)) continue; + byUri.set(fileData.fileUri, { + fileUri: fileData.fileUri, + mimeType: fileData.mimeType ?? '', + displayName: fileData.displayName ?? path.basename(fileData.fileUri), + }); + } + } + return [...byUri.values()]; +} + +/** Whether a retry-with-degradation is even applicable to this request. */ +export function contentsHaveOssMedia(contents: Content[]): boolean { + return contents.some((content) => { + for (const part of iterateParts(content)) { + if (part.fileData?.fileUri?.startsWith(OSS_URL_PREFIX)) return true; + } + return false; + }); +} + +/** Locate the content-addressed object file for a hash (extension is not + * recorded in the upload cache, so scan the two-level fanout dir). */ +async function findObjectPath( + store: OmniObjectStore, + sha256: string, +): Promise { + const dir = path.join(store.getObjectsDir(), sha256.slice(0, 2)); + let names: string[]; + try { + names = await fs.readdir(dir); + } catch { + return null; + } + const match = names.find((name) => name.startsWith(sha256)); + return match ? path.join(dir, match) : null; +} + +/** Transport-guard policy for the modality with the ladder rung merged + * in. Overrides only apply when the configured policy actually uses the + * modality's default degradation tool — a custom guard tool keeps its own + * arguments (re-running it unchanged is then caught as no-progress). + * Exported for direct unit testing. */ +export function buildLadderPolicy( + base: NormalizedFixedPolicy, + modality: OmniModality, + attempt: number, +): NormalizedFixedPolicy { + const ladder = REACTIVE_LADDERS[modality]; + const rung = ladder.steps[Math.min(attempt, ladder.steps.length - 1)]; + const args = + base.toolName === ladder.toolName + ? { ...base.arguments, ...rung } + : base.arguments; + return { + ...base, + id: `${base.id}.reactive-${attempt}`, + when: undefined, + onConditionUnavailable: 'run', + arguments: args, + maxRunsPerLineage: 1, + output: { + reprocessMedia: false, + source: 'omit', + artifacts: { '*': 'include' }, + }, + stage: 'transport_guard', + }; +} + +/** One replacement produced by the degradation loop. */ +export interface OssMediaReplacement { + fileUri: string; + mimeType: string; + disclosureText: string; +} + +/** + * In-place history swap: replace each fileData part whose fileUri is in + * `replacements` with [disclosure text Part, degraded fileData Part]. + * Disclosure precedes the media Part — the ordering the provider + * converters key on. Returns the number of parts swapped. Exported for + * direct unit testing. + */ +export function applyOssMediaReplacements( + contents: Content[], + replacements: Map, +): number { + let replacedParts = 0; + /** Swap matching fileData parts in one part list, returning the rebuilt + * list or null when nothing matched. */ + const swapInParts = (parts: Part[]): Part[] | null => { + const nextParts: Part[] = []; + let changed = false; + for (const part of parts) { + const uri = part.fileData?.fileUri; + const replacement = uri ? replacements.get(uri) : undefined; + if (replacement && part.fileData) { + nextParts.push({ text: replacement.disclosureText }); + nextParts.push({ + fileData: { + ...part.fileData, + fileUri: replacement.fileUri, + mimeType: replacement.mimeType, + }, + }); + replacedParts++; + changed = true; + continue; + } + // Media nested in a tool result's functionResponse.parts (see + // iterateParts) is swapped inside the SAME nested array — the + // disclosure must land immediately before its media part (D8), + // which hoisting to the top level would break. + const nested = part.functionResponse?.parts; + if (Array.isArray(nested)) { + const swappedNested = swapInParts(nested as Part[]); + if (swappedNested) { + nextParts.push({ + ...part, + functionResponse: { + ...part.functionResponse, + parts: swappedNested, + }, + } as Part); + changed = true; + continue; + } + } + nextParts.push(part); + } + return changed ? nextParts : null; + }; + for (const content of contents) { + if (!content.parts?.length) continue; + const swapped = swapInParts(content.parts); + if (swapped) content.parts = swapped; + } + return replacedParts; +} + +/** + * Degrade every oss:// media delivery found in `contents` one ladder rung + * further and swap the parts in place (fileUri + mimeType + a fresh + * disclosure Part inserted before the media, decision D8). `contents` + * must be the live chat history: the swap must persist so follow-up + * turns keep fitting under the server limit. + * + * Best-effort by contract: returns the outcome of whatever progressed; + * a resource that cannot be reverse-mapped, re-derived, or re-uploaded + * is skipped. `replacedParts === 0` tells the caller to stop retrying + * and let the original server error propagate. Only abort errors throw. + */ +export async function degradeOmniMediaAfterServerReject( + config: Config, + contents: Content[], + attempt: number, + options?: { + signal?: AbortSignal; + /** Server-reported input ceiling parsed from the rejection. */ + observedLimitTokens?: number; + }, +): Promise { + const none: OmniReactiveDegradeOutcome = { + replacedParts: 0, + degradedResources: 0, + }; + const signal = options?.signal; + if (!isOmniDeliveryActive(config)) return none; + const processingConfig = config.getOmniProcessingConfig?.(); + if (!processingConfig) return none; + const refs = collectOssMediaRefs(contents); + if (refs.length === 0) return none; + + const model = config.getModel(); + if (options?.observedLimitTokens !== undefined) { + recordObservedServerInputLimit(model, options.observedLimitTokens); + } + + const cgc = config.getContentGeneratorConfig(); + const store = new OmniObjectStore(config.storage.getQwenDir()); + // Same scope fingerprint as the delivery pipeline: entries minted for + // one (origin, apiKey) pair never serve another. + const cacheScope = createHash('sha256') + .update(`${cgc.baseUrl ?? ''}|${cgc.apiKey ?? ''}`) + .digest('hex') + .slice(0, 16); + const uploadCache = new OmniUploadCache( + store.getOmniRootDir(), + config.getOmniUploadUrlTtlHours?.() ?? DEFAULT_UPLOAD_CACHE_TTL_HOURS, + cacheScope, + ); + + // old fileUri → replacement delivery. + const replacements = new Map(); + + for (const ref of refs) { + if (signal?.aborted) break; + try { + const sha256 = await uploadCache.findSha256ByUrl(ref.fileUri); + if (!sha256) { + debugLogger.debug( + `no upload-cache mapping for ${ref.fileUri}; skipping`, + ); + continue; + } + const objectPath = await findObjectPath(store, sha256); + if (!objectPath) { + debugLogger.debug( + `object ${sha256.slice(0, 12)}… not in store; skipping`, + ); + continue; + } + const recognized = await recognizeMediaFile(objectPath, { signal }); + const basePolicy = processingConfig.transportGuardPolicies.find((p) => + p.mediaTypes.includes(recognized.modality), + ); + if (!basePolicy) continue; + const policy = buildLadderPolicy( + basePolicy, + recognized.modality, + attempt, + ); + const { deliveries } = await runFixedPolicies( + config, + { + filePath: objectPath, + recognized, + displayName: ref.displayName, + origin: 'user', + }, + { + store, + policies: [policy], + signal, + limits: processingConfig.limits, + }, + ); + const delivery = deliveries[0]; + if (!delivery || delivery.filePath === objectPath) { + debugLogger.debug( + `reactive rung ${attempt} made no progress on ${ref.displayName}`, + ); + continue; + } + + const derivedSha = + delivery.sha256 ?? (await hashFileSha256(delivery.filePath, signal)); + let fileUri = await uploadCache.get(derivedSha, model); + if (!fileUri) { + const { objectPath: derivedPath } = await store.putFile( + delivery.filePath, + derivedSha, + extensionForMime(delivery.recognized.detectedMimeType), + signal, + ); + const uploader = new DashScopeUploader({ + apiKey: cgc.apiKey ?? '', + baseUrl: cgc.baseUrl, + }); + fileUri = await uploader.uploadFile({ + filePath: derivedPath, + model, + mimeType: delivery.recognized.detectedMimeType, + signal, + }); + await uploadCache.put(derivedSha, model, fileUri); + } + if (fileUri === ref.fileUri) continue; // no progress + + const disclosureText = formatDisclosureText( + ref.displayName, + `${delivery.disclosure ?? '已进一步降质'}（服务端输入超限，第 ${attempt + 1} 次降质重试）`, + ); + replacements.set(ref.fileUri, { + fileUri, + mimeType: delivery.recognized.detectedMimeType, + disclosureText, + }); + debugLogger.info( + `reactive degrade rung ${attempt}: ${ref.displayName} ` + + `${sha256.slice(0, 12)}… → ${derivedSha.slice(0, 12)}… (${recognized.modality})`, + ); + } catch (err) { + if (signal?.aborted) throw err; + debugLogger.warn( + `reactive degrade failed for ${ref.displayName}; skipping`, + err, + ); + } + } + + if (replacements.size === 0) return none; + + const replacedParts = applyOssMediaReplacements(contents, replacements); + + return { replacedParts, degradedResources: replacements.size }; +} diff --git a/packages/core/src/omni/recognition.test.ts b/packages/core/src/omni/recognition.test.ts index cb936b76634..a375576441c 100644 --- a/packages/core/src/omni/recognition.test.ts +++ b/packages/core/src/omni/recognition.test.ts @@ -162,6 +162,22 @@ describe('sniffMediaType (S2 modalities)', async () => { }); }); + it('detects ADTS AAC (layer bits 00) as audio/aac, not audio/mpeg', () => { + // ADTS header: syncword 0xFFF, MPEG-4, layer 00, no CRC → 0xFF 0xF1. + // Layer 00 is reserved in MPEG audio, so no valid MP3 is lost. + expect(sniffMediaType(Buffer.from([0xff, 0xf1, 0x50, 0x80]))).toMatchObject( + { mimeType: 'audio/aac', modality: 'audio' }, + ); + // MPEG-2 ADTS with CRC → 0xFF 0xF8. + expect(sniffMediaType(Buffer.from([0xff, 0xf8, 0x50, 0x80]))).toMatchObject( + { mimeType: 'audio/aac', modality: 'audio' }, + ); + // A real MP3 frame (layer III = bits 01) still sniffs as audio/mpeg. + expect(sniffMediaType(Buffer.from([0xff, 0xfb, 0x90, 0x00]))).toMatchObject( + { mimeType: 'audio/mpeg', modality: 'audio' }, + ); + }); + it('rejects non-media content', () => { expect(sniffMediaType(Buffer.from('#!/bin/sh\necho hi'))).toBeNull(); expect(sniffMediaType(Buffer.from(''))).toBeNull(); diff --git a/packages/core/src/omni/recognition.ts b/packages/core/src/omni/recognition.ts index 18fb9ab469a..73f82548789 100644 --- a/packages/core/src/omni/recognition.ts +++ b/packages/core/src/omni/recognition.ts @@ -120,6 +120,13 @@ export function sniffMediaType(header: Buffer): SniffedType | null { header[1] !== 0xfe && (header[1]! & 0xe0) === 0xe0 ) { + // Layer bits 00 are RESERVED in MPEG audio but mandatory in an ADTS + // header — an .aac stream starts 0xFFF with layer 00, so this shape + // is ADTS AAC, not MP3. Mislabeling it audio/mpeg would make the + // transcribe tool announce `format: "mp3"` for AAC bytes. + if ((header[1]! & 0x06) === 0) { + return { mimeType: 'audio/aac', modality: 'audio' }; + } return { mimeType: 'audio/mpeg', modality: 'audio' }; } } @@ -144,10 +151,13 @@ export function extensionForMime(mimeType: string): string { 'image/webp': '.webp', 'image/gif': '.gif', 'audio/mpeg': '.mp3', + 'audio/aac': '.aac', 'audio/wav': '.wav', 'audio/flac': '.flac', 'audio/ogg': '.ogg', 'audio/mp4': '.m4a', + // Non-media policy artifacts (transcripts) promoted into objects/. + 'text/plain': '.txt', }; return sniffTable[mimeType] ?? '.bin'; } diff --git a/packages/core/src/omni/recovery.test.ts b/packages/core/src/omni/recovery.test.ts index 760d24cbef6..405c6366d69 100644 --- a/packages/core/src/omni/recovery.test.ts +++ b/packages/core/src/omni/recovery.test.ts @@ -11,6 +11,7 @@ import os from 'node:os'; import path from 'node:path'; import { OmniObjectStore } from './storage.js'; import { OmniUploadCache } from './upload-cache.js'; +import { OmniDegradationCache } from './policy/degradation-cache.js'; import { runStartupRecoveryOnce, resetRecoveryLatchForTests, @@ -126,6 +127,41 @@ describe('runStartupRecoveryOnce', () => { expect(await cache.get(sha256, 'm')).toBeNull(); }); + it('cascades corrupt-object deletion into the degradation cache (source AND derivative sides)', async () => { + const { sha256, objectPath } = await putObject('corrupt-policy-source'); + await fs.writeFile(objectPath, 'tampered-bytes'); // break hash==name + const degradationCache = new OmniDegradationCache(store.getOmniRootDir()); + const otherSha = 'b'.repeat(64); + // Entry where the corrupt object is the SOURCE… + await degradationCache.put(sha256, 'fp-source', { + degradedSha256: otherSha, + extension: '.jpg', + disclosure: 'd1', + mimeType: 'image/jpeg', + }); + // …entry where it is the DERIVATIVE… + await degradationCache.put(otherSha, 'fp-derived', { + degradedSha256: sha256, + extension: '.jpg', + disclosure: 'd2', + mimeType: 'image/jpeg', + }); + // …and an unrelated entry that must survive the cascade. + await degradationCache.put(otherSha, 'fp-unrelated', { + degradedSha256: otherSha, + extension: '.jpg', + disclosure: 'd3', + mimeType: 'image/jpeg', + }); + + await runStartupRecoveryOnce(store, undefined, { degradationCache }); + + await expect(fs.access(objectPath)).rejects.toThrow(); + expect(await degradationCache.get(sha256, 'fp-source')).toBeNull(); + expect(await degradationCache.get(otherSha, 'fp-derived')).toBeNull(); + expect(await degradationCache.get(otherSha, 'fp-unrelated')).not.toBeNull(); + }); + it('keeps intact objects and runs only once per process', async () => { const { objectPath } = await putObject('intact'); await runStartupRecoveryOnce(store); @@ -241,6 +277,207 @@ describe('runStartupRecoveryOnce', () => { await expect(runStartupRecoveryOnce(store)).resolves.toBeUndefined(); }); + describe('staging sweep (storage design §6.1: uncommitted work is deleted)', () => { + /** Age a staging entry past the multi-process grace window (1h). */ + async function ageEntry(p: string): Promise { + const when = new Date(Date.now() - 2 * 3600_000); + await fs.utimes(p, when, when); + } + + it('deletes every stale staging entry, including nested artifact trees and stray files', async () => { + const stagingDir = store.getStagingDir(); + const invocationDir = path.join(stagingDir, '0123456789abcdef'); + await fs.mkdir(path.join(invocationDir, 'nested'), { recursive: true }); + await fs.writeFile( + path.join(invocationDir, 'nested', 'artifact.webp'), + 'half-written', + ); + const stray = path.join(stagingDir, 'stray.tmp'); + await fs.writeFile(stray, 'stray'); + await ageEntry(invocationDir); + await ageEntry(stray); + + await runStartupRecoveryOnce(store); + + await expect(fs.readdir(stagingDir)).resolves.toEqual([]); + }); + + it('keeps entries younger than the grace window (a concurrent process may still be transcoding into them)', async () => { + const stagingDir = store.getStagingDir(); + const liveDir = path.join(stagingDir, 'fedcba9876543210'); + await fs.mkdir(liveDir, { recursive: true }); + await fs.writeFile(path.join(liveDir, 'artifact.mp4'), 'in-flight'); + + await runStartupRecoveryOnce(store); + + await expect( + fs.readFile(path.join(liveDir, 'artifact.mp4'), 'utf8'), + ).resolves.toBe('in-flight'); + }); + + it('removes a symlink ENTRY regardless of age without following it', async () => { + const outside = await fs.mkdtemp(path.join(os.tmpdir(), 'omni-stage-')); + const victim = path.join(outside, 'victim.bin'); + await fs.writeFile(victim, 'external'); + try { + const stagingDir = store.getStagingDir(); + const link = path.join(stagingDir, 'planted-link'); + await fs.symlink(outside, link); + + await runStartupRecoveryOnce(store); + + await expect(fs.lstat(link)).rejects.toThrow(); + await expect(fs.readFile(victim, 'utf8')).resolves.toBe('external'); + } finally { + await fs.rm(outside, { recursive: true, force: true }); + } + }); + + it('a symlinked staging ROOT is never swept', async () => { + const outside = await fs.mkdtemp(path.join(os.tmpdir(), 'omni-stage-')); + const victim = path.join(outside, 'victim.bin'); + await fs.writeFile(victim, 'external'); + try { + const stagingDir = store.getStagingDir(); + await fs.rm(stagingDir, { recursive: true, force: true }); + await fs.symlink(outside, stagingDir); + + await runStartupRecoveryOnce(store); + + await expect(fs.readFile(victim, 'utf8')).resolves.toBe('external'); + expect((await fs.lstat(stagingDir)).isSymbolicLink()).toBe(true); + } finally { + await fs.rm(outside, { recursive: true, force: true }); + } + }); + }); + + describe('quarantine sweep (retention window + size budget)', () => { + async function makeQuarantineEntry( + name: string, + content: string, + ageMs: number, + ): Promise { + const dir = path.join(store.getQuarantineDir(), name); + await fs.mkdir(dir, { recursive: true }); + await fs.writeFile(path.join(dir, 'artifact.bin'), content); + await fs.writeFile(path.join(dir, 'reason.json'), '{}'); + const when = new Date(Date.now() - ageMs); + await fs.utimes(dir, when, when); + return dir; + } + + it('removes entries past the retention window, keeps younger ones', async () => { + const expired = await makeQuarantineEntry( + 'aaaaaaaaaaaaaaaa', + 'old', + 8 * 86_400_000, + ); + const fresh = await makeQuarantineEntry( + 'bbbbbbbbbbbbbbbb', + 'new', + 1 * 86_400_000, + ); + + await runStartupRecoveryOnce(store, undefined, { + quarantineRetentionDays: 7, + }); + + await expect(fs.lstat(expired)).rejects.toThrow(); + await expect(fs.lstat(fresh)).resolves.toBeDefined(); + }); + + it('removes oldest entries first when over the size budget', async () => { + const oldest = await makeQuarantineEntry( + 'aaaaaaaaaaaaaaaa', + 'x'.repeat(100), + 3 * 3600_000, + ); + const middle = await makeQuarantineEntry( + 'bbbbbbbbbbbbbbbb', + 'y'.repeat(100), + 2 * 3600_000, + ); + const newest = await makeQuarantineEntry( + 'cccccccccccccccc', + 'z'.repeat(100), + 1 * 3600_000, + ); + + // ~300 bytes of artifacts (+ reason.json) against a 250-byte budget: + // dropping the single oldest entry brings the area back under. + await runStartupRecoveryOnce(store, undefined, { + quarantineMaxBytes: 250, + }); + + await expect(fs.lstat(oldest)).rejects.toThrow(); + await expect(fs.lstat(middle)).resolves.toBeDefined(); + await expect(fs.lstat(newest)).resolves.toBeDefined(); + }); + + it('keeps everything when under both retention and budget', async () => { + const a = await makeQuarantineEntry('aaaaaaaaaaaaaaaa', 'a', 3600_000); + const b = await makeQuarantineEntry('bbbbbbbbbbbbbbbb', 'b', 7200_000); + + await runStartupRecoveryOnce(store); + + await expect(fs.lstat(a)).resolves.toBeDefined(); + await expect(fs.lstat(b)).resolves.toBeDefined(); + }); + + it('a symlinked quarantine ENTRY is never traversed, sized, or deleted', async () => { + const outside = await fs.mkdtemp(path.join(os.tmpdir(), 'omni-quar-')); + const victim = path.join(outside, 'victim.bin'); + await fs.writeFile(victim, 'x'.repeat(10_000)); + const old = new Date(Date.now() - 30 * 86_400_000); + await fs.utimes(outside, old, old); + await fs.utimes(victim, old, old); + try { + const link = path.join(store.getQuarantineDir(), 'dddddddddddddddd'); + await fs.symlink(outside, link); + + // Aggressive limits: if the sweep treated the link as an entry it + // would be expired AND over budget — external bytes must survive. + await runStartupRecoveryOnce(store, undefined, { + quarantineRetentionDays: 1, + quarantineMaxBytes: 1, + }); + + await expect(fs.readFile(victim, 'utf8')).resolves.toBe( + 'x'.repeat(10_000), + ); + expect((await fs.lstat(link)).isSymbolicLink()).toBe(true); + } finally { + await fs.rm(outside, { recursive: true, force: true }); + } + }); + + it('a symlinked quarantine ROOT is never swept', async () => { + const outside = await fs.mkdtemp(path.join(os.tmpdir(), 'omni-quar-')); + const victimDir = path.join(outside, 'eeeeeeeeeeeeeeee'); + await fs.mkdir(victimDir); + await fs.writeFile(path.join(victimDir, 'victim.bin'), 'external'); + const old = new Date(Date.now() - 30 * 86_400_000); + await fs.utimes(victimDir, old, old); + try { + const quarantineDir = store.getQuarantineDir(); + await fs.rm(quarantineDir, { recursive: true, force: true }); + await fs.symlink(outside, quarantineDir); + + await runStartupRecoveryOnce(store, undefined, { + quarantineRetentionDays: 1, + }); + + await expect( + fs.readFile(path.join(victimDir, 'victim.bin'), 'utf8'), + ).resolves.toBe('external'); + expect((await fs.lstat(quarantineDir)).isSymbolicLink()).toBe(true); + } finally { + await fs.rm(outside, { recursive: true, force: true }); + } + }); + }); + describe('symlink containment (recovery must never leave the omni root)', () => { /** External dir with a victim file whose NAME makes recovery want to * delete it through every code path: hash-mismatched "object", expired diff --git a/packages/core/src/omni/recovery.ts b/packages/core/src/omni/recovery.ts index 852b03983de..d621e596fad 100644 --- a/packages/core/src/omni/recovery.ts +++ b/packages/core/src/omni/recovery.ts @@ -12,6 +12,7 @@ import { pipeline } from 'node:stream/promises'; import { createDebugLogger } from '../utils/debugLogger.js'; import type { OmniObjectStore } from './storage.js'; import type { OmniUploadCache } from './upload-cache.js'; +import type { OmniDegradationCache } from './policy/degradation-cache.js'; const debugLogger = createDebugLogger('omni:recovery'); @@ -31,12 +32,35 @@ const SAMPLE_VERIFY_MAX_BYTES = 64 * 1024 * 1024; * belong to a promotion in flight in ANOTHER process — deleting it would * fail that process's rename. Older survivors are crash leftovers. */ const TMP_GRACE_MS = 3600_000; +/** Grace window for staging entries, for the same multi-process reason: + * a second CLI process starting while another is mid-transcode must not + * delete the live invocation's work directory out from under its tool. + * One hour comfortably exceeds the 10-minute default policy-tool timeout + * (a directory's mtime is set at creation), so anything older is a crash + * leftover, not an in-flight run. Exported so config validation can cap + * `policyTools..runtime.timeoutMs` below it — a timeout the sweep + * could outrun would let a live invocation's staging be deleted. */ +export const STAGING_GRACE_MS = 3600_000; +/** Default retention for quarantined invocations (storage design §7). */ +const QUARANTINE_RETENTION_DAYS = 7; +/** Default size budget for the quarantine area (storage design §7). */ +const QUARANTINE_MAX_BYTES = 5 * 1024 * 1024 * 1024; /** Tunables for {@link runStartupRecoveryOnce}; production callers use * the defaults, tests inject small values. */ export interface StartupRecoveryOptions { sampleVerifyLimit?: number; sampleVerifyMaxBytes?: number; + /** Quarantined invocations older than this are removed. */ + quarantineRetentionDays?: number; + /** Above this total size, quarantined invocations are removed + * oldest-first until the area fits. */ + quarantineMaxBytes?: number; + /** When set, corrupt-object deletion also cascades into the + * degradation cache (both as source and as derivative), keeping + * `policy-cache.json` free of entries that can never be served + * again. */ + degradationCache?: OmniDegradationCache; } /** One latch per omni root: distinct stores in one process (multi-project @@ -91,6 +115,131 @@ async function sweepDownloads(downloadsDir: string): Promise { } } +/** + * Delete crash-orphaned entries under `staging/`. Staging entries belong + * to policy invocations that never committed (a successful commit deletes + * its own staging directory first), so anything past the grace window is + * garbage (storage design §6.1). Entries YOUNGER than the grace window + * are kept: they may be a concurrent process's live invocation, and its + * own commit/quarantine path cleans them up. The staging root itself must + * be a real directory — a symlinked root would redirect the recursive + * deletes outside the omni root. + */ +async function sweepStaging(stagingDir: string): Promise { + if (!(await isRealDirectory(stagingDir))) return; + let names: string[]; + try { + names = await fs.readdir(stagingDir); + } catch { + return; + } + for (const name of names) { + const p = path.join(stagingDir, name); + try { + const st = await fs.lstat(p); + // A young REAL entry may belong to an in-flight invocation in + // another process; symlinks are never live invocations (staging + // dirs are created with mkdir) and are removed regardless of age + // (rm on a symlink removes the link itself without following it, + // so no containment check is needed per entry). + if (!st.isSymbolicLink() && Date.now() - st.mtimeMs < STAGING_GRACE_MS) { + continue; + } + await fs.rm(p, { recursive: true, force: true }); + debugLogger.debug(`recovery: removed uncommitted staging ${name}`); + } catch { + // Best-effort sweep. + } + } +} + +/** Recursively sum the sizes of regular files under a REAL directory, + * never following symlinks (neither directory nor file entries). */ +async function directorySizeBytes(dir: string): Promise { + let total = 0; + let names: string[]; + try { + names = await fs.readdir(dir); + } catch { + return total; + } + for (const name of names) { + const p = path.join(dir, name); + try { + const st = await fs.lstat(p); + if (st.isFile()) { + total += st.size; + } else if (st.isDirectory()) { + total += await directorySizeBytes(p); + } + } catch { + // Unreadable entry contributes nothing. + } + } + return total; +} + +/** + * Enforce the quarantine retention window and size budget (storage design + * §4.4/§6.1): entries older than `retentionMs` are removed; if the + * remainder still exceeds `maxBytes`, the oldest entries are removed + * first until the area fits. Only REAL directories are treated as + * quarantine entries — symlinks are never traversed, sized, or deleted. + */ +async function sweepQuarantine( + quarantineDir: string, + retentionMs: number, + maxBytes: number, +): Promise { + if (!(await isRealDirectory(quarantineDir))) return; + let names: string[]; + try { + names = await fs.readdir(quarantineDir); + } catch { + return; + } + const entries: Array<{ name: string; mtimeMs: number; sizeBytes: number }> = + []; + const cutoff = Date.now() - retentionMs; + for (const name of names) { + const p = path.join(quarantineDir, name); + if (!(await isRealDirectory(p))) continue; + try { + const st = await fs.lstat(p); + if (st.mtimeMs < cutoff) { + await fs.rm(p, { recursive: true, force: true }); + debugLogger.debug(`recovery: removed expired quarantine ${name}`); + continue; + } + entries.push({ + name, + mtimeMs: st.mtimeMs, + sizeBytes: await directorySizeBytes(p), + }); + } catch { + // Best-effort sweep. + } + } + let total = entries.reduce((sum, e) => sum + e.sizeBytes, 0); + if (total <= maxBytes) return; + entries.sort((a, b) => a.mtimeMs - b.mtimeMs); + for (const entry of entries) { + if (total <= maxBytes) break; + try { + await fs.rm(path.join(quarantineDir, entry.name), { + recursive: true, + force: true, + }); + total -= entry.sizeBytes; + debugLogger.debug( + `recovery: removed quarantine ${entry.name} (over size budget)`, + ); + } catch { + // Best-effort sweep. + } + } +} + async function sweepTmpFiles(objectsDir: string): Promise { if (!(await isRealDirectory(objectsDir))) return; let shards: string[]; @@ -130,6 +279,7 @@ async function sweepTmpFiles(objectsDir: string): Promise { async function sampleVerifyObjects( objectsDir: string, uploadCache: OmniUploadCache | undefined, + degradationCache: OmniDegradationCache | undefined, limit: number, maxBytes: number, ): Promise { @@ -186,6 +336,13 @@ async function sampleVerifyObjects( if (hash.digest('hex') !== expected) { await fs.rm(full, { force: true }); await uploadCache?.removeBySha256(expected); + // The corrupt object may have been a policy SOURCE (its cached + // derivatives can never be re-verified against it) or a policy + // DERIVATIVE (entries pointing at it can never be served) — + // cascade both directions so policy-cache.json does not + // accumulate orphans. + await degradationCache?.removeByOriginalSha256(expected); + await degradationCache?.removeByDegradedSha256(expected); debugLogger.debug( `recovery: removed corrupt object ${rel} (hash mismatch)`, ); @@ -201,11 +358,18 @@ async function sampleVerifyObjects( * lazily the first time the omni pipeline is touched — zero cost when * omni is unused. * - * 1. crash-orphaned `downloads/*.part` older than the 48h debugging + * 1. staging entries older than the multi-process grace window are + * deleted — they belong to policy invocations that never committed; + * younger entries may be another process's live run (storage design + * §6.1); + * 2. crash-orphaned `downloads/*.part` older than the 48h debugging * retention window are removed; - * 2. `objects/…/.tmp-*` promotion orphans are removed (crash leftovers); - * 3. a small sample of objects is hash-verified; corrupt objects are - * deleted with their upload-cache entries cascaded. + * 3. `quarantine/` is trimmed to its retention window and size budget + * (oldest-first once over budget); + * 4. `objects/…/.tmp-*` promotion orphans are removed (crash leftovers); + * 5. a small sample of objects is hash-verified; corrupt objects are + * deleted with their upload-cache and degradation-cache entries + * cascaded. * * Never throws: recovery is hygiene, not a gate. That covers the latch * key lookup too — a store whose getOmniRootDir() throws yields a @@ -236,12 +400,20 @@ export function runStartupRecoveryOnce( // managed tree. (An absent directory fails the check too, which is // fine — there is nothing to sweep beneath it.) if (!(await isRealDirectory(root))) return; + await sweepStaging(path.join(root, 'staging')); await sweepDownloads(path.join(root, 'downloads')); + await sweepQuarantine( + path.join(root, 'quarantine'), + (options?.quarantineRetentionDays ?? QUARANTINE_RETENTION_DAYS) * + 86_400_000, + options?.quarantineMaxBytes ?? QUARANTINE_MAX_BYTES, + ); if (!(await isRealDirectory(path.join(root, 'objects')))) return; await sweepTmpFiles(store.getObjectsDir()); await sampleVerifyObjects( store.getObjectsDir(), uploadCache, + options?.degradationCache, options?.sampleVerifyLimit ?? SAMPLE_VERIFY_LIMIT, options?.sampleVerifyMaxBytes ?? SAMPLE_VERIFY_MAX_BYTES, ); diff --git a/packages/core/src/omni/storage.test.ts b/packages/core/src/omni/storage.test.ts index bff12275c17..97fbf64befb 100644 --- a/packages/core/src/omni/storage.test.ts +++ b/packages/core/src/omni/storage.test.ts @@ -9,7 +9,7 @@ import { createHash } from 'node:crypto'; import fs from 'node:fs/promises'; import os from 'node:os'; import path from 'node:path'; -import { OmniObjectStore } from './storage.js'; +import { OmniObjectStore, prepareOmniDownloadsDir } from './storage.js'; describe('OmniObjectStore', () => { let qwenDir: string; @@ -99,6 +99,47 @@ describe('OmniObjectStore', () => { ); }); + describe('objectPathFor validates cache-sourced components (path traversal)', () => { + const sha256 = 'a'.repeat(64); + + it.each([ + [ + 'traversal hash', + '../../../../etc/passwd', + '.mp4', + /invalid object hash/, + ], + ['uppercase hash', 'A'.repeat(64), '.mp4', /invalid object hash/], + ['short hash', 'abc123', '.mp4', /invalid object hash/], + [ + 'traversal extension', + sha256, + '/../../../../tmp/evil', + /invalid object extension/, + ], + [ + 'multi-segment extension', + sha256, + '.jpg/../x', + /invalid object extension/, + ], + ['dotless extension', sha256, 'jpg', /invalid object extension/], + ['double-dot extension', sha256, '..', /invalid object extension/], + ['overlong extension', sha256, '.abcdefghi', /invalid object extension/], + ])('throws on %s', (_label, hash, ext, message) => { + expect(() => store.objectPathFor(hash, ext)).toThrow(message); + }); + + it('accepts every extension recognition can emit', () => { + for (const ext of ['.mp4', '.webp', '.m4a', '.bin', '.jpg']) { + const p = store.objectPathFor(sha256, ext); + expect(p).toBe( + path.join(store.getObjectsDir(), 'aa', `${sha256}${ext}`), + ); + } + }); + }); + it('propagates copy failures without leaving temp files', async () => { const missing = path.join(qwenDir, 'does-not-exist.mp4'); const sha256 = createHash('sha256').update('missing').digest('hex'); @@ -159,4 +200,178 @@ describe('OmniObjectStore', () => { await fs.rm(linkedQwen, { recursive: true, force: true }); } }); + + describe('staging and quarantine areas', () => { + const INVOCATION_ID = '0123456789abcdef'; + + it('ensureLayout creates staging/ and quarantine/ with 0o700', async () => { + await store.ensureLayout(); + for (const dir of [store.getStagingDir(), store.getQuarantineDir()]) { + const st = await fs.stat(dir); + expect(st.isDirectory()).toBe(true); + if (process.platform !== 'win32') { + expect(st.mode & 0o777).toBe(0o700); + } + } + expect(store.getStagingDir()).toBe(path.join(qwenDir, 'omni', 'staging')); + expect(store.getQuarantineDir()).toBe( + path.join(qwenDir, 'omni', 'quarantine'), + ); + }); + + it('creates an exclusive per-invocation staging directory', async () => { + const dir = await store.createStagingDir(INVOCATION_ID); + expect(dir).toBe(path.join(store.getStagingDir(), INVOCATION_ID)); + const st = await fs.stat(dir); + expect(st.isDirectory()).toBe(true); + if (process.platform !== 'win32') { + expect(st.mode & 0o777).toBe(0o700); + } + // A second create with the same id must fail, never silently reuse. + await expect(store.createStagingDir(INVOCATION_ID)).rejects.toThrow(); + }); + + it.each([ + ['path traversal', '../../escape00'], + ['uppercase hex', '0123456789ABCDEF'], + ['wrong length', '0123456789abcde'], + ['separator smuggling', '0123456789abcde/'], + ])('rejects an invalid invocation id: %s', async (_label, id) => { + await expect(store.createStagingDir(id)).rejects.toThrow( + /Invalid omni policy invocation id/, + ); + await expect(store.removeStagingDir(id)).rejects.toThrow( + /Invalid omni policy invocation id/, + ); + await expect( + store.quarantineInvocation(id, { + policyId: 'p', + toolName: 't', + reason: 'r', + }), + ).rejects.toThrow(/Invalid omni policy invocation id/); + }); + + it('removeStagingDir deletes the invocation directory recursively', async () => { + const dir = await store.createStagingDir(INVOCATION_ID); + await fs.mkdir(path.join(dir, 'nested')); + await fs.writeFile(path.join(dir, 'nested', 'artifact.webp'), 'bytes'); + await store.removeStagingDir(INVOCATION_ID); + await expect(fs.lstat(dir)).rejects.toThrow(); + // Idempotent on a missing directory. + await expect( + store.removeStagingDir(INVOCATION_ID), + ).resolves.toBeUndefined(); + }); + + it('quarantineInvocation moves artifacts and writes reason.json', async () => { + const dir = await store.createStagingDir(INVOCATION_ID); + await fs.writeFile(path.join(dir, 'partial.mp4'), 'half-transcoded'); + const quarantineDir = await store.quarantineInvocation(INVOCATION_ID, { + policyId: 'video-downscale-v1', + toolName: 'omni_downscale_video', + reason: 'required output missing', + }); + + expect(quarantineDir).toBe( + path.join(store.getQuarantineDir(), INVOCATION_ID), + ); + // Staging entry is gone; artifacts moved with original names. + await expect(fs.lstat(dir)).rejects.toThrow(); + await expect( + fs.readFile(path.join(quarantineDir, 'partial.mp4'), 'utf8'), + ).resolves.toBe('half-transcoded'); + const reason = JSON.parse( + await fs.readFile(path.join(quarantineDir, 'reason.json'), 'utf8'), + ); + expect(reason).toMatchObject({ + policyId: 'video-downscale-v1', + toolName: 'omni_downscale_video', + reason: 'required output missing', + }); + expect(new Date(reason.failedAt).getTime()).not.toBeNaN(); + }); + + it('quarantineInvocation fails when the staging directory is missing', async () => { + await store.ensureLayout(); + await expect( + store.quarantineInvocation(INVOCATION_ID, { + policyId: 'p', + toolName: 't', + reason: 'r', + }), + ).rejects.toThrow(); + }); + + it('quarantineInvocation refuses a symlinked staging entry', async () => { + await store.ensureLayout(); + const outside = path.join(qwenDir, 'outside-staging'); + await fs.mkdir(outside); + await fs.symlink( + outside, + path.join(store.getStagingDir(), INVOCATION_ID), + ); + await expect( + store.quarantineInvocation(INVOCATION_ID, { + policyId: 'p', + toolName: 't', + reason: 'r', + }), + ).rejects.toThrow(/not a real directory/); + // Nothing was written through the link. + await expect(fs.readdir(outside)).resolves.toEqual([]); + }); + }); +}); + +describe('prepareOmniDownloadsDir', () => { + let root: string; + + beforeEach(async () => { + root = await fs.mkdtemp(path.join(os.tmpdir(), 'omni-dl-prep-')); + }); + + afterEach(async () => { + await fs.rm(root, { recursive: true, force: true }); + }); + + it('creates a missing downloads dir with 0o700 and returns its path', async () => { + const dir = path.join(root, 'omni', 'downloads'); + await expect(prepareOmniDownloadsDir(dir)).resolves.toBe(dir); + const st = await fs.stat(dir); + expect(st.isDirectory()).toBe(true); + if (process.platform !== 'win32') { + expect(st.mode & 0o777).toBe(0o700); + } + }); + + it('is idempotent over an existing real directory', async () => { + const dir = path.join(root, 'downloads'); + await fs.mkdir(dir); + await fs.writeFile(path.join(dir, 'keep.part'), 'x'); + await expect(prepareOmniDownloadsDir(dir)).resolves.toBe(dir); + // Existing contents survive — prepare never wipes the staging area. + await expect( + fs.readFile(path.join(dir, 'keep.part'), 'utf8'), + ).resolves.toBe('x'); + }); + + it('refuses a symlink planted at the downloads path (mkdir succeeds silently on it)', async () => { + const outside = path.join(root, 'outside-target'); + await fs.mkdir(outside); + const dir = path.join(root, 'downloads'); + await fs.symlink(outside, dir); + await expect(prepareOmniDownloadsDir(dir)).rejects.toThrow( + /not a real directory/, + ); + // Nothing was created through the link. + await expect(fs.readdir(outside)).resolves.toEqual([]); + }); + + it('refuses a regular file planted at the downloads path', async () => { + const dir = path.join(root, 'downloads'); + await fs.writeFile(dir, 'not a directory'); + // mkdir itself fails with EEXIST/ENOTDIR here — either way it must throw. + await expect(prepareOmniDownloadsDir(dir)).rejects.toThrow(); + }); }); diff --git a/packages/core/src/omni/storage.ts b/packages/core/src/omni/storage.ts index 56e875157a2..b4f4d638012 100644 --- a/packages/core/src/omni/storage.ts +++ b/packages/core/src/omni/storage.ts @@ -19,6 +19,34 @@ export interface PutObjectResult { deduped: boolean; } +/** Why a policy invocation's staging directory was quarantined; persisted + * as `reason.json` beside the failed artifacts for debugging. */ +export interface QuarantineReason { + /** The fixed policy whose invocation failed. */ + policyId: string; + /** The media policy tool that ran (or failed to run). */ + toolName: string; + /** Human-readable failure description. */ + reason: string; +} + +/** Policy invocation IDs are orchestrator-generated 16-hex tokens; anything + * else (path separators, dots, uppercase) is refused before touching the + * filesystem so staging/quarantine paths can never escape their area. */ +const INVOCATION_ID_RE = /^[0-9a-f]{16}$/; + +/** Object-store extensions are a single dotted alphanumeric component + * (".jpg", ".m4a", ".bin" — see extensionForMime). Anything else — path + * separators, dots beyond the leading one — is rejected so an extension + * can never smuggle traversal segments into an object path. */ +export const OBJECT_EXTENSION_RE = /^\.[A-Za-z0-9]{1,8}$/; + +function assertInvocationId(invocationId: string): void { + if (!INVOCATION_ID_RE.test(invocationId)) { + throw new Error(`Invalid omni policy invocation id: ${invocationId}`); + } +} + /** Reject paths that exist but are not what the store expects (symlinks, * devices, …). The store never follows symlinks for its own entries. */ async function assertRealDirIfExists(p: string): Promise { @@ -35,14 +63,40 @@ async function assertRealDirIfExists(p: string): Promise { } } +/** + * Prepare a `downloads/` staging area and verify it is a REAL directory + * before returning. Every caller that writes `.part` files (the URL + * funnel, the tool-result funnel) must go through this: + * `mkdir { recursive: true }` succeeds silently when the path already + * exists as a symlink to a directory, so a link planted at + * `.qwen/omni/downloads` would otherwise redirect the write to an + * attacker-chosen location (and outside the recovery sweep, which + * deliberately refuses to descend symlinked directories). Fails closed — + * callers degrade per their own contract (inline part, delivery error). + */ +export async function prepareOmniDownloadsDir( + downloadsDir: string, +): Promise { + await fs.mkdir(downloadsDir, { recursive: true, mode: 0o700 }); + const st = await fs.lstat(downloadsDir); + if (st.isSymbolicLink() || !st.isDirectory()) { + throw new Error( + `Omni downloads path is not a real directory (symlink or special file refused): ${downloadsDir}`, + ); + } + return downloadsDir; +} + /** * Content-addressed, immutable object store under `/.qwen/omni/`. * - * S1 scope: only the `objects/` area exists. Layout: + * Layout (storage design §4): * * .qwen/omni/ * ├── .gitignore # "*" — self-ignoring - * └── objects/sha256// + * ├── objects/sha256// + * ├── staging// # policy tool work dirs (pre-commit) + * └── quarantine// # failed invocations + reason.json * * Write protocol: stream-copy to a sibling `.tmp-*` file in the final * directory while re-computing the content hash, verify it matches the @@ -68,8 +122,34 @@ export class OmniObjectStore { return path.join(this.omniRoot, 'objects', 'sha256'); } - /** Compute the final object path for a content hash + extension. */ + /** Root of the policy-invocation work area (stale entries deleted by + * startup recovery — anything past the grace window belongs to an + * uncommitted, crashed run). */ + getStagingDir(): string { + return path.join(this.omniRoot, 'staging'); + } + + /** Root of the failed-invocation debris area, kept for debugging under + * a retention/size budget and never re-entering recognition/delivery. */ + getQuarantineDir(): string { + return path.join(this.omniRoot, 'quarantine'); + } + + /** + * Compute the final object path for a content hash + extension. + * + * Both components are validated here, not just at putFile: callers may + * feed values read back from on-disk cache files (policy-cache.json), + * and a crafted hash or extension ("/../../…") would otherwise turn + * this join into a path-traversal primitive pointing outside the store. + */ objectPathFor(sha256: string, extension: string): string { + if (!/^[0-9a-f]{64}$/.test(sha256)) { + throw new Error(`invalid object hash: ${JSON.stringify(sha256)}`); + } + if (!OBJECT_EXTENSION_RE.test(extension)) { + throw new Error(`invalid object extension: ${JSON.stringify(extension)}`); + } return path.join( this.getObjectsDir(), sha256.slice(0, 2), @@ -87,7 +167,14 @@ export class OmniObjectStore { await assertRealDirIfExists(this.omniRoot); await assertRealDirIfExists(path.join(this.omniRoot, 'objects')); await assertRealDirIfExists(this.getObjectsDir()); + await assertRealDirIfExists(this.getStagingDir()); + await assertRealDirIfExists(this.getQuarantineDir()); await fs.mkdir(this.getObjectsDir(), { recursive: true, mode: 0o700 }); + await fs.mkdir(this.getStagingDir(), { recursive: true, mode: 0o700 }); + await fs.mkdir(this.getQuarantineDir(), { + recursive: true, + mode: 0o700, + }); const gitignorePath = path.join(this.omniRoot, '.gitignore'); try { // 'wx' fails when the file already exists — atomic create-once, @@ -106,6 +193,71 @@ export class OmniObjectStore { return this.layoutReady; } + /** + * Create the exclusive work directory for one policy invocation and + * return its absolute path. The directory is the ONLY location the + * policy tool is allowed to write to (storage design §4.3). Creation is + * non-recursive and exclusive: a pre-existing entry (id collision or a + * planted path) fails instead of being silently reused. + */ + async createStagingDir(invocationId: string): Promise { + assertInvocationId(invocationId); + await this.ensureLayout(); + const dir = path.join(this.getStagingDir(), invocationId); + await fs.mkdir(dir, { mode: 0o700 }); + return dir; + } + + /** + * Delete one invocation's staging directory (after a successful commit, + * or as the failure path while quarantine is not involved). + */ + async removeStagingDir(invocationId: string): Promise { + assertInvocationId(invocationId); + await fs.rm(path.join(this.getStagingDir(), invocationId), { + recursive: true, + force: true, + }); + } + + /** + * Move a failed invocation's staging directory into + * `quarantine//`, preserving the artifact files and adding + * a `reason.json` (storage design §4.4). The reason file is written into + * the staging directory BEFORE the rename so the quarantine entry appears + * complete in one atomic step; a crash in between leaves it in staging, + * which startup recovery deletes once past the grace window. + */ + async quarantineInvocation( + invocationId: string, + reason: QuarantineReason, + ): Promise { + assertInvocationId(invocationId); + await this.ensureLayout(); + const stagingDir = path.join(this.getStagingDir(), invocationId); + // The rename source must be a real directory: a symlink here would + // make the reason.json write (and the quarantined "content") point + // outside the omni root. + const st = await fs.lstat(stagingDir); + if (st.isSymbolicLink() || !st.isDirectory()) { + throw new Error( + `Staging path is not a real directory (symlink or special file refused): ${stagingDir}`, + ); + } + await fs.writeFile( + path.join(stagingDir, 'reason.json'), + JSON.stringify( + { ...reason, failedAt: new Date().toISOString() }, + null, + 2, + ), + { mode: 0o600 }, + ); + const quarantineDir = path.join(this.getQuarantineDir(), invocationId); + await fs.rename(stagingDir, quarantineDir); + return quarantineDir; + } + /** * Promote a local file into the object store under its content hash. * The bytes are re-hashed while copying and verified against `sha256`, diff --git a/packages/core/src/omni/tool-result-media.test.ts b/packages/core/src/omni/tool-result-media.test.ts index 98c3559b7ed..58e36c779b1 100644 --- a/packages/core/src/omni/tool-result-media.test.ts +++ b/packages/core/src/omni/tool-result-media.test.ts @@ -21,7 +21,10 @@ import type { Config } from '../config/config.js'; const deliverMock = vi.hoisted(() => vi.fn()); const gateMock = vi.hoisted(() => vi.fn()); -vi.mock('./index.js', () => ({ +vi.mock('./index.js', async (importOriginal) => ({ + // buildAdditionalMediaParts stays REAL: these tests pin the funnel's + // materialization of multi-output deliveries end to end. + ...(await importOriginal()), isOmniDeliveryActive: gateMock, processMediaForOmniDelivery: deliverMock, })); @@ -196,6 +199,125 @@ describe('processToolResultOmniMedia', () => { expect(result[1]!.fileData?.fileUri).toBe('oss://bucket/key3'); }); + it('withholds the part when guard-stage PROCESSING fails (never inline the rejected bytes)', async () => { + // A guard-policy execution failure arrives as OmniTransportGuardError + // with the underlying error as `cause` (see processMediaForOmniDelivery's + // guard loop): the violation verdict already stands, so falling back to + // inline would deliver exactly the over-limit bytes the guard rejected. + const { OmniTransportGuardError } = await import('./guard.js'); + deliverMock.mockRejectedValueOnce( + new OmniTransportGuardError( + 'Transport-guard processing failed for x.png: ffmpeg failed (exit 1)', + { cause: new Error('ffmpeg failed (exit 1)') }, + ), + ); + const result = await processToolResultOmniMedia( + [inlinePart('image/png', PNG_BYTES)], + cfg({ image: true }), + signal, + ); + expect(result[0]!.inlineData).toBeUndefined(); + expect(result[0]!.text).toMatch(/withheld by the omni transport guard/); + expect(result[0]!.text).toMatch(/Transport-guard processing failed/); + }); + + it('replaces an explicitly omitted delivery with the omission notice text', async () => { + // Stage B (policy design §10.2): the pipeline itself withheld the media + // after the guard policies could not bring it within limits. Not an + // error — the notice stands in for the part. + deliverMock.mockResolvedValueOnce({ + fileUri: '', + mimeType: 'image/png', + sha256: '', + recognized: { modality: 'image' }, + tokenEstimate: { + estimatedTokenCount: 1, + method: 'raw-resource-v1', + status: 'ok', + }, + deduped: false, + omission: { reason: 'still 900 bytes over the upload limit' }, + }); + const parts = [inlinePart('image/png', PNG_BYTES)]; + const result = await processToolResultOmniMedia( + parts, + cfg({ image: true }), + signal, + ); + expect(result).not.toBe(parts); + expect(result).toHaveLength(1); + expect(result[0]).toEqual({ + text: '【媒体省略】tool-media.image：still 900 bytes over the upload limit', + }); + }); + + it('charges uploaded additionalMedia extras against the upload-count budget', async () => { + // One part whose delivery carries 7 uploaded extras uses 1 + 7 = 8 + // upload slots — a multi-output policy must not let a tool result fan + // out past MAX_UPLOADS_PER_TOOL_RESULT. The next part stays inline. + deliverMock.mockResolvedValueOnce({ + fileUri: 'oss://bucket/primary', + mimeType: 'image/jpeg', + sha256: 'b'.repeat(64), + recognized: { modality: 'image' }, + tokenEstimate: { + estimatedTokenCount: 1, + method: 'raw-resource-v1', + status: 'ok', + }, + deduped: false, + additionalMedia: Array.from({ length: 8 }, (_, i) => ({ + fileUri: i === 0 ? '' : `oss://bucket/frame${i}`, + mimeType: 'image/jpeg', + sha256: String(i).repeat(64).slice(0, 64), + // The omitted extra was NOT uploaded — it must not be charged. + ...(i === 0 ? { omission: { reason: 'too big' } } : {}), + })), + }); + const parts = [ + inlinePart('image/png', PNG_BYTES), + inlinePart('image/png', PNG_BYTES), + ]; + const result = await processToolResultOmniMedia( + parts, + cfg({ image: true }), + signal, + ); + // Second part never started a delivery (budget exhausted). + expect(deliverMock).toHaveBeenCalledTimes(1); + expect(result[result.length - 1]!.inlineData).toBeDefined(); + expect(result.filter((p) => p.fileData).length).toBe(8); + }); + + it('an omission does not consume the per-result upload budgets', async () => { + // Nothing was uploaded for an omitted part, so all 8 upload slots must + // remain for the following parts. + deliverMock.mockResolvedValueOnce({ + fileUri: '', + mimeType: 'image/png', + sha256: '', + recognized: { modality: 'image' }, + tokenEstimate: { + estimatedTokenCount: 1, + method: 'raw-resource-v1', + status: 'ok', + }, + deduped: false, + omission: { reason: 'over limit' }, + }); + const parts = Array.from({ length: 9 }, () => + inlinePart('image/png', PNG_BYTES), + ); + const result = await processToolResultOmniMedia( + parts, + cfg({ image: true }), + signal, + ); + expect(deliverMock).toHaveBeenCalledTimes(9); + expect(result.filter((p) => p.fileData).length).toBe(8); + expect(result.filter((p) => p.inlineData).length).toBe(0); + }); + it('keeps the part inline when staging-dir setup itself fails', async () => { // ~/.qwen/omni existing as a regular FILE makes mkdir fail with ENOTDIR. // That failure must degrade THIS part to inline like any other delivery @@ -221,6 +343,39 @@ describe('processToolResultOmniMedia', () => { } }); + it('keeps the part inline when downloads/ is a planted symlink (no bytes through the link)', async () => { + // mkdir { recursive: true } succeeds silently on a symlink-to-dir, so + // without the lstat guard the staged bytes would land at an + // attacker-chosen location outside the omni root. + const qwenDir = await nodeFs.mkdtemp( + nodePath.join(os.tmpdir(), 'omni-trm-link-'), + ); + const outside = await nodeFs.mkdtemp( + nodePath.join(os.tmpdir(), 'omni-trm-out-'), + ); + try { + await nodeFs.mkdir(nodePath.join(qwenDir, 'omni'), { recursive: true }); + await nodeFs.symlink( + outside, + nodePath.join(qwenDir, 'omni', 'downloads'), + ); + const config = { + isOmniEnabled: () => true, + getContentGeneratorConfig: () => ({ modalities: { image: true } }), + storage: { getQwenDir: () => qwenDir }, + } as unknown as Config; + const parts = [inlinePart('image/png', PNG_BYTES)]; + const result = await processToolResultOmniMedia(parts, config, signal); + expect(result).toBe(parts); + expect(deliverMock).not.toHaveBeenCalled(); + // Nothing was written through the link. + await expect(nodeFs.readdir(outside)).resolves.toEqual([]); + } finally { + await nodeFs.rm(qwenDir, { recursive: true, force: true }); + await nodeFs.rm(outside, { recursive: true, force: true }); + } + }); + it('returns the original array untouched when the omni gate is off', async () => { gateMock.mockReturnValue(false); const parts = [inlinePart('image/png', PNG_BYTES)]; @@ -233,6 +388,168 @@ describe('processToolResultOmniMedia', () => { expect(deliverMock).not.toHaveBeenCalled(); }); + it('emits the degradation disclosure text immediately before the fileData part', async () => { + deliverMock.mockResolvedValue({ + fileUri: 'oss://bucket/degraded', + mimeType: 'image/jpeg', + sha256: 'b'.repeat(64), + recognized: { modality: 'image' }, + tokenEstimate: { + estimatedTokenCount: 1, + method: 'raw-resource-v1', + status: 'ok', + }, + deduped: false, + disclosure: 'downsampled to 1568px', + degraded: true, + }); + const parts = [inlinePart('image/png', PNG_BYTES)]; + const result = await processToolResultOmniMedia( + parts, + cfg({ image: true }), + signal, + ); + expect(result).toHaveLength(2); + expect(result[0]!.text).toBe( + '【媒体降质】tool-media.image：downsampled to 1568px', + ); + expect(result[1]!.fileData?.fileUri).toBe('oss://bucket/degraded'); + // The pipeline was told this media came from a tool (policy origins). + expect(deliverMock).toHaveBeenCalledWith( + expect.any(String), + expect.anything(), + expect.objectContaining({ + origin: 'tool', + displayName: 'tool-media.image', + expectedModality: 'image', + }), + ); + }); + + it('materializes additionalMedia extras as [disclosure, fileData] pairs after the primary', async () => { + deliverMock.mockResolvedValue({ + fileUri: 'oss://bucket/frame1', + mimeType: 'image/jpeg', + sha256: 'b'.repeat(64), + recognized: { modality: 'image' }, + tokenEstimate: { + estimatedTokenCount: 1, + method: 'raw-resource-v1', + status: 'ok', + }, + deduped: false, + disclosure: '帧 1/3', + degraded: true, + additionalMedia: [ + { + fileUri: 'oss://bucket/frame2', + mimeType: 'image/jpeg', + sha256: 'd'.repeat(64), + disclosure: '帧 2/3', + }, + { + fileUri: '', + mimeType: 'image/jpeg', + sha256: 'e'.repeat(64), + disclosure: '帧 3/3', + omission: { reason: 'too big' }, + }, + ], + }); + const parts = [inlinePart('image/png', PNG_BYTES)]; + const result = await processToolResultOmniMedia( + parts, + cfg({ image: true }), + signal, + ); + // [primary disclosure, primary fileData, extra disclosure, extra + // fileData, omitted-extra disclosure, omission notice] — D8 adjacency + // per pair; a violating extra is an explicit omission text Part. + expect(result).toHaveLength(6); + expect(result[0]!.text).toBe('【媒体降质】tool-media.image：帧 1/3'); + expect(result[1]!.fileData?.fileUri).toBe('oss://bucket/frame1'); + expect(result[2]!.text).toBe('【媒体降质】tool-media.image：帧 2/3'); + expect(result[3]!.fileData?.fileUri).toBe('oss://bucket/frame2'); + expect(result[4]!.text).toBe('【媒体降质】tool-media.image：帧 3/3'); + expect(result[5]!.text).toContain('【媒体省略】tool-media.image'); + expect(result[5]!.text).toContain('too big'); + }); + + it('materializes additionalMedia extras even when the primary has no disclosure', async () => { + // The undisclosed-primary branch is separate code from the disclosed + // one — both must splice the extras in. + deliverMock.mockResolvedValue({ + fileUri: 'oss://bucket/frame1', + mimeType: 'image/jpeg', + sha256: 'b'.repeat(64), + recognized: { modality: 'image' }, + tokenEstimate: { + estimatedTokenCount: 1, + method: 'raw-resource-v1', + status: 'ok', + }, + deduped: false, + additionalMedia: [ + { + fileUri: 'oss://bucket/frame2', + mimeType: 'image/jpeg', + sha256: 'd'.repeat(64), + }, + ], + }); + const parts = [inlinePart('image/png', PNG_BYTES)]; + const result = await processToolResultOmniMedia( + parts, + cfg({ image: true }), + signal, + ); + expect(result).toHaveLength(2); + expect(result[0]!.fileData?.fileUri).toBe('oss://bucket/frame1'); + expect(result[1]!.fileData?.fileUri).toBe('oss://bucket/frame2'); + }); + + it('expands a disclosed delivery inside functionResponse.parts', async () => { + deliverMock.mockResolvedValue({ + fileUri: 'oss://bucket/degraded', + mimeType: 'image/jpeg', + sha256: 'b'.repeat(64), + recognized: { modality: 'image' }, + tokenEstimate: { + estimatedTokenCount: 1, + method: 'raw-resource-v1', + status: 'ok', + }, + deduped: false, + disclosure: 'downsampled to 1568px', + degraded: true, + }); + const parts: Part[] = [ + { + functionResponse: { + id: 'call_1', + name: 'Read', + response: { output: 'ok' }, + parts: [ + { text: 'caption' }, + inlinePart('image/png', PNG_BYTES), + ] as Part[], + }, + } as Part, + ]; + const result = await processToolResultOmniMedia( + parts, + cfg({ image: true }), + signal, + ); + const nested = result[0]!.functionResponse?.parts as Part[]; + expect(nested).toHaveLength(3); + expect(nested[0]!.text).toBe('caption'); + expect(nested[1]!.text).toBe( + '【媒体降质】tool-media.image：downsampled to 1568px', + ); + expect(nested[2]!.fileData?.fileUri).toBe('oss://bucket/degraded'); + }); + it('converts media nested inside functionResponse.parts (the production funnel shape)', async () => { // Both physical funnels deliver tool-result media wrapped by // convertToFunctionResponse as {functionResponse: {…, parts: diff --git a/packages/core/src/omni/tool-result-media.ts b/packages/core/src/omni/tool-result-media.ts index 0b15915a3a0..1dbc7bee67b 100644 --- a/packages/core/src/omni/tool-result-media.ts +++ b/packages/core/src/omni/tool-result-media.ts @@ -10,9 +10,15 @@ import path from 'node:path'; import type { Part } from '@google/genai'; import type { Config } from '../config/config.js'; import { createDebugLogger } from '../utils/debugLogger.js'; -import { isOmniDeliveryActive, processMediaForOmniDelivery } from './index.js'; +import { + buildAdditionalMediaParts, + buildTranscriptParts, + isOmniDeliveryActive, + processMediaForOmniDelivery, +} from './index.js'; +import { formatDisclosureText, formatOmissionText } from './disclosure.js'; import { OmniTransportGuardError } from './guard.js'; -import { OmniObjectStore } from './storage.js'; +import { OmniObjectStore, prepareOmniDownloadsDir } from './storage.js'; import { sniffMediaType } from './recognition.js'; const debugLogger = createDebugLogger('omni:tool-result'); @@ -61,11 +67,15 @@ export async function processToolResultOmniMedia( let uploadsRemaining = MAX_UPLOADS_PER_TOOL_RESULT; let uploadBytesRemaining = MAX_UPLOAD_BYTES_PER_TOOL_RESULT; - const convertPart = async (part: Part): Promise => { + /** Returns the replacement Parts for one Part: `[part]` (unchanged), + * `[fileData]`, or `[disclosureText, fileData]` when a fixed policy + * degraded the media — the disclosure must sit IMMEDIATELY before its + * media part (decision D8) so converters can move the pair together. */ + const convertPart = async (part: Part): Promise => { const inline = part.inlineData; - if (!inline?.data || !inline.mimeType) return part; + if (!inline?.data || !inline.mimeType) return [part]; const top = inline.mimeType.split('/')[0]; - if (top !== 'image' && top !== 'audio' && top !== 'video') return part; + if (top !== 'image' && top !== 'audio' && top !== 'video') return [part]; // Sniff the decoded bytes before touching disk — non-media or // unsupported containers stay inline untouched. The SNIFFED modality @@ -74,13 +84,13 @@ export async function processToolResultOmniMedia( // config on the strength of its declared MIME type. const bytes = Buffer.from(inline.data, 'base64'); const sniffed = sniffMediaType(bytes.subarray(0, 4096)); - if (!sniffed) return part; - if (!modalities[sniffed.modality]) return part; + if (!sniffed) return [part]; + if (!modalities[sniffed.modality]) return [part]; if (uploadsRemaining <= 0 || bytes.length > uploadBytesRemaining) { debugLogger.debug( `tool-result media budget exhausted; keeping part inline (${bytes.length} bytes)`, ); - return part; + return [part]; } // Everything from staging-dir setup onward sits inside the try: mkdir @@ -89,28 +99,88 @@ export async function processToolResultOmniMedia( // part leaves THAT part inline — not that the whole tool result rejects, // which would report a tool that succeeded as failed. const store = new OmniObjectStore(config.storage.getQwenDir()); - const stagingDir = path.join(store.getOmniRootDir(), 'downloads'); - const tempPath = path.join( - stagingDir, - `${randomBytes(8).toString('hex')}.part`, - ); + let tempPath: string | undefined; try { - await fs.mkdir(stagingDir, { recursive: true, mode: 0o700 }); + // Symlink-guarded (fail closed → this part stays inline): a link + // planted at downloads/ would redirect the write outside the store. + const stagingDir = await prepareOmniDownloadsDir( + path.join(store.getOmniRootDir(), 'downloads'), + ); + tempPath = path.join( + stagingDir, + `${randomBytes(8).toString('hex')}.part`, + ); await fs.writeFile(tempPath, bytes, { mode: 0o600 }); + const displayName = inline.displayName ?? `tool-media.${top}`; const delivery = await processMediaForOmniDelivery(tempPath, config, { expectedModality: sniffed.modality, signal, + displayName, + origin: 'tool', }); + // §6.2/D8 ordering contract documented on buildTranscriptParts. + const transcriptParts: Part[] = buildTranscriptParts( + displayName, + delivery.transcripts, + ); + // Additional media Parts (multi-output fixed policies): follow the + // primary media slot in every branch below. Each non-omitted extra + // is a real upload the pipeline already performed — charge it + // against the per-result upload-count budget so a multi-output + // policy cannot multiply a tool result's fan-out past the cap + // (extras carry no byte size, so only the count budget applies). + const additionalParts: Part[] = buildAdditionalMediaParts( + displayName, + delivery.additionalMedia, + ); + uploadsRemaining -= + delivery.additionalMedia?.filter((e) => !e.omission).length ?? 0; + if (delivery.omission) { + // Explicit omission (policy design §10.2): the transport guard + // could not bring the part within limits even after the guard + // policies ran — the media is withheld, the notice stands in for + // it, and nothing was uploaded FOR THE PRIMARY (uploaded extras + // were already charged above). + changed = true; + return [ + { text: formatOmissionText(displayName, delivery.omission.reason) }, + ...additionalParts, + ...transcriptParts, + ]; + } + if (!delivery.fileUri && transcriptParts.length > 0) { + // Pure-transcript delivery (§6.2): the policies replaced the media + // with text-only deliverables — nothing was uploaded for the + // primary (uploaded extras were already charged above). The + // primary disclosure (chained prior lossy steps, decision D8) + // still renders: the transcript was derived through those steps. + changed = true; + return delivery.disclosure + ? [ + { text: formatDisclosureText(displayName, delivery.disclosure) }, + ...additionalParts, + ...transcriptParts, + ] + : [...additionalParts, ...transcriptParts]; + } changed = true; uploadsRemaining--; uploadBytesRemaining -= bytes.length; - return { + const fileDataPart: Part = { fileData: { fileUri: delivery.fileUri, mimeType: delivery.mimeType, - displayName: inline.displayName ?? `tool-media.${top}`, + displayName, }, }; + return delivery.disclosure + ? [ + { text: formatDisclosureText(displayName, delivery.disclosure) }, + fileDataPart, + ...additionalParts, + ...transcriptParts, + ] + : [fileDataPart, ...additionalParts, ...transcriptParts]; } catch (err) { if (signal.aborted) throw err; if (err instanceof OmniTransportGuardError) { @@ -121,18 +191,22 @@ export async function processToolResultOmniMedia( // rationale ("produced locally, already in memory") covers only // failures of the *transfer*. changed = true; - return { - text: `[Tool media part withheld by the omni transport guard: ${err.message}]`, - }; + return [ + { + text: `[Tool media part withheld by the omni transport guard: ${err.message}]`, + }, + ]; } debugLogger.debug( `tool-result media upload failed, keeping inline: ${ err instanceof Error ? err.message : String(err) }`, ); - return part; + return [part]; } finally { - await fs.rm(tempPath, { force: true }).catch(() => {}); + if (tempPath !== undefined) { + await fs.rm(tempPath, { force: true }).catch(() => {}); + } } }; @@ -144,8 +218,10 @@ export async function processToolResultOmniMedia( let nestedChanged = false; for (const nestedPart of nested as Part[]) { const converted = await convertPart(nestedPart); - if (converted !== nestedPart) nestedChanged = true; - convertedNested.push(converted); + if (converted.length !== 1 || converted[0] !== nestedPart) { + nestedChanged = true; + } + convertedNested.push(...converted); } if (nestedChanged) { result.push({ @@ -161,7 +237,7 @@ export async function processToolResultOmniMedia( } continue; } - result.push(await convertPart(part)); + result.push(...(await convertPart(part))); } return changed ? result : responseParts; diff --git a/packages/core/src/omni/upload-cache.test.ts b/packages/core/src/omni/upload-cache.test.ts index 14acdf22e7c..9e7c8f59935 100644 --- a/packages/core/src/omni/upload-cache.test.ts +++ b/packages/core/src/omni/upload-cache.test.ts @@ -55,6 +55,28 @@ describe('OmniUploadCache', () => { expect(await cache.get(SHA, 'model-b')).toBeNull(); }); + it('replaces a symlink at the cache path instead of writing through it', async () => { + // A link planted at the cache path (same-UID malware, dotfile + // managers) must never redirect the save onto its target: the atomic + // rename replaces the link itself, leaving the victim file untouched. + // The target holds VALID cache JSON so the operation takes the normal + // load→save path (invalid content would divert into corrupt-backup, + // which renames the link away before any write). + const victimContent = JSON.stringify({ version: 1, entries: {} }); + const victim = path.join(root, 'victim.txt'); + await fs.writeFile(victim, victimContent, { mode: 0o644 }); + const cachePath = path.join(root, 'upload-cache.json'); + await fs.symlink(victim, cachePath); + + const cache = new OmniUploadCache(root); + await cache.put(SHA, 'm', 'oss://bucket/x'); + + expect(await fs.readFile(victim, 'utf8')).toBe(victimContent); + const st = await fs.lstat(cachePath); + expect(st.isSymbolicLink()).toBe(false); + expect(await cache.get(SHA, 'm')).toBe('oss://bucket/x'); + }); + it('keys by scope — a different scope is a miss, same scope hits', async () => { const cacheA = new OmniUploadCache(root, 47, 'scope-a'); const cacheB = new OmniUploadCache(root, 47, 'scope-b'); @@ -81,6 +103,34 @@ describe('OmniUploadCache', () => { expect(after.entries[`${SHA}|m|`]).toBeUndefined(); // pruned }); + it('findSha256ByUrl reverse-maps a delivered URL to its object hash', async () => { + const cache = new OmniUploadCache(root, 47, 'scope-a'); + await cache.put(SHA, 'm1', 'oss://bucket/key1'); + await cache.put('b'.repeat(64), 'm1', 'oss://bucket/key2'); + expect(await cache.findSha256ByUrl('oss://bucket/key1')).toBe(SHA); + expect(await cache.findSha256ByUrl('oss://bucket/key2')).toBe( + 'b'.repeat(64), + ); + expect(await cache.findSha256ByUrl('oss://bucket/unknown')).toBeNull(); + // Scope-agnostic like invalidateByUrl: the caller knows only the URL. + const otherScope = new OmniUploadCache(root, 47, 'scope-b'); + expect(await otherScope.findSha256ByUrl('oss://bucket/key1')).toBe(SHA); + }); + + it('findSha256ByUrl still resolves expired entries (the URL was just sent)', async () => { + const cache = new OmniUploadCache(root); + await cache.put(SHA, 'm', 'oss://bucket/old'); + const file = path.join(root, 'upload-cache.json'); + const data = JSON.parse(await fs.readFile(file, 'utf8')); + for (const entry of Object.values(data.entries) as Array<{ + expiresAt: string; + }>) { + entry.expiresAt = new Date(Date.now() - 1000).toISOString(); + } + await fs.writeFile(file, JSON.stringify(data)); + expect(await cache.findSha256ByUrl('oss://bucket/old')).toBe(SHA); + }); + it('invalidateByUrl drops every entry with that URL', async () => { const cache = new OmniUploadCache(root); await cache.put(SHA, 'm1', 'oss://bucket/shared'); @@ -322,7 +372,7 @@ describe('OmniUploadCache', () => { // content must be byte-identical. expect(await fs.readFile(file, 'utf8')).toBe(before); const names = await fs.readdir(root); - expect(names.filter((n) => n.includes('.tmp-'))).toEqual([]); + expect(names.filter((n) => n.endsWith('.tmp'))).toEqual([]); expect(await cache.get(SHA, 'm')).toBe('oss://bucket/original'); }, ); diff --git a/packages/core/src/omni/upload-cache.ts b/packages/core/src/omni/upload-cache.ts index f76660a2cf1..59c9c4ac99d 100644 --- a/packages/core/src/omni/upload-cache.ts +++ b/packages/core/src/omni/upload-cache.ts @@ -4,38 +4,12 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { randomBytes } from 'node:crypto'; -import fs from 'node:fs/promises'; import path from 'node:path'; import { createDebugLogger } from '../utils/debugLogger.js'; +import { OmniJsonCacheFile } from './json-cache-file.js'; const debugLogger = createDebugLogger('omni:upload-cache'); -/** Per-cache-file operation serializer: cache instances are constructed - * per delivery, and safe-tool batches run deliveries concurrently in one - * process — unserialized load-modify-save would drop entries (worst case - * one extra re-upload, but cheap to prevent). Module scope is deliberate: - * two instances on the same root must share the chain. Cross-process - * writes remain last-writer-wins (documented). */ -const fileOps = new Map>(); - -function serialize(key: string, fn: () => Promise): Promise { - const prev = fileOps.get(key) ?? Promise.resolve(); - const run = prev.then(fn, fn); - const settled = run.then( - () => {}, - () => {}, - ); - fileOps.set(key, settled); - void settled.then(() => { - // Drop the tail once it settles — otherwise the map grows with every - // distinct cache file touched over the process lifetime. Only delete - // when OUR promise is still the tail: a later op may have chained on. - if (fileOps.get(key) === settled) fileOps.delete(key); - }); - return run; -} - /** Default oss:// URL validity horizon: 47h (official 48h minus margin). */ export const DEFAULT_UPLOAD_CACHE_TTL_HOURS = 47; @@ -43,28 +17,21 @@ export const DEFAULT_UPLOAD_CACHE_TTL_HOURS = 47; * so a longer local TTL would confidently serve dead URLs. */ const MAX_UPLOAD_CACHE_TTL_HOURS = 48; -/** Keep at most this many `.corrupt-*` backups (newest wins): a crash - * loop over a corrupt file must not litter the directory without bound. */ -const MAX_CORRUPT_BACKUPS = 2; - interface UploadCacheEntry { ossUrl: string; uploadedAt: string; expiresAt: string; } -interface UploadCacheFile { - version: 1; - /** Key: `||`. */ - entries: Record; -} - /** * Persistent map from object identity to a still-valid DashScope temporary * URL: `(sha256, model, scope) → { ossUrl, uploadedAt, expiresAt }` * (storage design §8). Lives at `.qwen/omni/upload-cache.json`. * - * Invariants: + * File mechanics (serialized ops, atomic writes, corrupt backup+rebuild, + * unreadable-file no-op) live in {@link OmniJsonCacheFile}. Entry + * invariants: + * * - the cache file is the ONLY place an oss:// URL is persisted by omni — * the URL is a delivery cache, never an identity; * - keys include the model: the docs declare uploads model-bound (looser @@ -78,18 +45,11 @@ interface UploadCacheFile { * - expired entries are misses; they are pruned on read and swept * wholesale on every {@link put} (so never-read-again entries cannot * accumulate forever); - * - a corrupt cache file is backed up and rebuilt empty (never fatal), - * keeping at most the newest {@link MAX_CORRUPT_BACKUPS} backups; - * - a cache file that exists but cannot be READ (EACCES, EMFILE, …) makes - * the current operation a no-op instead of an empty-file rebuild — a - * transient read failure must never lead to a save that wipes every - * previously persisted entry; - * - writes are atomic (tmp + rename, 0600) and last-writer-wins across - * processes — acceptable for the experiment (worst case: a lost entry - * causes one extra re-upload). + * - writes are last-writer-wins across processes — acceptable for the + * experiment (worst case: a lost entry causes one extra re-upload). */ export class OmniUploadCache { - private readonly filePath: string; + private readonly file: OmniJsonCacheFile; private readonly ttlMs: number; private readonly scope: string; @@ -98,7 +58,10 @@ export class OmniUploadCache { ttlHours = DEFAULT_UPLOAD_CACHE_TTL_HOURS, scope = '', ) { - this.filePath = path.join(omniRootDir, 'upload-cache.json'); + this.file = new OmniJsonCacheFile( + path.join(omniRootDir, 'upload-cache.json'), + 'omni:upload-cache', + ); // Positive TTLs are clamped to the 48h server URL lifetime — a // configured 168 must not outlive the URL. 0/negative still disables. this.ttlMs = Math.min(ttlHours, MAX_UPLOAD_CACHE_TTL_HOURS) * 3600_000; @@ -110,93 +73,6 @@ export class OmniUploadCache { return this.ttlMs > 0; } - /** - * Load the cache file. Returns null when the file exists but could not - * be read (EACCES, EMFILE, …): the caller must skip its operation for - * this call — proceeding with an empty snapshot and later saving it - * would overwrite N valid entries with one (self-inflicted cache wipe). - * Only a genuinely missing file means empty-and-writable. - */ - private async load(): Promise { - let raw: string; - try { - raw = await fs.readFile(this.filePath, 'utf8'); - } catch (err) { - const code = (err as NodeJS.ErrnoException).code; - // ENOENT: no cache file yet (POSIX and Windows). ENOTDIR: a parent - // path component is a plain file — POSIX raises ENOTDIR where - // Windows reports ENOENT for the same condition. - if (code === 'ENOENT' || code === 'ENOTDIR') { - return { version: 1, entries: {} }; - } - debugLogger.debug( - `upload cache read failed, operation skipped: ${err instanceof Error ? err.message : err}`, - ); - return null; - } - try { - const parsed = JSON.parse(raw) as UploadCacheFile; - // `entries` must be a plain non-null object: `typeof null` and - // `typeof []` are both 'object', and either shape would throw raw - // TypeErrors from every accessor below (escaping the never-fatal - // contract and skipping backup+rebuild). - if ( - parsed?.version === 1 && - typeof parsed.entries === 'object' && - parsed.entries !== null && - !Array.isArray(parsed.entries) - ) { - return parsed; - } - throw new Error('unexpected shape'); - } catch { - // Corrupt cache: preserve for inspection, start fresh. Losing the - // cache only costs re-uploads — never fail the pipeline over it. - const backup = `${this.filePath}.corrupt-${Date.now()}`; - await fs.rename(this.filePath, backup).catch(() => {}); - await this.pruneCorruptBackups(); - debugLogger.debug(`corrupt upload cache backed up to ${backup}`); - return { version: 1, entries: {} }; - } - } - - /** Best-effort: keep only the newest {@link MAX_CORRUPT_BACKUPS}. */ - private async pruneCorruptBackups(): Promise { - const dir = path.dirname(this.filePath); - const prefix = `${path.basename(this.filePath)}.corrupt-`; - try { - const backups = (await fs.readdir(dir)) - .filter((n) => n.startsWith(prefix)) - // Millisecond timestamps are fixed-width for centuries, so the - // lexicographic sort is chronological; newest first. - .sort() - .reverse(); - for (const name of backups.slice(MAX_CORRUPT_BACKUPS)) { - await fs.rm(path.join(dir, name), { force: true }).catch(() => {}); - } - } catch { - // Pruning is hygiene; never let it affect the read path. - } - } - - private async save(data: UploadCacheFile): Promise { - const tmp = `${this.filePath}.tmp-${randomBytes(4).toString('hex')}`; - try { - await fs.mkdir(path.dirname(this.filePath), { - recursive: true, - mode: 0o700, - }); - await fs.writeFile(tmp, JSON.stringify(data, null, 1), { mode: 0o600 }); - await fs.rename(tmp, this.filePath); - } catch (err) { - await fs.rm(tmp, { force: true }).catch(() => {}); - // Cache persistence is best-effort by design. - debugLogger.debug( - `upload cache write failed: ${err instanceof Error ? err.message : err}`, - ); - } - } - private key(sha256: string, model: string): string { return `${sha256}|${model}|${this.scope}`; } @@ -204,35 +80,25 @@ export class OmniUploadCache { /** Valid cached URL or null. Expired entries are pruned on read. */ async get(sha256: string, model: string): Promise { if (!this.enabled) return null; - return serialize(this.filePath, () => this.getInner(sha256, model)); - } - - private async getInner( - sha256: string, - model: string, - ): Promise { - const data = await this.load(); - if (!data) return null; - const k = this.key(sha256, model); - const entry = data.entries[k]; - if (!entry) return null; - const expiresAtMs = Date.parse(entry.expiresAt); - // Malformed timestamps (NaN) must expire, not live forever. - if (!Number.isFinite(expiresAtMs) || expiresAtMs <= Date.now()) { - delete data.entries[k]; - await this.save(data); - return null; - } - return entry.ossUrl; + return this.file.access(null, (entries) => { + const k = this.key(sha256, model); + const entry = entries[k]; + if (!entry) return { result: null }; + const expiresAtMs = Date.parse(entry.expiresAt); + // Malformed timestamps (NaN) must expire, not live forever. + if (!Number.isFinite(expiresAtMs) || expiresAtMs <= Date.now()) { + delete entries[k]; + return { result: null, changed: true }; + } + return { result: entry.ossUrl }; + }); } async put(sha256: string, model: string, ossUrl: string): Promise { if (!this.enabled) return; - return serialize(this.filePath, async () => { - const data = await this.load(); - if (!data) return; + return this.file.access(undefined, (entries) => { const now = Date.now(); - data.entries[this.key(sha256, model)] = { + entries[this.key(sha256, model)] = { ossUrl, uploadedAt: new Date(now).toISOString(), expiresAt: new Date(now + this.ttlMs).toISOString(), @@ -242,11 +108,33 @@ export class OmniUploadCache { // otherwise accumulate forever — every load/save re-parses and // rewrites the whole table, monotonically slowing with dead history. // put() is the natural hook: it already holds the serialized write. - for (const [k, v] of Object.entries(data.entries)) { + for (const [k, v] of Object.entries(entries)) { const t = Date.parse(v.expiresAt); - if (!Number.isFinite(t) || t <= now) delete data.entries[k]; + if (!Number.isFinite(t) || t <= now) delete entries[k]; + } + return { result: undefined, changed: true }; + }); + } + + /** + * Reverse lookup: the object hash behind a delivered oss:// URL. + * + * Serves the reactive server-limit fallback, which only knows the URL + * embedded in the rejected request and must find the local object to + * degrade. Scope-agnostic like {@link invalidateByUrl} (the caller + * knows the URL, not the endpoint scope that minted it); expired + * entries still resolve — the URL was just sent, so the object mapping + * is trustworthy even if the cached URL is past its validity horizon. + */ + async findSha256ByUrl(ossUrl: string): Promise { + return this.file.access(null, (entries) => { + for (const [k, v] of Object.entries(entries)) { + if (v.ossUrl === ossUrl) { + const sha256 = k.split('|', 1)[0]; + if (sha256) return { result: sha256 }; + } } - await this.save(data); + return { result: null }; }); } @@ -260,20 +148,18 @@ export class OmniUploadCache { * not which endpoint scope minted it. */ async invalidateByUrl(ossUrl: string): Promise { - return serialize(this.filePath, async () => { - const data = await this.load(); - if (!data) return; + return this.file.access(undefined, (entries) => { let changed = false; - for (const [k, v] of Object.entries(data.entries)) { + for (const [k, v] of Object.entries(entries)) { if (v.ossUrl === ossUrl) { - delete data.entries[k]; + delete entries[k]; changed = true; } } if (changed) { debugLogger.debug(`invalidated upload cache entries for ${ossUrl}`); - await this.save(data); } + return { result: undefined, changed }; }); } @@ -284,18 +170,16 @@ export class OmniUploadCache { * corrupt for every endpoint. */ async removeBySha256(sha256: string): Promise { - return serialize(this.filePath, async () => { - const data = await this.load(); - if (!data) return; + return this.file.access(undefined, (entries) => { const prefix = `${sha256}|`; let changed = false; - for (const k of Object.keys(data.entries)) { + for (const k of Object.keys(entries)) { if (k.startsWith(prefix)) { - delete data.entries[k]; + delete entries[k]; changed = true; } } - if (changed) await this.save(data); + return { result: undefined, changed }; }); } } diff --git a/packages/core/src/tools/tool-names.ts b/packages/core/src/tools/tool-names.ts index b6478ca4977..3432b82b7e9 100644 --- a/packages/core/src/tools/tool-names.ts +++ b/packages/core/src/tools/tool-names.ts @@ -69,6 +69,16 @@ export const ToolNames = { RECORD_ARTIFACT: 'record_artifact', GET_GOAL: 'get_goal', UPDATE_GOAL: 'update_goal', + // Omni media-policy tools (fixed-policy-only by default; modelAccess + // config can open them to the model). + OMNI_DOWNSAMPLE_IMAGE: 'omni_downsample_image', + OMNI_DOWNSCALE_VIDEO: 'omni_downscale_video', + OMNI_DOWNSAMPLE_AUDIO: 'omni_downsample_audio', + OMNI_EXTRACT_KEYFRAMES: 'omni_extract_keyframes', + OMNI_EXTRACT_AUDIO: 'omni_extract_audio', + OMNI_CLIP_VIDEO: 'omni_clip_video', + OMNI_CONVERT_IMAGE: 'omni_convert_image', + OMNI_TRANSCRIBE_AUDIO: 'omni_transcribe_audio', } as const; /** @@ -123,6 +133,14 @@ export const ToolDisplayNames = { RECORD_ARTIFACT: 'RecordArtifact', GET_GOAL: 'Goal', UPDATE_GOAL: 'UpdateGoal', + OMNI_DOWNSAMPLE_IMAGE: 'DownsampleImage', + OMNI_DOWNSCALE_VIDEO: 'DownscaleVideo', + OMNI_DOWNSAMPLE_AUDIO: 'DownsampleAudio', + OMNI_EXTRACT_KEYFRAMES: 'ExtractKeyframes', + OMNI_EXTRACT_AUDIO: 'ExtractAudio', + OMNI_CLIP_VIDEO: 'ClipVideo', + OMNI_CONVERT_IMAGE: 'ConvertImage', + OMNI_TRANSCRIBE_AUDIO: 'TranscribeAudio', } as const; // Migration from old tool names to new tool names diff --git a/packages/core/src/tools/tool-registry.test.ts b/packages/core/src/tools/tool-registry.test.ts index 74ab9f712b0..46524364c80 100644 --- a/packages/core/src/tools/tool-registry.test.ts +++ b/packages/core/src/tools/tool-registry.test.ts @@ -17,6 +17,7 @@ import { mcpToTool } from '@google/genai'; import { spawn } from 'node:child_process'; import fs from 'node:fs'; import { MockTool } from '../test-utils/mock-tool.js'; +import type { MediaPolicyToolDescriptor } from './tools.js'; import { CHARS_PER_TOKEN } from '../services/tokenEstimation.js'; import { McpClientManager } from './mcp-client-manager.js'; @@ -384,6 +385,78 @@ describe('ToolRegistry', () => { }); }); + describe('media-policy tool visibility', () => { + class MockMediaPolicyTool extends MockTool { + override get mediaPolicyDescriptor(): MediaPolicyToolDescriptor { + return { + kind: 'media_policy', + inputMediaTypes: ['image'], + outputs: [{ kind: 'media', required: true }], + }; + } + } + + const enabledConfig = () => + new Config({ + ...baseConfigParams, + omniPolicyTools: { + omni_compress_image: { modelAccess: { enabled: true } }, + }, + }); + + it('excludes media-policy tools from getFunctionDeclarations by default', () => { + toolRegistry.registerTool(new MockTool({ name: 'visible' })); + toolRegistry.registerTool( + new MockMediaPolicyTool({ name: 'omni_compress_image' }), + ); + + const names = toolRegistry.getFunctionDeclarations().map((d) => d.name); + expect(names).toEqual(['visible']); + }); + + it('keeps media-policy tools hidden even with includeDeferred: true', () => { + // agent-core's wildcard/default branches call + // getFunctionDeclarations({ includeDeferred: true }); the media-policy + // filter must hold there too. + toolRegistry.registerTool( + new MockMediaPolicyTool({ name: 'omni_compress_image' }), + ); + + const names = toolRegistry + .getFunctionDeclarations({ includeDeferred: true }) + .map((d) => d.name); + expect(names).toEqual([]); + }); + + it('excludes media-policy tools from getFunctionDeclarationsFiltered even when named explicitly', () => { + toolRegistry.registerTool(new MockTool({ name: 'visible' })); + toolRegistry.registerTool( + new MockMediaPolicyTool({ name: 'omni_compress_image' }), + ); + + const names = toolRegistry + .getFunctionDeclarationsFiltered(['visible', 'omni_compress_image']) + .map((d) => d.name); + expect(names).toEqual(['visible']); + }); + + it('declares media-policy tools when modelAccess.enabled is true', () => { + const registry = new ToolRegistry(enabledConfig()); + registry.registerTool( + new MockMediaPolicyTool({ name: 'omni_compress_image' }), + ); + + expect(registry.getFunctionDeclarations().map((d) => d.name)).toEqual([ + 'omni_compress_image', + ]); + expect( + registry + .getFunctionDeclarationsFiltered(['omni_compress_image']) + .map((d) => d.name), + ).toEqual(['omni_compress_image']); + }); + }); + describe('deferred tool filtering', () => { it('sorts visible function declarations by canonical name', () => { toolRegistry.registerTool(new MockTool({ name: 'zeta' })); diff --git a/packages/core/src/tools/tool-registry.ts b/packages/core/src/tools/tool-registry.ts index 2d6dc129ba9..06cd0a2978f 100644 --- a/packages/core/src/tools/tool-registry.ts +++ b/packages/core/src/tools/tool-registry.ts @@ -13,6 +13,7 @@ import type { } from './tools.js'; import { Kind, BaseDeclarativeTool, BaseToolInvocation } from './tools.js'; import { type Config, matchesAnyServerPattern } from '../config/config.js'; +import { isMediaPolicyToolHiddenFromModel } from '../omni/policy/model-access.js'; import { spawn } from 'node:child_process'; import { StringDecoder } from 'node:string_decoder'; import type { SendSdkMcpMessage } from './mcp-client.js'; @@ -734,16 +735,22 @@ export class ToolRegistry { includeDeferred?: boolean; }): FunctionDeclaration[] { const includeDeferred = options?.includeDeferred === true; - return Array.from(this.tools.values()) - .filter( - (tool) => - includeDeferred || - !tool.shouldDefer || - tool.alwaysLoad || - !this.isDeferredAndHidden(tool.name), - ) - .sort(ToolRegistry.compareToolsByDeclarationName) - .map((tool) => tool.schema); + return ( + Array.from(this.tools.values()) + .filter( + (tool) => + includeDeferred || + !tool.shouldDefer || + tool.alwaysLoad || + !this.isDeferredAndHidden(tool.name), + ) + // Omni media-policy tools without modelAccess.enabled are registered + // (the fixed-policy orchestrator needs them) but never declared to + // the model — including for subagents, which force includeDeferred. + .filter((tool) => !isMediaPolicyToolHiddenFromModel(this.config, tool)) + .sort(ToolRegistry.compareToolsByDeclarationName) + .map((tool) => tool.schema) + ); } /** @@ -903,7 +910,10 @@ export class ToolRegistry { const declarations: FunctionDeclaration[] = []; for (const name of toolNames) { const tool = this.tools.get(name); - if (tool) { + // Same modelAccess gate as getFunctionDeclarations: an explicit + // subagent tool list must not become a leak path for media-policy + // tools the model can't call. + if (tool && !isMediaPolicyToolHiddenFromModel(this.config, tool)) { declarations.push(tool.schema); } } diff --git a/packages/core/src/tools/tool-search.test.ts b/packages/core/src/tools/tool-search.test.ts index 065d7480767..92dd3f7582e 100644 --- a/packages/core/src/tools/tool-search.test.ts +++ b/packages/core/src/tools/tool-search.test.ts @@ -12,7 +12,7 @@ import { ToolRegistry } from './tool-registry.js'; import { DiscoveredMCPTool } from './mcp-tool.js'; import { MockTool } from '../test-utils/mock-tool.js'; import { ToolSearchTool, scoreTool, tokenize } from './tool-search.js'; -import type { ToolResult } from './tools.js'; +import type { MediaPolicyToolDescriptor, ToolResult } from './tools.js'; import { CronCreateTool } from './cron-create.js'; import { CronDeleteTool } from './cron-delete.js'; import { CronListTool } from './cron-list.js'; @@ -252,6 +252,95 @@ describe('ToolSearchTool', () => { expect(registry.isDeferredToolRevealed('bravo')).toBe(true); }); + describe('media-policy tool hiding', () => { + class MockMediaPolicyTool extends MockTool { + override get mediaPolicyDescriptor(): MediaPolicyToolDescriptor { + return { + kind: 'media_policy', + inputMediaTypes: ['image'], + outputs: [{ kind: 'media', required: true }], + }; + } + } + + const registerPolicyTool = (reg: ToolRegistry) => { + reg.registerTool( + new MockMediaPolicyTool({ + name: 'omni_compress_image', + description: 'compress an image to a target size', + shouldDefer: true, + }), + ); + }; + + function makeEnabledConfigWithRegistry(): { + config: Config; + registry: ToolRegistry; + } { + const enabledConfig = new Config({ + ...baseConfigParams, + omniPolicyTools: { + omni_compress_image: { modelAccess: { enabled: true } }, + }, + }); + const enabledRegistry = new ToolRegistry(enabledConfig); + vi.spyOn(enabledConfig, 'getToolRegistry').mockReturnValue( + enabledRegistry, + ); + vi.spyOn(enabledConfig, 'getGeminiClient').mockReturnValue({ + setTools: vi.fn().mockResolvedValue(undefined), + } as never); + return { config: enabledConfig, registry: enabledRegistry }; + } + + it('keyword search never surfaces a hidden media-policy tool', async () => { + registerPolicyTool(registry); + + const tool = new ToolSearchTool(config); + const result = await tool + .build({ query: 'compress image' }) + .execute(new AbortController().signal); + + expect(String(result.llmContent)).toContain('No tools found'); + expect(String(result.llmContent)).not.toContain('omni_compress_image'); + }); + + it('select: mode blocks a hidden media-policy tool without revealing it', async () => { + registerPolicyTool(registry); + + const tool = new ToolSearchTool(config); + const result = await tool + .build({ query: 'select:omni_compress_image' }) + .execute(new AbortController().signal); + + const content = String(result.llmContent); + expect(content).toContain('media policy tool'); + expect(content).not.toContain(''); + expect(result.error?.message).toContain('media policy tool'); + expect(registry.isDeferredToolRevealed('omni_compress_image')).toBe( + false, + ); + }); + + it('surfaces the tool in both modes once modelAccess.enabled is true', async () => { + const { config: enabledConfig, registry: enabledRegistry } = + makeEnabledConfigWithRegistry(); + registerPolicyTool(enabledRegistry); + + const tool = new ToolSearchTool(enabledConfig); + const keywordResult = await tool + .build({ query: 'compress image' }) + .execute(new AbortController().signal); + expect(String(keywordResult.llmContent)).toContain( + '"name":"omni_compress_image"', + ); + + expect( + enabledRegistry.isDeferredToolRevealed('omni_compress_image'), + ).toBe(true); + }); + }); + it('keyword search returns top-N ranked tools', async () => { registry.registerTool( new MockTool({ diff --git a/packages/core/src/tools/tool-search.ts b/packages/core/src/tools/tool-search.ts index e923b4ab876..a788ac6418d 100644 --- a/packages/core/src/tools/tool-search.ts +++ b/packages/core/src/tools/tool-search.ts @@ -36,6 +36,7 @@ import { isLeaderOnlyToolUnavailableInSubagent, isPlanLifecycleToolUnavailableInSubagent, } from '../agents/runtime/subagent-plan-tool-policy.js'; +import { isMediaPolicyToolHiddenFromModel } from '../omni/policy/model-access.js'; const debugLogger = createDebugLogger('TOOL_SEARCH'); @@ -246,9 +247,13 @@ class ToolSearchInvocation extends BaseToolInvocation< */ private collectCandidates(): AnyDeclarativeTool[] { const registry = this.config.getToolRegistry(); - return registry - .getAllTools() - .filter((t) => registry.isDeferredAndHidden(t.name)); + return registry.getAllTools().filter( + (t) => + registry.isDeferredAndHidden(t.name) && + // Media-policy tools without modelAccess.enabled must never be + // surfaced to the model — not even via keyword discovery. + !isMediaPolicyToolHiddenFromModel(this.config, t), + ); } private async loadAndReturnSchemas( @@ -320,6 +325,14 @@ class ToolSearchInvocation extends BaseToolInvocation< missing.push(requested); continue; } + // Hidden media-policy tools cannot be revealed by exact-name lookup + // either: modelAccess.enabled is the only switch that exposes them. + // Blocking here (after ensureTool, which is where the descriptor + // becomes inspectable) guarantees no schema reveal happens below. + if (isMediaPolicyToolHiddenFromModel(this.config, tool)) { + blocked.push(canonical); + continue; + } // Only reveal + count toward the setTools() trigger when the tool // is actually deferred. `select:` mode also accepts already-loaded // / alwaysLoad tools (the model may use it to re-inspect a schema) @@ -424,7 +437,9 @@ class ToolSearchInvocation extends BaseToolInvocation< const blockedMessages = blocked.map((name) => isLeaderOnlyToolUnavailableInSubagent(name) ? getLeaderOnlyToolUnavailableMessage(name) - : getSubagentPlanToolUnavailableMessage(name), + : isPlanLifecycleToolUnavailableInSubagent(name) + ? getSubagentPlanToolUnavailableMessage(name) + : `Tool "${name}" is a media policy tool and is not available to the model.`, ); blockedErrorMessage = blockedMessages.join('\n'); const header = llmContent ? '\n\n' : ''; diff --git a/packages/core/src/tools/tools.ts b/packages/core/src/tools/tools.ts index 23d0e8efb0c..e0eeef7b3c2 100644 --- a/packages/core/src/tools/tools.ts +++ b/packages/core/src/tools/tools.ts @@ -152,6 +152,62 @@ export abstract class BaseToolInvocation< */ export type AnyToolInvocation = ToolInvocation; +/** One declared output of a media-policy tool (see + * {@link MediaPolicyToolDescriptor}). */ +export interface MediaPolicyToolOutputSpec { + /** What the output is: a derived media artifact, a disclosure text, or + * a non-media file artifact (e.g. a `role: 'transcript'` UTF-8 + * text/plain file — policy design §6.2). */ + kind: 'media' | 'text' | 'file'; + /** Role label for text/file outputs (e.g. 'disclosure', 'transcript'). */ + role?: string; + /** MIME types the output may carry (media and file outputs). */ + mimeTypes?: string[]; + /** Whether a successful run MUST produce this output. */ + required: boolean; + /** Whether the output is a lossy transformation of its input. A lossy + * media output obligates a disclosure text alongside it. */ + lossy?: boolean; +} + +/** + * Code-registration fact marking a tool as an omni media-policy tool — + * declared by the tool class itself, immutable at runtime, and never + * configurable. Its presence is what the scheduler's modelAccess gate, + * the declaration surfaces, and the fixed-policy orchestrator key off: + * config can never turn an ordinary tool into a policy tool (or the + * reverse). + */ +export interface MediaPolicyToolDescriptor { + kind: 'media_policy'; + /** Media modalities the tool accepts as input. */ + inputMediaTypes: Array<'image' | 'audio' | 'video'>; + /** Outputs a successful run may/must produce. */ + outputs: MediaPolicyToolOutputSpec[]; + /** JSON schema for `omni.processing.policyTools..settings`. */ + settingsSchema?: object; + /** + * Parameter names only the OPERATOR may set — via + * `policyTools..settings` or `modelAccess.defaultArguments` / + * `lockedArguments` — never the caller of a gated model/client call. + * For endpoint/credential selectors (e.g. a request base URL plus the + * NAME of the env var read for its bearer token): a model-controlled + * pair would let injected content exfiltrate arbitrary environment + * secrets to an attacker host. The modelAccess gate rejects gated calls + * that name these keys, and the declaration projection hides them from + * the model. Fixed-policy arguments (operator-authored settings.json) + * are unaffected. + */ + operatorOnlyParams?: readonly string[]; + /** + * Transform-semantics version, part of the degradation-cache + * fingerprint (decision D2). Bump it whenever the tool starts producing + * different bytes for the same input and arguments (encoder change, + * default pipeline change), so stale cached derivatives are not reused. + */ + version?: string; +} + /** * Interface for a tool builder that validates parameters and creates invocations. */ @@ -248,6 +304,16 @@ export abstract class DeclarativeTool< }; } + /** + * Present iff this tool is an omni media-policy tool. A code-level fact + * of the tool class (not configuration): the scheduler's modelAccess + * gate, the declaration surfaces, and the fixed-policy orchestrator all + * key off it. Default: not a media-policy tool. + */ + get mediaPolicyDescriptor(): MediaPolicyToolDescriptor | undefined { + return undefined; + } + /** * Max model-facing characters for this tool's output before the scheduler * spills it to disk (mirrors Claude Code's per-tool `maxResultSizeChars`). diff --git a/packages/core/src/utils/contextLengthError.test.ts b/packages/core/src/utils/contextLengthError.test.ts index 3cb565b8411..c9eeae26ddb 100644 --- a/packages/core/src/utils/contextLengthError.test.ts +++ b/packages/core/src/utils/contextLengthError.test.ts @@ -39,6 +39,17 @@ describe('contextLengthError', () => { expect(isContextLengthExceededError(new Error(message))).toBe(false); }); + it('parses the DashScope input-range upper bound as the limit', () => { + const info = getContextLengthExceededInfo( + new Error( + '<400> InternalError.Algo.InvalidParameter: Range of input length should be [1, 196608]', + ), + ); + expect(info.isExceeded).toBe(true); + expect(info.limitTokens).toBe(196608); + expect(info.actualTokens).toBeUndefined(); + }); + it('parses prompt-too-long actual and limit token counts', () => { const info = getContextLengthExceededInfo( new Error('prompt is too long: 137500 tokens > 135000 maximum'), diff --git a/packages/core/src/utils/contextLengthError.ts b/packages/core/src/utils/contextLengthError.ts index 4a4df642b6f..fbc2bea1acc 100644 --- a/packages/core/src/utils/contextLengthError.ts +++ b/packages/core/src/utils/contextLengthError.ts @@ -77,6 +77,18 @@ function parseTokenCounts(text: string): { }; } + // DashScope: "Range of input length should be [1, 196608]" — the upper + // bound is the server's REAL input ceiling (observed to differ from the + // configured context window), so parse it out for the retry paths. + const rangeMatch = text.match( + /range of input length should be\s*\[\s*\d[\d,]*\s*,\s*(\d[\d,]*)\s*\]/i, + ); + if (rangeMatch) { + return { + limitTokens: parseInteger(rangeMatch[1]!), + }; + } + return {}; } diff --git a/packages/core/src/utils/fileUtils.test.ts b/packages/core/src/utils/fileUtils.test.ts index d8f4efb9e0d..04b67bb9048 100644 --- a/packages/core/src/utils/fileUtils.test.ts +++ b/packages/core/src/utils/fileUtils.test.ts @@ -915,6 +915,22 @@ describe('fileUtils', () => { expect(await detectFileType('tutorial.m4v')).toBe('video'); }); + it.each([ + ['movie.mkv', 'video'], + ['clip.avi', 'video'], + ['song.flac', 'audio'], + ['stream.aac', 'audio'], + ] as const)( + 'should detect %s via the mime/lite override map as %s', + async (fileName, expected) => { + // Same mime/lite gap as .m4v: the standard database returns null for + // these container extensions, so only the override map keeps a real + // media file out of the binary content sampler. + mockMimeGetType.mockReturnValueOnce(null); + expect(await detectFileType(fileName)).toBe(expected); + }, + ); + it('should detect known binary extensions as binary (e.g. .zip)', async () => { mockMimeGetType.mockReturnValueOnce('application/zip'); expect(await detectFileType('archive.zip')).toBe('binary'); @@ -1682,6 +1698,28 @@ describe('fileUtils', () => { expect(result.returnDisplay).toContain('Read video file'); }); + it('processes an .mkv video as inline data despite the mime/lite gap', async () => { + // Same regression class as .m4v: mime/lite's standard database has no + // .mkv entry, so without the override map a Matroska movie fell into + // the binary/size-cap path instead of the media pipeline. + const fakeVideo = Buffer.from('fake mkv data'); + const testVideoPath = path.join(tempRootDir, 'movie.mkv'); + actualNodeFs.writeFileSync(testVideoPath, fakeVideo); + mockMimeGetType.mockReturnValue(null); + + const result = await processSingleFileContent(testVideoPath, mockConfig); + + expect(typeof result.llmContent).toBe('object'); + expect( + (result.llmContent as { inlineData: { data: string } }).inlineData.data, + ).toBe(fakeVideo.toString('base64')); + expect( + (result.llmContent as { inlineData: { mimeType: string } }).inlineData + .mimeType, + ).toBe('video/x-matroska'); + expect(result.returnDisplay).toContain('Read video file'); + }); + it('should fall back to pdftotext when model does not support PDF', async () => { const fakePdfData = Buffer.from('fake pdf data'); actualNodeFs.writeFileSync(testPdfFilePath, fakePdfData); diff --git a/packages/core/src/utils/fileUtils.ts b/packages/core/src/utils/fileUtils.ts index e0687583c5c..5d5a8394f3a 100644 --- a/packages/core/src/utils/fileUtils.ts +++ b/packages/core/src/utils/fileUtils.ts @@ -784,14 +784,21 @@ function isTextMime(lookedUpMimeType: string): boolean { } /** - * Video containers whose MIME type `mime/lite` does not carry in its default - * "standard" database. `.m4v`'s `video/x-m4v` mapping lives only in the - * non-default "other" set, so `mime.getType('clip.m4v')` returns null and — - * without this override — {@link detectFileType} falls through to the content - * sampler and misclassifies a real video as binary. + * Media containers whose MIME type `mime/lite` does not carry in its default + * "standard" database (the mappings below live only in the non-default + * "other" set), so `mime.getType()` returns null and — without this + * override — {@link detectFileType} falls through to the content sampler and + * misclassifies a real video/audio file as binary. Scope rule: only list + * extensions whose bytes the omni recognizer can also sniff-confirm + * (Matroska/EBML, RIFF/AVI, FLAC, ADTS AAC), so a lie-by-extension still + * falls back to the legacy path instead of entering media delivery. */ -const MIME_LITE_MISSING_VIDEO_TYPES: ReadonlyMap = new Map([ +const MIME_LITE_MISSING_MEDIA_TYPES: ReadonlyMap = new Map([ ['.m4v', 'video/x-m4v'], + ['.mkv', 'video/x-matroska'], + ['.avi', 'video/x-msvideo'], + ['.flac', 'audio/x-flac'], + ['.aac', 'audio/x-aac'], ]); /** @@ -821,11 +828,11 @@ export async function detectFileType(filePath: string): Promise { } // Returns null if not found, or the mime type string. `mime/lite` omits a - // few video containers (see MIME_LITE_MISSING_VIDEO_TYPES), so fall back to + // few media containers (see MIME_LITE_MISSING_MEDIA_TYPES), so fall back to // that override before giving up — otherwise a real video falls through to // the content sampler and is misclassified as binary. const lookedUpMimeType = - mime.getType(filePath) ?? MIME_LITE_MISSING_VIDEO_TYPES.get(ext) ?? null; + mime.getType(filePath) ?? MIME_LITE_MISSING_MEDIA_TYPES.get(ext) ?? null; if (lookedUpMimeType) { if (lookedUpMimeType.startsWith('image/')) { return 'image'; @@ -1113,7 +1120,7 @@ export async function processSingleFileContent( const fileType = await detectFileType(filePath); const mediaMimeType = mime.getType(filePath) ?? - MIME_LITE_MISSING_VIDEO_TYPES.get(path.extname(filePath).toLowerCase()) ?? + MIME_LITE_MISSING_MEDIA_TYPES.get(path.extname(filePath).toLowerCase()) ?? 'application/octet-stream'; const shouldRenderImageOverview = fileType === 'image' && CANONICAL_IMAGE_MIME_TYPES.has(mediaMimeType); @@ -1292,7 +1299,7 @@ export async function processSingleFileContent( // 100 MB source cap protects the overview DECODER, so it only applies // when the overview will actually decode — i.e. when omni is not taking // this file. The omni path uploads original bytes without decoding and - // enforces its own omni.upload.maxFileBytes ceiling (1 GiB default); + // enforces its own maxUploadFileBytes ceiling (1 GiB default); // gating it here too would reject a 150 MB PNG while delivering a // 500 MB GIF, purely on whether the format has an overview renderer. if ( diff --git a/packages/vscode-ide-companion/schemas/settings.schema.json b/packages/vscode-ide-companion/schemas/settings.schema.json index 8dcdf71cec2..71ebe6639c2 100644 --- a/packages/vscode-ide-companion/schemas/settings.schema.json +++ b/packages/vscode-ide-companion/schemas/settings.schema.json @@ -3268,45 +3268,157 @@ "type": "boolean", "default": false }, - "upload": { - "description": "Upload-channel limits for omni media delivery.", + "processing": { + "description": "Media policy processing: fixed-policy orchestration, transport guard, per-root derivation limits, and policy tool overrides.", "type": "object", "properties": { - "maxFileBytes": { - "type": "number", - "minimum": 1, - "default": 1073741824, - "description": "Per-file byte ceiling for omni media uploads. Defaults to 1 GiB, the DashScope temporary-upload per-file cap. Inputs above the limit fail closed with an explanatory error." + "limits": { + "description": "Per-invocation derivation budgets. Exceeding a budget stops further derivation for that root resource (already committed artifacts stand).", + "type": "object", + "properties": { + "maxConcurrentResources": { + "type": "number", + "minimum": 1, + "default": 1, + "description": "Number of media resources processed by policies in parallel within one request." + }, + "reservedOutputTokens": { + "type": "number", + "minimum": 0, + "default": 8192, + "description": "Tokens reserved for model output when computing session.availableContextTokens for when-conditions." + }, + "maxLineageDepth": { + "type": "number", + "minimum": 1, + "default": 8, + "description": "Maximum derivation chain length from a root resource." + }, + "maxPolicyRunsPerRoot": { + "type": "number", + "minimum": 1, + "default": 64, + "description": "Maximum policy invocations attributable to one root resource within a single orchestrator run." + }, + "maxArtifactsPerRoot": { + "type": "number", + "minimum": 1, + "default": 256, + "description": "Maximum derived artifacts attributable to one root resource within a single orchestrator run." + }, + "maxDerivedBytesPerRoot": { + "type": "number", + "minimum": 1, + "default": 1073741824, + "description": "Byte budget for derived artifacts per root resource within a single orchestrator run. Defaults to 1 GiB." + }, + "maxTransportPasses": { + "type": "number", + "minimum": 1, + "default": 3, + "description": "Maximum transport-guard policy passes per resource before the media is removed with an explicit omission note." + } + } }, - "cacheTtlHours": { - "type": "number", - "minimum": 0, - "default": 47, - "description": "Validity horizon for cached oss:// upload URLs. DashScope temporary uploads live 48h; the default keeps a 1h margin. 0 disables the upload cache (every delivery re-uploads)." + "fixedPolicies": { + "description": "User fixed policies keyed by policy id. There are no built-in default policies: nothing runs unless configured here. Across settings scopes entries merge by id (whole-entry replacement); a null entry tombstones a policy from a lower-priority scope. Validated and normalized at startup.", + "type": "object", + "additionalProperties": true + }, + "transportGuard": { + "description": "Delivery-boundary enforcement: hard limits plus mandatory guard policies applied when the final delivery set still exceeds limits. Cannot be disabled.", + "type": "object", + "properties": { + "maxUploadFileBytes": { + "type": "number", + "minimum": 1, + "maximum": 1073741824, + "default": 1073741824, + "description": "Per-file byte ceiling for omni media uploads. Defaults to 1 GiB, the DashScope temporary-upload per-file cap (values above it are a startup configuration error). Media still above the limit after guard policies fail closed with an explanatory error." + }, + "maxEstimatedTokens": { + "type": "number", + "minimum": 0, + "default": 0, + "description": "Estimated-token ceiling for a single omni media input, checked at the delivery boundary using the versioned raw-resource estimator. 0 disables the token guard — the estimation formula is pending confirmation with the model provider; set a positive threshold to enforce fail-closed rejection." + }, + "policies": { + "description": "Guard policies keyed by policy id, run only when the final delivery set exceeds transport limits. Merged with system defaults by id. The merged set must cover image, video, and audio and must not be empty; every policy output must use source: omit.", + "type": "object", + "additionalProperties": true + } + } + }, + "policyTools": { + "description": "Per-tool overrides keyed by policy tool name: settings (default arguments), runtime (timeoutMs), and modelAccess (enabled, defaultArguments, lockedArguments, parameterSchema, output).", + "type": "object", + "additionalProperties": true } } }, - "transport": { - "description": "Transport guard dimensions beyond the byte ceiling for omni media delivery.", + "delivery": { + "description": "Model-delivery settings for omni media.", "type": "object", "properties": { - "maxEstimatedTokens": { - "type": "number", - "minimum": 0, - "default": 0, - "description": "Estimated-token ceiling for a single omni media input, checked before upload using the versioned raw-resource estimator. 0 disables the token guard — the estimation formula is pending confirmation with the model provider; set a positive threshold to enforce fail-closed rejection." + "upload": { + "description": "Upload-channel delivery settings.", + "type": "object", + "properties": { + "urlTtlHours": { + "type": "number", + "minimum": 0, + "default": 47, + "description": "Validity horizon for cached oss:// upload URLs. DashScope temporary uploads live 48h; the default keeps a 1h margin. 0 disables the upload cache (every delivery re-uploads)." + } + } } } }, - "download": { - "description": "URL media localization limits for omni delivery.", + "ingestion": { + "description": "Media input ingestion settings for omni delivery.", "type": "object", "properties": { - "maxFileBytes": { - "type": "number", - "minimum": 0, - "default": 0, - "description": "Byte ceiling for downloading URL media inputs. 0 or unset follows omni.upload.maxFileBytes (downloading more than the upload channel can deliver is pointless)." + "localization": { + "description": "Remote-media localization settings.", + "type": "object", + "properties": { + "url": { + "description": "URL media download settings.", + "type": "object", + "properties": { + "maxFileBytes": { + "type": "number", + "minimum": 0, + "default": 0, + "description": "Byte ceiling for downloading URL media inputs. 0 or unset follows omni.processing.transportGuard.maxUploadFileBytes (downloading more than the upload channel can deliver is pointless)." + } + } + } + } + } + } + }, + "storage": { + "description": "Managed storage settings under .qwen/omni/.", + "type": "object", + "properties": { + "quarantine": { + "description": "Retention for failed policy invocations moved to .qwen/omni/quarantine/ for diagnosis. Quarantined content is never recalled into recognition or delivery.", + "type": "object", + "properties": { + "retentionDays": { + "type": "number", + "minimum": 1, + "default": 7, + "description": "Days a quarantined invocation directory is kept before startup recovery removes it. Must be at least 1; non-positive values fall back to the default." + }, + "maxBytes": { + "type": "number", + "minimum": 1, + "default": 5368709120, + "description": "Total byte budget for the quarantine directory. Startup recovery removes oldest entries first until within budget. Defaults to 5 GiB. Must be at least 1; non-positive values fall back to the default." + } + } } } } diff --git a/packages/web-shell/client/components/messages/toolFormatting.ts b/packages/web-shell/client/components/messages/toolFormatting.ts index 2708f879ab8..6433a483aff 100644 --- a/packages/web-shell/client/components/messages/toolFormatting.ts +++ b/packages/web-shell/client/components/messages/toolFormatting.ts @@ -58,6 +58,14 @@ export const TOOL_DISPLAY_NAMES: Record = { record_artifact: 'RecordArtifact', web_search: 'WebSearch', image_gen: 'ImageGen', + omni_downsample_image: 'DownsampleImage', + omni_downscale_video: 'DownscaleVideo', + omni_downsample_audio: 'DownsampleAudio', + omni_extract_keyframes: 'ExtractKeyframes', + omni_extract_audio: 'ExtractAudio', + omni_clip_video: 'ClipVideo', + omni_convert_image: 'ConvertImage', + omni_transcribe_audio: 'TranscribeAudio', bash: 'Shell', shell: 'Shell Command', read: 'ReadFile', diff --git a/packages/web-shell/client/i18n.tsx b/packages/web-shell/client/i18n.tsx index fa41c47b7d2..776635c81cc 100644 --- a/packages/web-shell/client/i18n.tsx +++ b/packages/web-shell/client/i18n.tsx @@ -2665,6 +2665,14 @@ const ZH: Messages = { 'toolName.artifact': '制品', 'toolName.record_artifact': '记录制品', 'toolName.image_gen': '生成图片', + 'toolName.omni_downsample_image': '降采样图像', + 'toolName.omni_downscale_video': '降采样视频', + 'toolName.omni_downsample_audio': '降采样音频', + 'toolName.omni_extract_keyframes': '提取关键帧', + 'toolName.omni_extract_audio': '提取音轨', + 'toolName.omni_clip_video': '剪辑视频', + 'toolName.omni_convert_image': '转换图像', + 'toolName.omni_transcribe_audio': '转写音频', // web-shell-only wire aliases (see TOOL_DISPLAY_NAMES in toolFormatting.ts) 'toolName.bash': '运行命令', 'toolName.shell': 'Shell 命令',