diff --git a/docs/design/2026-07-29-omni-multimodal-memory.md b/docs/design/2026-07-29-omni-multimodal-memory.md index 63f37b28f50..5d3e0f632fc 100644 --- a/docs/design/2026-07-29-omni-multimodal-memory.md +++ b/docs/design/2026-07-29-omni-multimodal-memory.md @@ -507,6 +507,23 @@ interface MediaMemoryRecallRequest { 召回返回的可用媒体衍生物会绑定出当前 session 的 `resourceId`，Agent 随后可把它传给 另一个 MediaPolicyTool。召回本身不复制文件、不创建新 File，也不触发 policy。 +#### 9.2.1 "按路径/内容 hash 查询"的一致解读 + +issue #8188 同一条验收项里写了"按路径/内容 hash 查询"与"返回中永不暴露真实本地路径"。 +按本设计 §5.2/§15 的路径隔离不变量，二者的一致解读是**按调用方分层**： + +| 调用方 | 允许的入口 | 理由 | +| ------------------------------------- | ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 模型（active Tool / passive selector) | 仅 session `resourceId` | 句柄是不可伪造的能力凭证：模型只能问本会话真的投递给它的媒体。放开路径入参会让媒体文件里的注入内容驱使模型探测任意路径，而召回返回的是转写正文——典型 confused-deputy 提权；即使 miss，"有/无记忆"本身也是存在性泄露 | +| Harness（受管管线内部) | 路径 / 内容 hash | 不经模型，无提权面。已由 `findBindingBySha256` 提供并被反应式降质阶梯使用 | + +因此"系统能按路径/hash 找到记忆"这一能力是满足的，只是不向模型暴露。 + +代价与已知缺口：句柄只在投递时签发，而投递要求文件存在——所以在新 session 中，被 +删除或移动的文件无法再获得句柄，其记忆暂时不可达。补法不是放开模型的路径入参，而是 +在**授权层**补：让 `@` 引用一个"记忆里有、磁盘上没了"的文件也能凭记录的 +`fileRef`+`sha256` 签发句柄（用户的 `@` 引用即授权）。留作后续阶段。 + ### 9.3 被动 side-query 召回 被动召回只处理当前请求已经显式引用并完成识别的文件。它不从普通文本猜路径，也不 @@ -551,6 +568,12 @@ interface MediaMemoryRecallResult { kind: 'metadata' | 'derived_media' | 'policy_result' | 'execution'; role?: string; content?: string; + /** content 是 memory 所存文本的前缀时为 true（被 recall.maxTextChars 截断）。 + * coverage 讲的是"处理过什么"，所以一条被截断的转写仍合法地报 complete； + * 没有这个标记，模型看到前缀 + 完整覆盖 + 无 gap，就会对它从未读到的后半段 + * 音频作答。默认值本身就会撞上：maxTextChars（24000 字符）小于收集期界 + * （65536 字节），任何长转写都在读取期被切。 */ + contentTruncated?: boolean; resourceId?: string; scope: MediaScope; channels: MediaChannel[]; @@ -578,6 +601,10 @@ interface MediaMemoryRecallResult { arguments: Record; reason: string; }>; + /** 匹配总数，仅当条目预算把列表截短时出现。没有它，一页被截断的结果与穷尽结果 + * 无法区分：真实审计在 limit: 12 下读到 6 段 clip，就得出"从未抽过关键帧"， + * 而库里有 72 条——读的人没有说谎，它只是无从知道自己看到的是一页。 */ + matchedEntries?: number; } ``` @@ -705,6 +732,31 @@ managed auto-memory 配置，也不增加独立配置文件、环境变量或 CL 复用时每个 File 仍保留自己的 root、source 和 lineage 引用，不能把一个文件的 权限或 provenance 泄漏给另一个文件。 +#### 11.2.1 三层身份：为什么"不合并"与"不重复建节点"并不矛盾 + +issue #8188 的验收项写作"内容相同的两个文件不重复建节点（复用同一底层身份)", +与本节标题字面冲突。二者实际落在**不同层**，实现同时满足： + +| 层 | 键 | 承载 | 满足 | +| ----------- | ------------------ | --------------------------------------------------------------- | ---------------------------------------- | +| Content | `sha256` | 物理对象（`objects/sha256/…`)、可复用的计算结果与 policy 派生物 | #8188「不重复建节点（复用同一底层身份)」 | +| File | locator | `CURRENT_VERSION`、source、provenance、权限/workspace 归属 | 本节「不合并不同文件」、#8189 版本隔离 | +| FileVersion | `(fileId, sha256)` | 二者的连接，不可变 | 11.1 版本链 | + +关键约束：**身份键不能是内容**。若 `fileId = hash(content)`，则"内容变化"直接 +产生另一个 `fileId`,11.1 的版本链（新 FileVersion → 更新 `CURRENT_VERSION` → +旧版本留作历史 → 默认排除旧版本）与 #8189 验收项"修改文件内容后旧衍生物不默认 +召回"**都无法表达**——编辑文件会凭空出现无关节点，改回去则历史成环。因此逻辑文件 +必须由跨内容稳定的 locator 定键。 + +于是"不重复付费"由 content 层承担：同字节的第二个文件不重跑任何工具（见 11.3 的 +复用键与 `reusedExecutionId`),不重复存储字节（共享同一 `objects/` 对象），只额外 +写几行属于自己的廉价元数据行。 + +派生物同样按 `(rootFileId, objectPath)` 定键而非仅 `objectPath`:两个 root 派生出 +逐字节相同的产物时，若共享一个 File 节点，该节点的 `rootFileId` 只能属于先创建者, +既泄漏 lineage 又让第二个 root 的有界遍历（§8)够不到它。 + ### 11.3 Policy 结果复用键 可复用 policy 结果的复用键必须与 11.2 的跨文件复用语义一致，因此以**内容身份** diff --git a/docs/design/2026-07-30-omni-managed-media-storage.md b/docs/design/2026-07-30-omni-managed-media-storage.md index e89802f398b..219be063f1f 100644 --- a/docs/design/2026-07-30-omni-managed-media-storage.md +++ b/docs/design/2026-07-30-omni-managed-media-storage.md @@ -140,30 +140,41 @@ ## 6. 生命周期与垃圾回收 -### 6.1 启动恢复 - -每次启动（或首次触碰 Omni 链路时）执行一次恢复扫描： - -1. 删除 `staging/` 下所有目录——它们属于未完成的 invocation，对应的 - `OmniPolicySucceeded` 必然未提交（提交成功即已删除 staging），Memory 中不 - 存在引用； -2. 删除超过保留期的 `downloads/*.part`（续传窗口内的保留）； -3. 按预算清理 `quarantine/`； -4. 校验 `objects/` 中随机抽样对象的文件名与实际 hash 一致（廉价的损坏探测， - 不做全库校验）。 - -### 6.2 mark-and-sweep GC - -- **根集合**：Memory store 中全部 active 记录引用的 managedId（含历史版本仍 - 被 provenance 引用的对象）+ 当前运行 session 的 MediaResourceRegistry 正在 - 使用的对象； -- **清扫对象**：不在根集合中、且自晋升起超过 `retentionDays` 的对象；宽限期 - 保证"晋升成功但 Memory 提交失败"的孤儿和跨进程 race 不被立刻误删； +### 6.1 启动恢复（已实现，`omni/recovery.ts`） + +每次启动（或首次触碰 Omni 链路时）执行一次恢复扫描。实现相对初稿的修正—— +多进程共存迫使"删除所有 staging"退化为宽限期语义： + +1. 删除 `staging/` 下**超过宽限期（1h）**的目录——宽限期必须长于策略工具 + 最长超时（配置校验强制 `runtime.timeoutMs` 低于宽限期），否则第二个 CLI + 进程的恢复扫描会把另一个进程正在转码的工作目录删掉。晋升半成品 `.tmp` + 同理（1h 宽限）； +2. 删除超过保留窗口（48h）的 `downloads/*.part`；无续传逻辑，窗口纯为事后 + 检查中断下载留的调试期； +3. 按保留天数与容量预算清理 `quarantine/`（超预算最旧优先）； +4. 抽样校验 `objects/` 对象名与实际 hash 一致（每次至多 3 个、单个 ≤64MB， + 避免在首次投递前的内联扫描里 hash 多 GB 视频）；发现损坏对象删除时**级联 + 清理降质缓存**（`policy-cache.json` 中以它为源或产物的条目），防止缓存 + 命中一个永远无法投递的对象。 + +### 6.2 mark-and-sweep GC（S6 落点） + +- **根集合**：Memory store 中全部记录引用的 managedId——`entries[].artifactRef.managedId` + 与 `versions[].source.locator`（`protocol: 'managed'`，tool/URL 来源媒体的 + 身份锚，见 Memory 设计 §11.2.1 实现注记）两处都算，含仅被 provenance 引用的 + 历史版本对象——加上当前进程 MediaResourceRegistry 正在使用的对象； +- **清扫对象**：不在根集合中、且自晋升起超过 `retentionDays` 的对象；保留期 + 兼作宽限期，保证"晋升成功但 Memory 提交失败"的孤儿和跨进程 race 不被立刻 + 误删； - **触发时机**：启动恢复后异步执行；超过 `maxTotalBytes` 时提前触发，仍超限 则从最旧的无引用对象继续删除；**有引用对象永不删除**，即使超预算——此时 - 告警并停止新的衍生物产生（等价于 Policy 设计 §8.4 的预算停止语义）； -- GC 与 Memory 的一致性：先从 Memory 确认无引用，再删除文件；不存在"先删 - 文件再改记录"的窗口。 + 告警并置"停止新衍生物"标志，由 orchestrator 消费（等价于 Policy 设计 §8.4 + 的预算停止语义）； +- **级联**：对象删除联动清理其 `upload-cache.json` 条目与降质缓存条目 + （复用恢复扫描已实现的级联，见 §6.1 第 4 条）；反向不成立； +- GC 与 Memory 的一致性：先从 Memory 快照确认无引用，再删除文件；不存在 + "先删文件再改记录"的窗口。快照读取失败时 GC 整体跳过（fail-closed： + 读不到根集合就不删任何东西）。 ## 7. 配置 diff --git a/packages/cli/src/acp-integration/session/Session.ts b/packages/cli/src/acp-integration/session/Session.ts index 614ef7b0a26..8f1512fbb1d 100644 --- a/packages/cli/src/acp-integration/session/Session.ts +++ b/packages/cli/src/acp-integration/session/Session.ts @@ -8191,6 +8191,45 @@ export class Session implements SessionContext { ? 'error' : 'success'; const succeeded = status === 'success'; + // ACP runs its own tool executor, so it must capture the policy + // artifact batch itself and call the SAME core memory boundary + // the scheduler and the fixed-policy orchestrator use (memory + // design M §17). Captured from the tool's OWN result, before any + // PostToolUse hook artifacts are merged below — hook artifacts + // must never impersonate policy outputs. Never throws; a + // collection failure cannot affect the tool result (D12). + if ( + succeeded && + tool.mediaPolicyDescriptor && + toolResult.artifacts && + toolResult.artifacts.length > 0 + ) { + // Deep subpath, not the root or `omni` barrel: both are + // statically imported across the CLI, so re-exporting this + // through either drags the whole policy graph — and, via + // iconvHelper's top-level iconv-lite import, its ~550 KB of + // encoding tables — into the ACP agent's static closure + // (scripts/check-serve-fast-path-bundle.js enforces that). + const { collectModelPolicyCall } = await import( + '@qwen-code/qwen-code-core/omniPolicyCollection' + ); + await collectModelPolicyCall({ + config: this.config, + batch: { + toolName, + invocationId: callId, + // Same pin as the modelAccess gate above: every + // ACP-originated call is a model call. Recording 'client' + // here made the PolicyExecution provenance contradict the + // gate that admitted the very same call. + executionOrigin: { kind: 'model' }, + artifacts: toolResult.artifacts, + }, + descriptor: tool.mediaPolicyDescriptor, + args, + signal: activeToolAbortSignal ?? abortSignal, + }); + } const responseError = toolResult.error ? new Error(toolResult.error.message) : aborted diff --git a/packages/cli/src/config/config.ts b/packages/cli/src/config/config.ts index 90a1f280190..6ef4e2f30bd 100755 --- a/packages/cli/src/config/config.ts +++ b/packages/cli/src/config/config.ts @@ -47,6 +47,7 @@ import { hooksCommand } from '../commands/hooks.js'; import { normalizeDisabledToolList } from './normalizeDisabledTools.js'; import type { LoadedSettings, Settings } from './settings.js'; import { loadSettings, SettingScope } from './settings.js'; +import { getSettingsSchema } from './settingsSchema.js'; import { resolveCliGenerationConfig, getAuthTypeFromEnv, @@ -1476,6 +1477,44 @@ export function buildDisabledSkillNamesProvider( return () => resolveSkillSettings(loadedSettings).disabledNames; } +/** + * Reject unknown keys directly under `omni`. + * + * The generic settings loader only scans TOP-LEVEL keys, and only writes a + * debug line — so a nested typo is caught by nothing: `omni.memoryy` or + * `omni.processingg` leaves `settings.omni?.memory` / `?.processing` + * undefined, every downstream normalizer sees "not configured" and returns + * defaults, and the session silently runs with the operator's entire + * configuration discarded (probe: `omni.memoryy.recall.mode = sideQuery` + * still registered the active-mode recall tool). The omni namespace's + * declared stance is that a misconfiguration must fail loud, so this + * mirrors the nested checks its own normalizers already perform. + * + * The allowed set is derived from the settings schema rather than + * hardcoded, so it cannot drift as the namespace grows. + */ +function assertKnownOmniSettingKeys(settings: Settings): void { + const omni = settings.omni; + if (omni === undefined || omni === null || typeof omni !== 'object') return; + const schemaOmni = getSettingsSchema()['omni'] as + | { properties?: Record } + | undefined; + const allowed = new Set(Object.keys(schemaOmni?.properties ?? {})); + // No schema properties resolved (unexpected): stay silent rather than + // rejecting every valid key. + if (allowed.size === 0) return; + const unknown = Object.keys(omni).filter((key) => !allowed.has(key)); + if (unknown.length > 0) { + throw new Error( + `Invalid settings: unknown key(s) under "omni": ` + + `${unknown.map((k) => `"${k}"`).join(', ')}. ` + + `Allowed: ${[...allowed].sort().join(', ')}. ` + + `An unrecognized omni section would be silently ignored, leaving ` + + `the session running with defaults instead of your configuration.`, + ); + } +} + export async function loadCliConfig( settings: Settings, argv: CliArgs, @@ -1521,6 +1560,7 @@ export async function loadCliConfig( */ settingsWatcher?: { stopWatching(): void }, ): Promise { + assertKnownOmniSettingKeys(settings); const debugMode = isDebugMode(argv); if (debugMode && process.env['QWEN_DEBUG_LOG_FILE'] === undefined) { process.env['QWEN_DEBUG_LOG_FILE'] = '1'; @@ -2197,6 +2237,8 @@ export async function loadCliConfig( settings.omni?.processing?.transportGuard?.maxUploadFileBytes, omniMaxEstimatedTokens: settings.omni?.processing?.transportGuard?.maxEstimatedTokens, + omniMaxDurationSeconds: + settings.omni?.processing?.transportGuard?.maxDurationSeconds, omniUrlDownloadMaxFileBytes: settings.omni?.ingestion?.localization?.url?.maxFileBytes, omniUploadUrlTtlHours: settings.omni?.delivery?.upload?.urlTtlHours, @@ -2214,6 +2256,7 @@ export async function loadCliConfig( omniQuarantineRetentionDays: settings.omni?.storage?.quarantine?.retentionDays, omniQuarantineMaxBytes: settings.omni?.storage?.quarantine?.maxBytes, + omniMemory: settings.omni?.memory as Record | undefined, // CDP tunnel (Plan C, #5626): with the tunnel on, browser automation goes // through the CDP tunnel (far lighter than the OS-level computer-use // driver), so disable computer-use to keep the agent off that heavy path. diff --git a/packages/cli/src/config/settingsSchema.ts b/packages/cli/src/config/settingsSchema.ts index fc549616014..14e4278e784 100644 --- a/packages/cli/src/config/settingsSchema.ts +++ b/packages/cli/src/config/settingsSchema.ts @@ -3838,6 +3838,26 @@ const SETTINGS_SCHEMA = { default: 0, }, }, + maxDurationSeconds: { + type: 'number', + label: 'Max Duration Seconds', + category: 'Experimental', + requiresRestart: true, + default: 0, + description: + 'Duration ceiling in seconds for a single omni media ' + + 'input, checked at the delivery boundary. 0 disables it. ' + + 'Byte and token limits cannot express a provider duration ' + + 'cap: a long film downscaled under the byte ceiling still ' + + 'gets rejected by the API, after paying for the transcode. ' + + 'Set this and the guard omits it honestly instead.', + showInDialog: false, + jsonSchemaOverride: { + type: 'integer', + minimum: 0, + default: 0, + }, + }, policies: { type: 'object', label: 'Omni Transport Guard Policies', @@ -4019,6 +4039,268 @@ const SETTINGS_SCHEMA = { }, }, }, + memory: { + type: 'object', + label: 'Omni Media Memory', + category: 'Experimental', + requiresRestart: true, + default: {}, + description: + 'Persistent multimodal media memory (collection of recognized ' + + 'files and policy execution results, plus cross-session recall). ' + + 'Invalid values abort startup.', + showInDialog: false, + properties: { + collection: { + type: 'object', + label: 'Omni Memory Collection', + category: 'Experimental', + requiresRestart: true, + default: {}, + description: 'Collection-side budgets.', + showInDialog: false, + properties: { + maxInlineTextBytes: { + type: 'number', + label: 'Max Inline Text Bytes', + category: 'Experimental', + requiresRestart: true, + default: 65536, + description: + 'Upper bound for inline text persisted on a memory entry ' + + '(transcripts, OCR). Longer text is truncated on the ' + + 'entry; the stored artifact keeps the full content.', + showInDialog: false, + jsonSchemaOverride: { + type: 'integer', + minimum: 1, + default: 65536, + }, + }, + }, + }, + recall: { + type: 'object', + label: 'Omni Memory Recall', + category: 'Experimental', + requiresRestart: true, + default: {}, + description: 'Recall-side exposure and budgets.', + showInDialog: false, + properties: { + mode: { + type: 'string', + label: 'Recall Mode', + category: 'Experimental', + requiresRestart: true, + default: 'active', + description: + 'Mutually exclusive recall exposure: "active" registers ' + + 'the recall tool for the model; "sideQuery" runs a ' + + 'passive selector before the main request instead.', + showInDialog: false, + jsonSchemaOverride: { + type: 'string', + enum: ['active', 'sideQuery'], + default: 'active', + }, + }, + maxEntries: { + type: 'number', + label: 'Max Recall Entries', + category: 'Experimental', + requiresRestart: true, + default: 12, + description: 'Maximum entries one recall may return.', + showInDialog: false, + jsonSchemaOverride: { + type: 'integer', + minimum: 1, + default: 12, + }, + }, + maxTextChars: { + type: 'number', + label: 'Max Recall Text Chars', + category: 'Experimental', + requiresRestart: true, + default: 24000, + description: + 'Total character budget across all text in one recall ' + + 'result.', + showInDialog: false, + jsonSchemaOverride: { + type: 'integer', + minimum: 1, + default: 24000, + }, + }, + kinds: { + type: 'array', + label: 'Recall Entry Kinds', + category: 'Experimental', + requiresRestart: true, + default: [ + 'metadata', + 'derived_media', + 'policy_result', + 'execution', + ] as string[], + description: + 'Entry kinds recall may surface. Replaces wholesale when ' + + 'set (never element-merged).', + showInDialog: false, + mergeStrategy: MergeStrategy.REPLACE, + jsonSchemaOverride: { + type: 'array', + items: { + type: 'string', + enum: [ + 'metadata', + 'derived_media', + 'policy_result', + 'execution', + ], + }, + minItems: 1, + }, + }, + includeHistoricalVersions: { + type: 'boolean', + label: 'Include Historical Versions', + category: 'Experimental', + requiresRestart: true, + default: false, + description: + 'Whether recall may surface entries for non-current file ' + + 'versions by default.', + showInDialog: false, + }, + active: { + type: 'object', + label: 'Active Recall', + category: 'Experimental', + requiresRestart: true, + default: {}, + description: 'Budgets for the active recall tool.', + showInDialog: false, + properties: { + maxFilesPerCall: { + type: 'number', + label: 'Max Files Per Call', + category: 'Experimental', + requiresRestart: true, + default: 8, + description: + 'Maximum distinct files one recall tool call may ' + + 'query.', + showInDialog: false, + jsonSchemaOverride: { + type: 'integer', + minimum: 1, + default: 8, + }, + }, + }, + }, + sideQuery: { + type: 'object', + label: 'Side-Query Recall', + category: 'Experimental', + requiresRestart: true, + default: {}, + description: 'Budgets for the passive side-query selector.', + showInDialog: false, + properties: { + model: { + type: 'string', + label: 'Selector Model', + category: 'Experimental', + requiresRestart: true, + default: undefined as string | undefined, + description: + 'Model for the passive selector; unset uses the ' + + 'side-query default (the configured fast model, ' + + 'falling back to the session model).', + showInDialog: false, + jsonSchemaOverride: { + type: ['string', 'null'], + default: null, + }, + }, + timeoutMs: { + type: 'number', + label: 'Selector Timeout (ms)', + category: 'Experimental', + requiresRestart: true, + default: 30000, + description: + 'Side-query timeout; on timeout the turn proceeds ' + + 'with an empty recall.', + showInDialog: false, + jsonSchemaOverride: { + type: 'integer', + minimum: 1, + default: 30000, + }, + }, + maxCandidateEntries: { + type: 'number', + label: 'Max Candidate Entries', + category: 'Experimental', + requiresRestart: true, + default: 100, + description: + 'Maximum candidate entries shown to the selector.', + showInDialog: false, + jsonSchemaOverride: { + type: 'integer', + minimum: 1, + default: 100, + }, + }, + maxSelectedEntries: { + type: 'number', + label: 'Max Selected Entries', + category: 'Experimental', + requiresRestart: true, + default: 12, + description: + 'Maximum entries the selector may pick. Must not ' + + 'exceed recall.maxEntries.', + showInDialog: false, + jsonSchemaOverride: { + type: 'integer', + minimum: 1, + default: 12, + }, + }, + maxAttempts: { + type: 'number', + label: 'Max Selector Attempts', + category: 'Experimental', + requiresRestart: true, + default: 1, + description: + 'Attempts allowed for the selector call itself ' + + '(unparseable or schema-invalid output is retried up ' + + 'to this many times). A selection that parses but ' + + 'names entries outside the manifest is refused ' + + 'without a retry; the turn then proceeds with an ' + + 'empty recall.', + showInDialog: false, + jsonSchemaOverride: { + type: 'integer', + minimum: 1, + default: 1, + }, + }, + }, + }, + }, + }, + }, + }, }, }, } as const satisfies SettingsSchema; diff --git a/packages/cli/src/i18n/index.test.ts b/packages/cli/src/i18n/index.test.ts index 027855ff4c3..2e6266d25c3 100644 --- a/packages/cli/src/i18n/index.test.ts +++ b/packages/cli/src/i18n/index.test.ts @@ -189,6 +189,27 @@ describe('localizeToolDisplayName', () => { expect(localizeToolDisplayName('MysteryTool')).toBe('MysteryTool'); }); + // Both translating locales, not just zh: `t()` has no cross-locale + // fallback, so a tool added to zh.js alone renders its raw English badge + // in a zh-TW session, next to fully-translated siblings. + it.each(['zh', 'zh-TW'] as const)( + 'has a %s translation for every core tool display name', + async (locale) => { + const { setLanguageAsync, localizeToolDisplayName } = await import( + './index.js' + ); + const { ToolDisplayNames } = await import('@qwen-code/qwen-code-core'); + await setLanguageAsync(locale); + + const KEEP_ENGLISH = new Set(['Agent', 'Grep', 'Glob']); + const untranslated = Object.values(ToolDisplayNames).filter( + (name) => + !KEEP_ENGLISH.has(name) && localizeToolDisplayName(name) === name, + ); + expect(untranslated).toEqual([]); + }, + ); + it('has a zh translation for every core tool display name', async () => { const { setLanguageAsync, localizeToolDisplayName } = await import( './index.js' diff --git a/packages/cli/src/i18n/locales/en.js b/packages/cli/src/i18n/locales/en.js index b58e4f325fc..6ddb35b8257 100644 --- a/packages/cli/src/i18n/locales/en.js +++ b/packages/cli/src/i18n/locales/en.js @@ -231,6 +231,7 @@ export default { 'toolDisplayName.ClipVideo': 'toolDisplayName.ClipVideo', 'toolDisplayName.ConvertImage': 'toolDisplayName.ConvertImage', 'toolDisplayName.TranscribeAudio': 'toolDisplayName.TranscribeAudio', + 'toolDisplayName.RecallMediaMemory': 'toolDisplayName.RecallMediaMemory', '[fixed-only: runs via media policies, not the model]': '[fixed-only: runs via media policies, not the model]', // ============================================================================ diff --git a/packages/cli/src/i18n/locales/zh-TW.js b/packages/cli/src/i18n/locales/zh-TW.js index 87d9a0cedb3..d56d7b2b5cb 100644 --- a/packages/cli/src/i18n/locales/zh-TW.js +++ b/packages/cli/src/i18n/locales/zh-TW.js @@ -222,6 +222,7 @@ export default { 'toolDisplayName.ClipVideo': '剪輯影片', 'toolDisplayName.ConvertImage': '轉換圖像', 'toolDisplayName.TranscribeAudio': '轉寫音訊', + 'toolDisplayName.RecallMediaMemory': '召回媒體記憶', '[fixed-only: runs via media policies, not the model]': '［僅固定策略：由媒體策略調用，不開放給模型］', diff --git a/packages/cli/src/i18n/locales/zh.js b/packages/cli/src/i18n/locales/zh.js index f306ad7e38f..275fee93e2d 100644 --- a/packages/cli/src/i18n/locales/zh.js +++ b/packages/cli/src/i18n/locales/zh.js @@ -223,6 +223,7 @@ export default { 'toolDisplayName.ClipVideo': '剪辑视频', 'toolDisplayName.ConvertImage': '转换图像', 'toolDisplayName.TranscribeAudio': '转写音频', + 'toolDisplayName.RecallMediaMemory': '召回媒体记忆', '[fixed-only: runs via media policies, not the model]': '［仅固定策略：由媒体策略调用，不开放给模型］', // ============================================================================ diff --git a/packages/cli/src/ui/hooks/atCommandProcessor.test.ts b/packages/cli/src/ui/hooks/atCommandProcessor.test.ts index 89e580c5b40..2b10c98c648 100644 --- a/packages/cli/src/ui/hooks/atCommandProcessor.test.ts +++ b/packages/cli/src/ui/hooks/atCommandProcessor.test.ts @@ -1758,6 +1758,8 @@ describe('handleAtCommand', () => { `[omission ${name}: ${reason}]`, formatDisclosureText: (name: string, disclosure: string) => `[disclosure ${name}: ${disclosure}]`, + formatResourceHandleText: (name: string, resourceId: string) => + `[handle ${name}: ${resourceId}]`, // Deterministic stand-in for the shared multi-output materializer: // one marker part per extra. The real builder's output shape is // covered by its core unit tests; these tests pin the wiring (that @@ -1961,6 +1963,72 @@ describe('handleAtCommand', () => { }); }); + it('leads the part group with the session resource handle (M §5.2)', async () => { + // A delivery mints a registry binding and writes memory records for + // the file. If the funnel never tells the model the handle, active + // recall rejects it as never-issued and the passive selector finds no + // handles to consult — the session can never recall media it just + // paid an upload to collect. + omniMocks.processMediaForOmniDelivery.mockResolvedValue({ + fileUri: 'oss://bucket/clip.mp4', + mimeType: 'video/mp4', + recognized: { modality: 'video', sizeBytes: 2 * 1024 * 1024 }, + resourceId: 'media-3-9f2a', + disclosure: '原 1080p → 480p', + }); + + const result = await handleAtCommand({ + query: 'summarize @https://example.com/clip.mp4 please', + config: omniConfig(true), + onDebugMessage: mockOnDebugMessage, + messageId: 731, + signal: abortController.signal, + }); + + const parts = result.processedQuery as Array>; + const handleIdx = parts.findIndex( + (p) => p['text'] === '[handle clip.mp4: media-3-9f2a]', + ); + expect(handleIdx).toBeGreaterThan(-1); + // Ahead of the disclosure, so the disclosure keeps its D8 adjacency + // to the media part it speaks for. + const disclosureIdx = parts.findIndex( + (p) => p['text'] === '[disclosure clip.mp4: 原 1080p → 480p]', + ); + expect(disclosureIdx).toBe(handleIdx + 1); + expect(parts[disclosureIdx + 1]).toMatchObject({ + fileData: { fileUri: 'oss://bucket/clip.mp4' }, + }); + }); + + it('still discloses the handle when the guard withholds the media', async () => { + // The withheld case is where recall matters MOST: the model cannot + // see the media, so a handle it can hand to omni_recall_media_memory + // (or a policy tool) is its only remaining route to the content. + omniMocks.processMediaForOmniDelivery.mockResolvedValue({ + recognized: { modality: 'video', sizeBytes: 900 * 1024 * 1024 }, + omission: { reason: 'video exceeds the transport limit' }, + resourceId: 'media-4-c1d0', + }); + + const result = await handleAtCommand({ + query: 'summarize @https://example.com/clip.mp4 please', + config: omniConfig(true), + onDebugMessage: mockOnDebugMessage, + messageId: 732, + signal: abortController.signal, + }); + + const parts = result.processedQuery as Array>; + const handleIdx = parts.findIndex( + (p) => p['text'] === '[handle clip.mp4: media-4-c1d0]', + ); + expect(handleIdx).toBeGreaterThan(-1); + expect(parts[handleIdx + 1]).toEqual({ + text: '[omission clip.mp4: video exceeds the transport limit]', + }); + }); + it('places the degradation disclosure text immediately before the fileData part (D8)', async () => { omniMocks.processMediaForOmniDelivery.mockResolvedValue({ fileUri: 'oss://bucket/clip.mp4', diff --git a/packages/cli/src/ui/hooks/atCommandProcessor.ts b/packages/cli/src/ui/hooks/atCommandProcessor.ts index df13e4ee942..1d367b01b3a 100644 --- a/packages/cli/src/ui/hooks/atCommandProcessor.ts +++ b/packages/cli/src/ui/hooks/atCommandProcessor.ts @@ -284,6 +284,13 @@ export async function resolveAtCommandQuery({ // store) after it. Only active when omni delivery is on; otherwise URL // tokens keep today's fall-through behavior (left as text). const urlMediaRefs: Array<{ originalAtPath: string; url: string }> = []; + /** `@`-referenced media whose bytes are gone but whose memory survives: + * re-anchored to a session handle so recall stays reachable. */ + const rememberedMediaRefs: Array<{ + originalAtPath: string; + pathName: string; + annotation: string; + }> = []; for (const atPathPart of atPathCommandParts) { const originalAtPath = atPathPart.content; // e.g., "@file.txt" or "@" @@ -473,9 +480,43 @@ export async function resolveAtCommandQuery({ } } if (!resolvedSuccessfully && sawNotFound) { - onDebugMessage( - `Path ${pathName} not found. Path ${pathName} will be skipped.`, - ); + // The bytes are gone, but media memory may still hold everything + // that was ever derived from them (transcripts, keyframes, the + // processing history). A handle is normally minted only at delivery, + // which needs the bytes — so that knowledge used to be unreachable + // for good. The user's own `@`-reference is the same authorization a + // delivery carries, so re-anchor from the recorded identity and hand + // the model a handle it can recall with (design M §9.2.1). + let reanchored = false; + if (config.isOmniEnabled?.()) { + const { reanchorRememberedMedia } = await import( + '@qwen-code/qwen-code-core/omni' + ); + for (const dir of config.getWorkspaceContext().getDirectories()) { + const anchor = await reanchorRememberedMedia( + config, + path.resolve(dir, pathName), + ); + if (!anchor) continue; + rememberedMediaRefs.push({ + originalAtPath, + pathName, + annotation: anchor.annotation, + }); + atPathToResolvedSpecMap.set(originalAtPath, pathName); + onDebugMessage( + `Path ${pathName} is gone but remembered; re-anchored as ` + + `${anchor.resourceId} (bytes unavailable).`, + ); + reanchored = true; + break; + } + } + if (!reanchored) { + onDebugMessage( + `Path ${pathName} not found. Path ${pathName} will be skipped.`, + ); + } } } @@ -620,8 +661,12 @@ export async function resolveAtCommandQuery({ downloaded.partPath, config, // displayName: guard/error messages must name the URL's file, not - // the opaque staging path the download landed under. - { signal, displayName: urlBase }, + // the opaque staging path the download landed under. sourceUrl: + // the staging path is deleted in the finally below, so memory + // must anchor this media's identity to the object store and + // record the URL as its source — a handle bound to the staging + // path would resolve to ENOENT for the rest of the session. + { signal, displayName: urlBase, sourceUrl: ref.url }, ); // §6.2/D8 ordering contract documented on buildTranscriptParts. const transcriptParts = core.buildTranscriptParts( @@ -634,11 +679,30 @@ export async function resolveAtCommandQuery({ urlBase, delivery.additionalMedia, ); + // Session resource handle (memory design M §5.2): leads the part + // group in every branch below, exactly as readMediaViaOmniDelivery + // does it. Without this, URL-delivered media accrues memory records + // and an issued registry binding that the model is never told about + // — active recall rejects the unknown handle and the passive + // selector finds no handles to consult, so the session can never + // recall what it just spent an upload collecting. Placed FIRST so + // the disclosure keeps its D8 adjacency to the media part. + const handleParts = delivery.resourceId + ? [ + { + text: core.formatResourceHandleText( + urlBase, + delivery.resourceId, + ), + }, + ] + : []; if (delivery.omission) { // Explicit omission (policy design §10.2): the media is withheld // and the omission notice text stands in its place — mirroring // readMediaViaOmniDelivery. Not an error: the fetch succeeded; // the transport guard's verdict is the content. + urlMediaParts.push(...handleParts); urlMediaParts.push({ text: core.formatOmissionText(urlBase, delivery.omission.reason), }); @@ -661,6 +725,7 @@ export async function resolveAtCommandQuery({ // for the primary (additional deliverables, if any, still are). // The primary disclosure (chained prior lossy steps, decision // D8) still renders: the transcript was derived through them. + urlMediaParts.push(...handleParts); if (delivery.disclosure) { urlMediaParts.push({ text: core.formatDisclosureText(urlBase, delivery.disclosure), @@ -679,6 +744,7 @@ export async function resolveAtCommandQuery({ }); continue; } + urlMediaParts.push(...handleParts); // Disclosure IMMEDIATELY before its media part (decision D8): // provider converters that relocate media move the pair together. if (delivery.disclosure) { @@ -1116,18 +1182,30 @@ export async function resolveAtCommandQuery({ // its content block via the "--- Content from ... ---" delimiter labels (and // the verbatim `@server:uri` / `@path` left in the prompt text), not by // positional alignment, so grouping is safe. + // Re-anchored media: a handle annotation plus an explicit note that the + // bytes are gone, so the model recalls instead of trying to read. + const rememberedMediaParts: Part[] = rememberedMediaRefs.flatMap((ref) => [ + { text: ref.annotation }, + { + text: + `【媒体缺失】${ref.pathName}：文件已不在磁盘上，无法投递画面/音频。` + + `其处理记忆仍可用——用上面的句柄调用 omni_recall_media_memory 取回。`, + }, + ]); const processedQueryParts: PartListUnion = [ { text: initialQueryText }, ...scopedMentionParts, ...fileParts, ...resourceParts, ...urlMediaParts, + ...rememberedMediaParts, ]; const allLabels = [ ...scopedMentionLabels, ...contentLabelsForDisplay, ...resourceLabels, ...urlMediaLabels, + ...rememberedMediaRefs.map((ref) => `${ref.pathName} (记忆)`), ]; return { diff --git a/packages/cli/vitest.config.ts b/packages/cli/vitest.config.ts index 2adb9ea9c84..321798b7d2e 100644 --- a/packages/cli/vitest.config.ts +++ b/packages/cli/vitest.config.ts @@ -19,6 +19,10 @@ export default defineConfig({ __dirname, '../core/src/utils/transcript-records.ts', ), + '@qwen-code/qwen-code-core/omniPolicyCollection': path.resolve( + __dirname, + '../core/src/omni/policy/model-call-collection.ts', + ), '@qwen-code/qwen-code-core/omni': path.resolve( __dirname, '../core/src/omni/index.ts', diff --git a/packages/core/package.json b/packages/core/package.json index 0fedf744890..2010a00670a 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -17,6 +17,10 @@ "types": "./dist/src/omni/index.d.ts", "import": "./dist/src/omni/index.js" }, + "./omniPolicyCollection": { + "types": "./dist/src/omni/policy/model-call-collection.d.ts", + "import": "./dist/src/omni/policy/model-call-collection.js" + }, "./transcriptRecords": { "types": "./dist/src/utils/transcript-records.d.ts", "import": "./dist/src/utils/transcript-records.js" diff --git a/packages/core/src/config/config.test.ts b/packages/core/src/config/config.test.ts index a3591ad12e1..67cde3880ae 100644 --- a/packages/core/src/config/config.test.ts +++ b/packages/core/src/config/config.test.ts @@ -6759,6 +6759,61 @@ describe('Server Config (config.ts)', () => { expect(wasShellToolRegistered).toBe(true); }); }); + + describe('omni media-memory recall exposure (D10)', () => { + /** Register a fresh omni-enabled registry and report what it holds. + * `createToolRegistry` (not `initialize`) is the unit under test: it + * is where the mode decision happens, and it runs before the omni + * normalization block in startup. */ + async function registeredToolNames( + omniMemory?: Record, + ): Promise { + const config = new Config({ + ...baseParams, + omniEnabled: true, + ...(omniMemory !== undefined ? { omniMemory } : {}), + }); + await config.createToolRegistry(undefined, { skipDiscovery: true }); + const registerFactoryMock = ( + (await vi.importMock('../tools/tool-registry')) as { + ToolRegistry: { prototype: { registerFactory: Mock } }; + } + ).ToolRegistry.prototype.registerFactory; + return (registerFactoryMock as Mock).mock.calls.map( + (call) => call[0] as string, + ); + } + + it('exposes the recall tool in active mode', async () => { + const names = await registeredToolNames({ recall: { mode: 'active' } }); + expect(names).toContain(ToolNames.OMNI_RECALL_MEDIA_MEMORY); + }); + + it('withholds the recall tool in sideQuery mode', async () => { + // The two recall surfaces are mutually exclusive: in sideQuery mode + // the harness injects recall itself before every request. Leaving + // the tool registered as well would let the model spend a tool call + // re-fetching memory it was already handed — and the registration + // is decided once, here, so nothing downstream can take it back. + const names = await registeredToolNames({ + recall: { mode: 'sideQuery' }, + }); + expect(names).not.toContain(ToolNames.OMNI_RECALL_MEDIA_MEMORY); + // The rest of the omni toolset still registered — proof the tool is + // missing because of the mode, not because omni was off. + expect(names).toContain(ToolNames.OMNI_DOWNSAMPLE_IMAGE); + }); + + it('aborts startup on an invalid omni.memory setting', async () => { + // A rejected `omni.memory` must never degrade to defaults: the + // default is `active`, so a typo in the mode would silently hand the + // model a recall tool in a session the user configured for passive + // injection — the exact silent fallback the normalizer forbids. + await expect( + registeredToolNames({ recall: { mode: 'passive' } }), + ).rejects.toThrow(/omni\.memory\.recall\.mode/); + }); + }); }); describe('getTruncateToolOutputThreshold', () => { diff --git a/packages/core/src/config/config.ts b/packages/core/src/config/config.ts index 9fa678a17c9..b33f382dade 100644 --- a/packages/core/src/config/config.ts +++ b/packages/core/src/config/config.ts @@ -32,6 +32,8 @@ import type { NormalizedOmniProcessingConfig, OmniPolicyToolsSettings, } from '../omni/policy/types.js'; +import type { NormalizedOmniMemoryConfig } from '../services/media-memory/config.js'; +import { MediaResourceRegistry } from '../services/media-memory/registry.js'; import type { ArenaManager } from '../agents/arena/ArenaManager.js'; import { ArenaAgentClient } from '../agents/arena/ArenaAgentClient.js'; import type { TeamManager } from '../agents/team/TeamManager.js'; @@ -1110,6 +1112,7 @@ export interface ConfigParameters { omniMaxUploadFileBytes?: number; /** Estimated-token ceiling for omni media (0/unset = guard disabled). */ omniMaxEstimatedTokens?: number; + omniMaxDurationSeconds?: number; /** Byte ceiling for omni URL downloads (unset = follow upload cap). */ omniUrlDownloadMaxFileBytes?: number; /** Upload URL TTL in hours (0 disables the cache; default 47). */ @@ -1128,6 +1131,9 @@ export interface ConfigParameters { omniQuarantineRetentionDays?: number; /** `omni.storage.quarantine.maxBytes` (default 5 GiB). */ omniQuarantineMaxBytes?: number; + /** Raw `omni.memory` settings (collection/recall). Normalized at + * startup; invalid configuration aborts startup. */ + omniMemory?: Record; /** Image generation model selected through `/model --image`. */ imageModel?: string; /** @@ -1956,6 +1962,7 @@ export class Config { private readonly omniEnabled: boolean = false; private readonly omniMaxUploadFileBytes?: number; private readonly omniMaxEstimatedTokens?: number; + private readonly omniMaxDurationSeconds?: number; private readonly omniUrlDownloadMaxFileBytes?: number; private readonly omniUploadUrlTtlHours?: number; private readonly omniPolicyTools?: OmniPolicyToolsSettings; @@ -1964,9 +1971,17 @@ export class Config { private readonly omniProcessingLimits?: Record; private readonly omniQuarantineRetentionDays?: number; private readonly omniQuarantineMaxBytes?: number; + private readonly omniMemory?: Record; /** Normalized `omni.processing` view; set once during initialize() * (after the tool registry exists) when omni is enabled. */ private omniProcessingConfig?: NormalizedOmniProcessingConfig; + /** Normalized `omni.memory` view; set once during initialize() when + * omni is enabled. */ + private omniMemoryConfig?: NormalizedOmniMemoryConfig; + /** Session-lifetime binder between persistent media-memory identities + * and the opaque resource handles the model sees (M §5.2); created + * lazily on first use, never persisted. */ + private omniMediaResourceRegistry?: MediaResourceRegistry; private workflowsEnabled = false; private readonly skipWorkflowUsageWarning: boolean = false; private readonly computerUseEnabled: boolean = true; @@ -2245,6 +2260,7 @@ export class Config { this.omniEnabled = params.omniEnabled ?? false; this.omniMaxUploadFileBytes = params.omniMaxUploadFileBytes; this.omniMaxEstimatedTokens = params.omniMaxEstimatedTokens; + this.omniMaxDurationSeconds = params.omniMaxDurationSeconds; this.omniUrlDownloadMaxFileBytes = params.omniUrlDownloadMaxFileBytes; this.omniUploadUrlTtlHours = params.omniUploadUrlTtlHours; this.omniPolicyTools = params.omniPolicyTools; @@ -2253,6 +2269,7 @@ export class Config { this.omniProcessingLimits = params.omniProcessingLimits; this.omniQuarantineRetentionDays = params.omniQuarantineRetentionDays; this.omniQuarantineMaxBytes = params.omniQuarantineMaxBytes; + this.omniMemory = params.omniMemory; this.workflowsEnabled = params.workflowsEnabled ?? false; this.skipWorkflowUsageWarning = params.skipWorkflowUsageWarning ?? false; this.computerUseEnabled = params.computerUseEnabled ?? true; @@ -3006,6 +3023,13 @@ export class Config { }, this.toolRegistry, ); + + // Same stance for `omni.memory`: normalized once (idempotent — + // createToolRegistry already ran it to gate the recall tool's + // registration on `recall.mode`); an invalid memory configuration + // (bad budgets, unknown recall mode) throws OmniMemoryConfigError + // and aborts startup. + await this.ensureOmniMemoryConfig(); } // Fire-and-forget MCP discovery. Each server's tools land in the @@ -6449,6 +6473,10 @@ export class Config { return this.omniMaxEstimatedTokens; } + getOmniMaxDurationSeconds(): number | undefined { + return this.omniMaxDurationSeconds; + } + getOmniUrlDownloadMaxFileBytes(): number | undefined { return this.omniUrlDownloadMaxFileBytes; } @@ -6467,6 +6495,37 @@ export class Config { return this.omniProcessingConfig; } + /** Normalized `omni.memory` view. Undefined until initialize() + * completes (or when omni is disabled). */ + getOmniMemoryConfig(): NormalizedOmniMemoryConfig | undefined { + return this.omniMemoryConfig; + } + + /** Normalize `omni.memory` on first use (idempotent). Called from + * createToolRegistry — which runs BEFORE initialize()'s omni + * normalization block and needs `recall.mode` to gate the recall + * tool's registration (D10) — and again from initialize() as a + * no-op backstop. Invalid settings throw OmniMemoryConfigError: + * startup-fatal on both paths. */ + private async ensureOmniMemoryConfig(): Promise { + if (!this.omniMemoryConfig) { + const { normalizeOmniMemoryConfig } = await import( + '../services/media-memory/config.js' + ); + this.omniMemoryConfig = normalizeOmniMemoryConfig(this.omniMemory); + } + return this.omniMemoryConfig; + } + + /** Session registry binding persistent media-memory identities to the + * opaque `resourceId` handles the model references (M §5.2). One + * instance per session; a handle is only meaningful in the session + * that minted it. */ + getOmniMediaResourceRegistry(): MediaResourceRegistry { + this.omniMediaResourceRegistry ??= new MediaResourceRegistry(); + return this.omniMediaResourceRegistry; + } + getOmniQuarantineRetentionDays(): number { // A zero/negative/NaN setting would make the recovery sweep treat the // whole quarantine as expired (or break its cutoff comparisons) — @@ -8199,6 +8258,23 @@ export class Config { for (const [name, factory] of omniPolicyToolFactories) { await registerLazy(name, factory); } + + // Active-mode memory recall (M §9, D10 mutual exclusion): the + // normalized `omni.memory` config decides at REGISTRATION time + // whether the tool exists at all — in sideQuery mode the passive + // selector runs instead and this tool must never be exposed. + // Normalized on demand because createToolRegistry runs before + // initialize()'s omni normalization block; invalid settings are + // startup-fatal on this path too. + const memoryConfig = await this.ensureOmniMemoryConfig(); + if (memoryConfig.recall.mode === 'active') { + await registerLazy(ToolNames.OMNI_RECALL_MEDIA_MEMORY, async () => { + const { OmniRecallMediaMemoryTool } = await import( + '../omni/recall-media-memory-tool.js' + ); + return new OmniRecallMediaMemoryTool(this); + }); + } } // Register monitor tool diff --git a/packages/core/src/core/client.test.ts b/packages/core/src/core/client.test.ts index e219c8d2cbc..c62abfbba2d 100644 --- a/packages/core/src/core/client.test.ts +++ b/packages/core/src/core/client.test.ts @@ -53,6 +53,8 @@ import { } from './turn.js'; import { LoopType } from '../telemetry/types.js'; import { logMemoryRecallDelivery } from '../telemetry/index.js'; +import { formatOmniMemorySideQueryReminder } from '../omni/memory-side-query.js'; +import type { MediaMemoryRecallResult } from '../services/media-memory/index.js'; type MockSessionStartProfiler = { time: Mock; @@ -384,6 +386,15 @@ vi.mock( ); import { microcompactHistory } from '../services/microcompaction/microcompact.js'; +// Only the selector itself is stubbed — the reminder the client injects is +// formatted by the real omni module, so the assertions below match the +// exact block a production passive recall would put on the wire. +const runOmniMemorySideQueryMock = vi.hoisted(() => vi.fn()); +vi.mock('../omni/memory-side-query.js', async (importOriginal) => ({ + ...(await importOriginal()), + runOmniMemorySideQuery: runOmniMemorySideQueryMock, +})); + // Mock RequestTokenizer to use simple character-based estimation vi.mock('../utils/request-tokenizer/requestTokenizer.js', () => ({ RequestTokenizer: class { @@ -1781,6 +1792,115 @@ describe('Gemini Client (client.ts)', () => { }); }); + describe('omni passive media-memory recall injection', () => { + /** Minimal materialized recall — the client neither builds nor reads + * this payload, it only has to get the formatted block onto the wire. */ + const recallResult = { + status: 'hit', + files: [{ resourceId: 'media-1-abcdef01', mediaType: 'image' }], + entries: [{ entryId: 'e-1', kind: 'derived_media' }], + gaps: [], + } as unknown as MediaMemoryRecallResult; + + function enableOmni(): void { + ( + mockConfig as unknown as { isOmniEnabled: () => boolean } + ).isOmniEnabled = () => true; + } + + async function runUserQuery(): Promise { + mockTurnRunFn.mockReturnValue( + (async function* () { + yield { type: GeminiEventType.Content, value: 'response' }; + })(), + ); + const stream = client.sendMessageStream( + [{ text: 'what changed in this clip?' }], + new AbortController().signal, + 'prompt-omni-recall', + { type: SendMessageType.UserQuery }, + ); + for await (const _ of stream) { + // drain + } + return mockTurnRunFn.mock.lastCall?.[1] as unknown[]; + } + + it('prepends the recalled block ahead of the user parts of the outgoing request', async () => { + // This is the ONLY place the sideQuery surface reaches a model + // request. The whole point of passive recall is that it lands + // BEFORE the main request is sent (M §9.3) — retrofitting it after + // the systemReminders spread, or dropping the call, leaves a + // pipeline that still selects and materializes entries and then + // throws them away, with no other symptom than the model behaving + // as if memory were empty. + enableOmni(); + runOmniMemorySideQueryMock.mockResolvedValue({ + result: recallResult, + resourceIds: ['media-1-abcdef01'], + }); + + const request = await runUserQuery(); + + const reminder = formatOmniMemorySideQueryReminder(recallResult); + expect(request).toContain(reminder); + // The user's own text has been flattened to a bare string by this + // point; the reminder block must sit ahead of it. + const userPartIndex = request.indexOf('what changed in this clip?'); + expect(userPartIndex).toBeGreaterThanOrEqual(0); + expect(request.indexOf(reminder)).toBeLessThan(userPartIndex); + + // The selector must see the request as it stands BEFORE injection — + // that text (and the media handles inside it) is the only thing + // scoping passive recall to what this request actually references. + const [params] = runOmniMemorySideQueryMock.mock.calls[0] as [ + { requestParts: unknown[]; promptId?: string }, + ]; + expect(params.promptId).toBe('prompt-omni-recall'); + expect(params.requestParts).toContain('what changed in this clip?'); + expect(params.requestParts).not.toContain(reminder); + }); + + it('injects nothing when the selector declines', async () => { + // A null outcome is the normal path in active mode (where the recall + // TOOL is registered instead — see memory-side-query.test.ts for the + // D10 gate), and also covers a request with no handles, a selector + // timeout, and every other degraded case. None of them may put an + // empty 【媒体记忆】 shell in front of the user's question. + enableOmni(); + runOmniMemorySideQueryMock.mockResolvedValue(null); + + const request = await runUserQuery(); + + expect(runOmniMemorySideQueryMock).toHaveBeenCalledTimes(1); + expect( + request.filter( + (part) => + typeof part === 'object' && + part !== null && + 'text' in part && + (part as { text?: string }).text?.includes('【媒体记忆】'), + ), + ).toEqual([]); + }); + + it('never consults the selector when omni is off', async () => { + // Omni is opt-in and experimental: a session without it must pay no + // recall latency and touch no memory store. + runOmniMemorySideQueryMock.mockResolvedValue({ + result: recallResult, + resourceIds: [], + }); + + const request = await runUserQuery(); + + expect(runOmniMemorySideQueryMock).not.toHaveBeenCalled(); + expect(request).not.toContain( + formatOmniMemorySideQueryReminder(recallResult), + ); + }); + }); + describe('setTools — progressive MCP reminders', () => { function getRegistryMock() { return vi.mocked(mockConfig.getToolRegistry)() as unknown as { diff --git a/packages/core/src/core/client.ts b/packages/core/src/core/client.ts index e612cb77029..3af7539974c 100644 --- a/packages/core/src/core/client.ts +++ b/packages/core/src/core/client.ts @@ -2933,6 +2933,43 @@ export class GeminiClient { systemReminders.unshift(userQueryMemory.prompt); } + // Omni passive media-memory recall (memory design M §9.3, D10 + // sideQuery mode): a bounded selector reads what memory knows + // about the media handles THIS request carries and the chosen + // entries are injected here — strictly before the main request + // is sent (never retrofitted into a later turn). Latency is + // bounded by sideQuery.timeoutMs; the no-op cases (mode active, + // memory off, no handles in the request) return null without + // model traffic, and every failure degrades to no injection. + // Optional call: stub configs in tests may omit the method. + if (this.config.isOmniEnabled?.()) { + const { runOmniMemorySideQuery, formatOmniMemorySideQueryReminder } = + await import('../omni/memory-side-query.js'); + const omniRecall = await runOmniMemorySideQuery({ + config: this.config, + requestParts: requestToSend, + promptId: prompt_id, + ...(signal !== undefined ? { signal } : {}), + }); + if (omniRecall?.result) { + systemReminders.push( + formatOmniMemorySideQueryReminder(omniRecall.result), + ); + } else if (omniRecall?.reason) { + // A degraded passive recall is invisible by construction: the + // turn proceeds normally, just without the memory it was + // supposed to carry. With a pinned-but-unavailable selector + // model that is a permanent outage of the feature with nothing + // to see, so the reason is recorded (memory design M §9.3 + // obliges recording it) rather than dropped on the floor. + debugLogger.debug( + `omni passive media-memory recall degraded ` + + `(${omniRecall.reason}) for ` + + `${omniRecall.resourceIds.length} resource(s)`, + ); + } + } + requestToSend = [...systemReminders, ...requestToSend]; } diff --git a/packages/core/src/core/coreToolScheduler.ts b/packages/core/src/core/coreToolScheduler.ts index f643ccb349d..c7e4545dd80 100644 --- a/packages/core/src/core/coreToolScheduler.ts +++ b/packages/core/src/core/coreToolScheduler.ts @@ -4849,6 +4849,37 @@ export class CoreToolScheduler { artifacts: toolResult.artifacts, } : undefined; + // Model/client-origin successes enter the SAME OmniPolicySucceeded + // boundary the fixed-policy orchestrator uses (memory design M + // §7.1/§17): without this, an evidence-gathering call the advisor + // suggested succeeds but is never recorded, so the next recall + // reports the identical gap and the work is re-paid every session. + // Awaited (the commit must not race the next turn's recall) and + // internally never-throwing — collection failure cannot affect the + // tool result (D12). + if ( + policyArtifacts && + policyArtifacts.executionOrigin.kind !== 'fixed_policy' && + scheduledCall.tool.mediaPolicyDescriptor + ) { + const { collectModelPolicyCall } = await import( + '../omni/policy/model-call-collection.js' + ); + await collectModelPolicyCall({ + config: this.config, + batch: policyArtifacts, + descriptor: scheduledCall.tool.mediaPolicyDescriptor, + args: scheduledCall.invocation.params as Record, + // The REAL execution window (approval wait excluded). Without + // it the record would hold the collection window instead, and + // a 98-minute audio extraction reads as 0.6s. + ...('executionStartTime' in scheduledCall && + typeof scheduledCall.executionStartTime === 'number' + ? { startedAt: scheduledCall.executionStartTime } + : {}), + signal, + }); + } const successResponse: ToolCallResponseInfo = { callId, responseParts: response, diff --git a/packages/core/src/omni/disclosure.ts b/packages/core/src/omni/disclosure.ts index 46a1e11f862..d990c261267 100644 --- a/packages/core/src/omni/disclosure.ts +++ b/packages/core/src/omni/disclosure.ts @@ -58,3 +58,29 @@ export function formatTranscriptText( ): string { return `${OMNI_TRANSCRIPT_TEXT_PREFIX}${displayName}：${transcript}`; } + +/** Marks a text Part as a session resource-handle annotation: the opaque + * `resourceId` minted for a delivered media resource (memory design M + * §5.2). The model references this handle in `omni_recall_media_memory` + * requests (and other omni tools that accept a resourceId) — it is the + * ONLY identity the model ever sees for the underlying file. */ +export const OMNI_RESOURCE_HANDLE_TEXT_PREFIX = '【媒体资源】'; + +/** Model-facing resource-handle annotation for one delivered resource. */ +export function formatResourceHandleText( + displayName: string, + resourceId: string, +): string { + return `${OMNI_RESOURCE_HANDLE_TEXT_PREFIX}${displayName}：${resourceId}`; +} + +/** Extract the resourceId from a handle annotation emitted by + * {@link formatResourceHandleText}, or undefined for any other text. The + * handle grammar is harness-minted (`media--`), so parsing keys + * on it rather than on the displayName (which may itself contain the + * separator). */ +export function parseResourceHandleText(text: string): string | undefined { + if (!text.startsWith(OMNI_RESOURCE_HANDLE_TEXT_PREFIX)) return undefined; + const match = /：(media-\d+-[0-9a-f]+)$/.exec(text); + return match?.[1]; +} diff --git a/packages/core/src/omni/guard.test.ts b/packages/core/src/omni/guard.test.ts index dc18999788b..c208b239089 100644 --- a/packages/core/src/omni/guard.test.ts +++ b/packages/core/src/omni/guard.test.ts @@ -8,16 +8,24 @@ import { describe, expect, it, vi } from 'vitest'; import type { Config } from '../config/config.js'; import { assertWithinByteLimit, + assertWithinDurationLimit, assertWithinTokenLimit, effectiveMaxUploadFileBytes, OmniTransportGuardError, } from './guard.js'; import type { RecognizedMedia } from './recognition.js'; -function cfg(overrides: { maxBytes?: number; maxTokens?: number }): Config { +function cfg(overrides: { + maxBytes?: number; + maxTokens?: number; + maxDurationSeconds?: number; +}): Config { return { getOmniMaxUploadFileBytes: vi.fn().mockReturnValue(overrides.maxBytes), getOmniMaxEstimatedTokens: vi.fn().mockReturnValue(overrides.maxTokens), + getOmniMaxDurationSeconds: vi + .fn() + .mockReturnValue(overrides.maxDurationSeconds), } as unknown as Config; } @@ -106,3 +114,75 @@ describe('token guard', () => { expect(est.status).toBe('unavailable'); }); }); + +describe('duration guard', () => { + const FILM_98MIN: RecognizedMedia = { + modality: 'video', + detectedMimeType: 'video/mp4', + sizeBytes: 474_864_706, + metadata: { width: 1920, height: 1040, durationMs: 5_895_767 }, + }; + + it('is disabled when unset or non-positive', () => { + for (const c of [cfg({}), cfg({ maxDurationSeconds: 0 })]) { + expect(() => + assertWithinDurationLimit(c, FILM_98MIN, 'film'), + ).not.toThrow(); + } + }); + + it('rejects a file that clears the byte limit but is still too long', () => { + // The real failure this guard exists for: 2.44 GB downscaled to 474 MB + // passes a 1 GiB byte ceiling, uploads, and is then refused by the + // provider for duration — after paying for the transcode. + const config = cfg({ maxDurationSeconds: 3600 }); + expect(() => + assertWithinByteLimit(config, FILM_98MIN.sizeBytes, 'film'), + ).not.toThrow(); + expect(() => assertWithinDurationLimit(config, FILM_98MIN, 'film')).toThrow( + OmniTransportGuardError, + ); + }); + + it('names duration as the reason and says downscaling will not help', () => { + try { + assertWithinDurationLimit( + cfg({ maxDurationSeconds: 3600 }), + FILM_98MIN, + 'film', + ); + throw new Error('expected a guard rejection'); + } catch (err) { + const message = (err as Error).message; + expect(message).toContain('duration limit'); + expect(message).toContain('5896s > 3600s'); + expect(message).toMatch(/Clip a shorter span/); + } + }); + + it('accepts media within the limit', () => { + expect(() => + assertWithinDurationLimit( + cfg({ maxDurationSeconds: 3600 }), + VIDEO_8MIN, + 'clip', + ), + ).not.toThrow(); + }); + + it('never rejects when the probe reported no duration', () => { + const noDuration: RecognizedMedia = { + modality: 'video', + detectedMimeType: 'video/mp4', + sizeBytes: 10, + metadata: {}, + }; + expect(() => + assertWithinDurationLimit( + cfg({ maxDurationSeconds: 1 }), + noDuration, + 'x', + ), + ).not.toThrow(); + }); +}); diff --git a/packages/core/src/omni/guard.ts b/packages/core/src/omni/guard.ts index 6acebb0865e..368f03f3569 100644 --- a/packages/core/src/omni/guard.ts +++ b/packages/core/src/omni/guard.ts @@ -18,6 +18,16 @@ export const DEFAULT_OMNI_MAX_UPLOAD_FILE_BYTES = 1024 * 1024 * 1024; * surface the message; there is no silent degradation. Messages must stay * free of absolute paths (they can reach model-visible content). */ export class OmniTransportGuardError extends Error { + /** + * Session handle for the SOURCE, when delivery had already bound one + * before the guard's verdict (memory design M §5.2). A rejection is the + * case where recall matters most — the model never sees the media, so + * this handle is its only remaining route to the content — and every + * throw site sits downstream of the bind, so dropping it would strand + * the resource for the rest of the session. + */ + sessionResourceId?: string; + constructor(message: string, options?: { cause?: unknown }) { super(message, options); this.name = 'OmniTransportGuardError'; @@ -50,6 +60,44 @@ export function assertWithinByteLimit( } } +/** + * Duration-dimension guard. Runs alongside the token check (both need + * probe metadata) and BEFORE store/upload. + * + * Byte and token limits alone cannot express the provider's duration cap, + * and that gap is not theoretical: a 98-minute film downscaled to 474 MB + * clears a 1 GB byte limit, gets uploaded, and is then rejected by the API + * for being too long. The guard reported success, the creator got an + * opaque 400, and a full transcode was paid for nothing. Duration is a + * transport limit like any other, so it belongs here — where exceeding it + * turns into an honest omission that names duration as the reason. + * + * Threshold semantics mirror the token guard + * (`omni.processing.transportGuard.maxDurationSeconds`): unset / 0 / + * negative disables it; a missing duration never rejects (metadata is + * never guessed). + */ +export function assertWithinDurationLimit( + config: Config, + media: RecognizedMedia, + displayName: string, +): void { + const maxSeconds = config.getOmniMaxDurationSeconds?.(); + if (maxSeconds === undefined || maxSeconds <= 0) return; + const durationMs = media.metadata.durationMs; + if (durationMs === undefined || !Number.isFinite(durationMs)) return; + const seconds = durationMs / 1000; + if (seconds > maxSeconds) { + throw new OmniTransportGuardError( + `${displayName} exceeds the omni duration limit: ` + + `${Math.round(seconds)}s > ${maxSeconds}s ` + + `(omni.processing.transportGuard.maxDurationSeconds). ` + + `Clip a shorter span — downscaling cannot bring a long file ` + + `within a duration limit.`, + ); + } +} + /** * Token-dimension guard. Runs AFTER recognition (needs probe metadata) and * BEFORE store/upload, so an oversized input costs one probe — not a copy diff --git a/packages/core/src/omni/index.test.ts b/packages/core/src/omni/index.test.ts index fe74ae809d4..3bbb3dd64b2 100644 --- a/packages/core/src/omni/index.test.ts +++ b/packages/core/src/omni/index.test.ts @@ -9,6 +9,10 @@ import fs from 'node:fs/promises'; import os from 'node:os'; import path from 'node:path'; import type { Config } from '../config/config.js'; +import { + MediaMemoryService, + MediaResourceRegistry, +} from '../services/media-memory/index.js'; import { AuthType } from '../core/contentGenerator.js'; import { isOmniDeliveryActive } from './index.js'; import { effectiveMaxDownloadFileBytes } from './index.js'; @@ -292,6 +296,72 @@ describe('readMediaViaOmniDelivery result shape', () => { }); }); + it('leads with the session resource handle when memory is on', async () => { + vi.doMock('./ffmpeg.js', () => ({ + isFfmpegAvailable: vi.fn().mockResolvedValue(true), + isFfprobeAvailable: vi.fn().mockResolvedValue(true), + })); + vi.doMock('./recognition.js', () => ({ + recognizeMediaFile: vi + .fn() + .mockResolvedValue( + mockRecognized('image', { width: 1920, height: 1080 }), + ), + hashFileSha256: vi.fn().mockResolvedValue('a'.repeat(64)), + extensionForMime: vi.fn().mockReturnValue('.png'), + })); + vi.doMock('./storage.js', () => ({ + OmniObjectStore: class { + async putFile() { + return { objectPath: '/tmp/obj.png', deduped: false }; + } + getOmniRootDir() { + return tmpDir; + } + }, + })); + vi.doMock('./upload.js', () => ({ + DashScopeUploader: class { + async uploadFile() { + return 'oss://bucket/key'; + } + }, + OSS_URL_PREFIX: 'oss://', + })); + const { readMediaViaOmniDelivery } = await import('./index.js'); + const { MediaResourceRegistry } = await import( + '../services/media-memory/index.js' + ); + const registry = new MediaResourceRegistry(); + + const result = await readMediaViaOmniDelivery({ + filePath: await realFile('pic.png'), + config: { + ...deliveryConfig(), + getOmniMemoryConfig: () => ({ + collection: { maxInlineTextBytes: 4096 }, + }), + getOmniMediaResourceRegistry: () => registry, + } as unknown as Config, + displayName: 'pic.png', + relativePathForDisplay: 'pic.png', + expectedModality: 'image', + }); + + const parts = result.llmContent as Array>; + expect(parts).toHaveLength(3); + // Handle part FIRST — the hint/disclosure chain keeps its adjacency + // to the media part (D8), and the model learns the recall handle. + const handleText = parts[0]!['text'] as string; + expect(handleText).toContain('【媒体资源】pic.png：'); + const resourceId = handleText.split('：')[1]!; + expect(registry.resolve(resourceId)).toMatchObject({ + mediaType: 'image', + }); + expect(parts[1]!['text']).toContain('zoom_image'); + expect(parts[2]).toHaveProperty('fileData'); + }); + it('returns a bare fileData part for audio (no zoom hint)', async () => { vi.doMock('./ffmpeg.js', () => ({ isFfmpegAvailable: vi.fn().mockResolvedValue(true), @@ -820,6 +890,9 @@ describe('processMediaForOmniDelivery fixed-policy integration', () => { getObjectsDir() { return objectsDir; } + objectPathFor(sha256: string, extension: string) { + return path.join(objectsDir, `${sha256}${extension}`); + } }, })); vi.doMock('./upload.js', () => ({ @@ -1783,6 +1856,60 @@ describe('processMediaForOmniDelivery fixed-policy integration', () => { expect(result.errorType).toBeUndefined(); }); + it('readMediaViaOmniDelivery keeps the recall handle on an omitted media', async () => { + // The omission branch is the one shape that puts NO media part in front + // of the model. Without the handle leading it, the withheld resource has + // no identity the model can name — it can neither recall what memory + // knows about it nor ask a policy tool to reprocess it into something + // deliverable, and paths are never surfaced (M §5.2). + const runMock = vi.fn().mockResolvedValue({ + deliveries: [ + { + filePath: path.join(tmpDir, 'objects', 'deadbeef.jpg'), + recognized: { ...DEGRADED_RECOGNIZED, sizeBytes: 900 }, + sha256: 'b'.repeat(64), + degraded: true, + }, + ], + records: [], + fileDeliveries: [], + }); + const { mod } = await armPipeline(runMock); + const { MediaResourceRegistry } = await import( + '../services/media-memory/index.js' + ); + const registry = new MediaResourceRegistry(); + + const result = await mod.readMediaViaOmniDelivery({ + filePath: await realFile('pic.png'), + config: { + ...policyConfig({ maxUploadFileBytes: 500 }), + getOmniMemoryConfig: () => ({ + collection: { maxInlineTextBytes: 4096 }, + }), + getOmniMediaResourceRegistry: () => registry, + } as unknown as Config, + displayName: 'pic.png', + relativePathForDisplay: 'pic.png', + expectedModality: 'image', + }); + + // With memory off this branch collapses to a bare notice string (test + // above); a bound handle must turn it into a part array led by the + // handle, with the notice standing in for the media behind it. + const parts = result.llmContent as Array>; + expect(parts).toHaveLength(2); + const handleText = parts[0]!['text'] as string; + expect(handleText).toContain('【媒体资源】pic.png：'); + expect(registry.resolve(handleText.split('：')[1]!)).toMatchObject({ + mediaType: 'image', + }); + expect(parts[1]!['text']).toMatch(/^【媒体省略】pic\.png：/); + expect(result.returnDisplay).toBe( + 'Media omitted by the omni transport guard: pic.png', + ); + }); + it('threads the quarantine retention settings into startup recovery', async () => { const recoveryMock = vi.fn().mockResolvedValue(undefined); vi.doMock('./recovery.js', () => ({ @@ -1825,4 +1952,156 @@ describe('processMediaForOmniDelivery fixed-policy integration', () => { }, ); }); + + it('anchors tool-result media to the object store, not its staging file', async () => { + // The tool-result funnel writes bytes to a staging `.part` and deletes + // it in `finally` the same turn, while this delivery promotes the same + // bytes into the content-addressed store. Recording the staging path + // as the persistent identity handed the model a handle resolving to a + // deleted file (ENOENT for any policy tool pointed at it) and made + // recall report `artifact_unavailable` for an artifact that persists. + const runMock = vi + .fn() + .mockResolvedValue({ deliveries: [], records: [], fileDeliveries: [] }); + const { mod } = await armPipeline(runMock); + const registry = new MediaResourceRegistry(); + const stagingPath = await realFile('tool-media.part'); + const config = { + ...policyConfig({ policies: [] }), + getOmniMemoryConfig: () => ({ collection: { maxInlineTextBytes: 4096 } }), + getOmniMediaResourceRegistry: () => registry, + } as unknown as Config; + + const delivery = await mod.processMediaForOmniDelivery( + stagingPath, + config, + { origin: 'tool' }, + ); + + const binding = registry.resolve(delivery.resourceId!); + expect(binding).toBeDefined(); + // Content-addressed location derived from the hash — survives the + // funnel's cleanup of the staging file. + expect(binding!.fileRef).toBe( + path.join(tmpDir, 'objects', `${'a'.repeat(64)}.jpg`), + ); + expect(binding!.fileRef).not.toBe(stagingPath); + // A user file keeps its own path (its bytes stay in place, S §4). + const userRegistry = new MediaResourceRegistry(); + const userDelivery = await mod.processMediaForOmniDelivery( + await realFile('photo.png'), + { + ...config, + getOmniMediaResourceRegistry: () => userRegistry, + } as unknown as Config, + ); + expect(userRegistry.resolve(userDelivery.resourceId!)!.fileRef).toContain( + 'photo.png', + ); + }); + + it('anchors URL media to the object store and records the URL as its source', async () => { + // The URL funnel stages its download under an opaque temp name and + // deletes it in `finally` the same turn — the same lifetime as + // tool-result media, missed when C9 fixed that funnel. Binding the + // staging path handed the model a handle that resolves to ENOENT for + // the rest of the session, and made cross-session recall report + // `artifact_unavailable` for bytes the object store still holds. + const runMock = vi + .fn() + .mockResolvedValue({ deliveries: [], records: [], fileDeliveries: [] }); + const { mod } = await armPipeline(runMock); + const registry = new MediaResourceRegistry(); + const stagingPath = await realFile('dl-3f9a.part'); + const config = { + ...policyConfig({ policies: [] }), + getOmniMemoryConfig: () => ({ collection: { maxInlineTextBytes: 4096 } }), + getOmniMediaResourceRegistry: () => registry, + } as unknown as Config; + + const delivery = await mod.processMediaForOmniDelivery( + stagingPath, + config, + { + displayName: 'clip.mp4', + sourceUrl: 'https://example.com/media/clip.mp4', + }, + ); + + const binding = registry.resolve(delivery.resourceId!); + expect(binding).toBeDefined(); + expect(binding!.fileRef).toBe( + path.join(tmpDir, 'objects', `${'a'.repeat(64)}.jpg`), + ); + expect(binding!.fileRef).not.toBe(stagingPath); + // The durable identity of URL media is the URL itself — recorded as + // the version's source so provenance names where the bytes came from. + const snapshot = JSON.parse( + await fs.readFile(path.join(tmpDir, 'memory.json'), 'utf8'), + ); + const version = Object.values( + snapshot.versions as Record, + ).find( + (v) => + JSON.stringify(v.source) === + JSON.stringify({ + protocol: 'url', + locator: 'https://example.com/media/clip.mp4', + }), + ); + expect(version).toBeDefined(); + }); + + it('mounts memory-known deliveries into the session resource registry', async () => { + const degradedPath = path.join(tmpDir, 'objects', 'deadbeef.jpg'); + const derivedBinding = { + fileId: 'f-derived', + fileVersionId: 'v-derived', + rootFileId: 'f-root', + }; + const runMock = vi.fn().mockResolvedValue({ + deliveries: [ + { + filePath: degradedPath, + recognized: DEGRADED_RECOGNIZED, + sha256: 'b'.repeat(64), + degraded: true, + memoryBinding: derivedBinding, + }, + ], + records: [], + fileDeliveries: [], + }); + const { mod } = await armPipeline(runMock); + const registry = new MediaResourceRegistry(); + const filePath = await realFile('pic.png'); + const config = { + ...policyConfig(), + getOmniMemoryConfig: () => ({ collection: { maxInlineTextBytes: 4096 } }), + getOmniMediaResourceRegistry: () => registry, + } as unknown as Config; + + const delivery = await mod.processMediaForOmniDelivery(filePath, config); + + // The derivative the model actually received is session-addressable, + // resolving back to its harness-side locator and memory identity. + const derived = registry.resolveVersion('v-derived'); + expect(derived).toMatchObject({ + ...derivedBinding, + fileRef: degradedPath, + mediaType: 'image', + }); + expect(registry.resolve(derived!.resourceId)).toBe(derived); + // The original source stays addressable too, under the version the + // collection pass recorded for its content hash. + const memory = new MediaMemoryService(tmpDir); + const sourceBinding = await memory.findBindingBySha256('a'.repeat(64)); + expect(sourceBinding).toBeDefined(); + const source = registry.resolveVersion(sourceBinding!.fileVersionId); + expect(source?.fileRef).toBe(filePath); + expect(source?.mediaType).toBe('image'); + // The delivery discloses the SOURCE handle (M §5.2): the model's way + // into recall is the source identity, not the derivative's. + expect(delivery.resourceId).toBe(source!.resourceId); + }); }); diff --git a/packages/core/src/omni/index.ts b/packages/core/src/omni/index.ts index dcc3055c8a6..8374bf8d6bf 100644 --- a/packages/core/src/omni/index.ts +++ b/packages/core/src/omni/index.ts @@ -18,6 +18,7 @@ import { } from './estimation.js'; import { assertWithinByteLimit, + assertWithinDurationLimit, assertWithinTokenLimit, effectiveMaxUploadFileBytes, OmniTransportGuardError, @@ -40,6 +41,7 @@ import { OmniDegradationCache } from './policy/degradation-cache.js'; import { formatDisclosureText, formatOmissionText, + formatResourceHandleText, formatTranscriptText, } from './disclosure.js'; import { @@ -49,6 +51,14 @@ import { } from './policy/orchestrator.js'; import { buildSessionConditionNamespace } from './policy/session-context.js'; import type { OmniProcessingConfigView } from './policy/types.js'; +import { + MediaMemoryService, + MEDIA_DETECTOR_VERSION, + type MediaMemoryBinding, + type MediaResourceRegistry, + type OmniMediaRegistryView, + type OmniMemoryConfigView, +} from '../services/media-memory/index.js'; export { assertOmniRuntimeDependencies, @@ -74,6 +84,7 @@ export { export { OmniTransportGuardError, DEFAULT_OMNI_MAX_UPLOAD_FILE_BYTES, + assertWithinDurationLimit, } from './guard.js'; export { downloadMediaUrl, @@ -99,6 +110,7 @@ export { OMNI_TRANSCRIPT_TEXT_PREFIX, formatDisclosureText, formatOmissionText, + formatResourceHandleText, formatTranscriptText, isDisclosureText, } from './disclosure.js'; @@ -196,6 +208,11 @@ export interface OmniMediaDelivery { * entry as [disclosure?, fileData] (or the omission notice) after the * primary media Part and before any transcripts. */ additionalMedia?: OmniAdditionalMediaDelivery[]; + /** Opaque session handle for the SOURCE media (M §5.2), present iff + * media memory recorded it. Consumers disclose it next to the delivered + * content so the model can reference the resource in recall requests — + * the handle stands in for the path the model must never see. */ + resourceId?: string; } /** One extra media deliverable of a multi-output fixed policy. */ @@ -294,6 +311,10 @@ function evaluateTransportLimits( ): { estimate: OmniTokenEstimate; violation?: string } { try { assertWithinByteLimit(config, recognized.sizeBytes, displayName); + // Duration is a transport limit too: downscaling can bring a long + // file under the byte ceiling but never under a duration cap, so the + // guard must reject it here instead of letting the provider do it. + assertWithinDurationLimit(config, recognized, displayName); return { estimate: assertWithinTokenLimit(config, recognized, displayName), }; @@ -344,6 +365,14 @@ export async function processMediaForOmniDelivery( /** Provenance for fixed-policy origin matching. Defaults to 'user'; * the tool-result funnel passes 'tool'. */ origin?: 'user' | 'tool'; + /** + * The URL this media was downloaded from. Set by the URL funnel, whose + * `filePath` is a staging download it deletes in its `finally` THIS + * turn — so, exactly like tool-result media, its memory identity must + * anchor to the content-addressed object store, never to the staging + * path. The URL itself is recorded as the version's source locator. + */ + sourceUrl?: string; }, ): Promise { const { expectedModality, signal } = options ?? {}; @@ -437,7 +466,124 @@ export async function processMediaForOmniDelivery( ), } : undefined; - let final: PolicyDeliveryResource = { filePath, recognized }; + // Media-memory collection (S5, design M §6). Same structural-view + // pattern as the processing config; absent accessor or config = off. + // Recording FileRecognized needs the source's content hash NOW, so the + // hash is paid upfront and seeded through the pipeline (source → + // WorkItem → final) — uploadResource's lazy hash never re-pays it. A + // memory failure (hash or store) never blocks delivery: collection is + // skipped and the pipeline continues exactly as without memory. + const memoryConfig = (config as OmniMemoryConfigView).getOmniMemoryConfig?.(); + const memoryService = memoryConfig + ? new MediaMemoryService(store.getOmniRootDir(), { + maxInlineTextBytes: memoryConfig.collection.maxInlineTextBytes, + }) + : undefined; + // Session resource registry (M §5.2): every memory-known resource this + // delivery puts in front of the model gets an opaque session handle, + // making it addressable by recall without ever exposing a path. + const registry = memoryService + ? (config as OmniMediaRegistryView).getOmniMediaResourceRegistry?.() + : undefined; + const bindSessionResource = ( + item: PolicyDeliveryResource, + /** Overrides the locator the handle resolves to — used for the source + * of tool-result media, whose `filePath` is an ephemeral staging file + * while its persistent bytes live in the object store. */ + fileRefOverride?: string, + ): ReturnType | undefined => + registry && item.memoryBinding + ? registry.bind({ + ...item.memoryBinding, + fileRef: fileRefOverride ?? item.filePath, + mediaType: item.recognized.modality, + }) + : undefined; + let sourceSha256: string | undefined; + let sourceBinding: MediaMemoryBinding | undefined; + let sourceFileRef = filePath; + if (memoryService) { + try { + sourceSha256 = await hashFileSha256(filePath, signal); + } catch (err) { + if (signal?.aborted) throw err; + debugLogger.debug( + `omni memory: hashing ${displayName} failed, skipping collection: ` + + `${sanitizeErrorMessage(err, [filePath])}`, + ); + } + if (sourceSha256) { + // Persistent identity of the SOURCE. A user file's bytes stay in + // place, so its own path is the identity (S §4). A tool-result + // file's path is a staging `.part` the funnel deletes in its + // `finally` THIS turn — recording it would hand out a handle that + // resolves to a deleted path (ENOENT for any policy tool the model + // points at it) and make recall report `artifact_unavailable` for + // an artifact that actually persists. Its bytes are promoted into + // the content-addressed object store by this same delivery, and + // that location is derivable from the hash — so name it directly. + // If promotion never happens (the transport guard omitted the + // media), the ref dangles and recall says `artifact_unavailable`: + // honest, because the bytes were genuinely not retained. + const origin = options?.origin ?? 'user'; + // URL media shares the tool-result lifetime: its local file is a + // staging download deleted this turn, so it anchors the same way. + const ephemeralSource = + origin === 'tool' || options?.sourceUrl !== undefined; + sourceFileRef = ephemeralSource + ? store.objectPathFor( + sourceSha256, + extensionForMime(recognized.detectedMimeType), + ) + : filePath; + sourceBinding = await memoryService.recordFileRecognized({ + fileRef: sourceFileRef, + sha256: sourceSha256, + mediaType: recognized.modality, + metadata: recognized.metadata, + sizeBytes: recognized.sizeBytes, + mimeType: recognized.detectedMimeType, + origin, + source: + options?.sourceUrl !== undefined + ? { protocol: 'url', locator: options.sourceUrl } + : origin === 'tool' + ? { protocol: 'managed', locator: `sha256/${sourceSha256}` } + : { protocol: 'local', locator: displayName }, + recognition: { + ingestionConfigHash: '', + detectorVersion: MEDIA_DETECTOR_VERSION, + probeStatus: 'complete', + }, + }); + } + } + let final: PolicyDeliveryResource = { + filePath, + recognized, + sha256: sourceSha256, + memoryBinding: sourceBinding, + }; + // The source is always addressable by recall, even when preprocessing + // later replaces the delivered bytes with a derivative. Its handle is + // the one disclosed to the model (M §5.2): recall requests and future + // evidence-gathering tool calls reference the SOURCE, never a path. + const sessionResourceId = bindSessionResource( + final, + sourceFileRef, + )?.resourceId; + /** A guard verdict that carries the source handle with it, so a caller + * withholding the bytes can still tell the model what to recall. */ + const guardRejection = ( + message: string, + options?: { cause?: unknown }, + ): OmniTransportGuardError => { + const err = new OmniTransportGuardError(message, options); + if (sessionResourceId !== undefined) { + err.sessionResourceId = sessionResourceId; + } + return err; + }; /** Media deliverables beyond the primary (multi-output fixed policies). */ let extraDeliveries: PolicyDeliveryResource[] = []; // Transcript-protocol text deliverables (upstream P §6.2) accumulated @@ -470,6 +616,7 @@ export async function processMediaForOmniDelivery( uploadCacheHit: false, degraded: true, transcripts, + resourceId: sessionResourceId, ...extras, }); const policies = processingConfig?.fixedPolicies ?? []; @@ -484,6 +631,7 @@ export async function processMediaForOmniDelivery( recognized, displayName, origin: options?.origin ?? 'user', + sha256: sourceSha256, }, { store, @@ -491,6 +639,9 @@ export async function processMediaForOmniDelivery( signal, limits: processingConfig?.limits, conditionContext, + memory: memoryService + ? { service: memoryService, sourceBinding } + : undefined, }, )); } catch (err) { @@ -525,6 +676,10 @@ export async function processMediaForOmniDelivery( } final = deliveries[0]; extraDeliveries = deliveries.slice(1); + // Every delivered derivative becomes session-addressable (bind is + // idempotent — the source keeps its already-issued handle if a no_op + // policy passed it through unchanged). + for (const delivery of deliveries) bindSessionResource(delivery); } // Hash → upload-cache lookup → store promotion → upload. Shared by the @@ -702,6 +857,7 @@ export async function processMediaForOmniDelivery( recognized: final.recognized, displayName, origin: options?.origin ?? 'user', + sha256: final.sha256, }, { store, @@ -709,6 +865,12 @@ export async function processMediaForOmniDelivery( signal, limits: processingConfig.limits, conditionContext, + // The guard pass derives FROM the current final resource, so + // its memory lineage hangs off that resource's own binding + // (a derivative's version when preprocessing degraded it). + memory: memoryService + ? { service: memoryService, sourceBinding: final.memoryBinding } + : undefined, }, )); } catch (err) { @@ -720,7 +882,7 @@ export async function processMediaForOmniDelivery( // over the limit" already stands, so consumers with an inline // fallback (the tool-result funnel) must withhold the bytes, not // fall back to delivering exactly what the guard rejected. - throw new OmniTransportGuardError( + throw guardRejection( `Transport-guard processing failed for ${displayName}: ` + `${sanitizeErrorMessage(err, [final.filePath, store.getOmniRootDir()])}`, { cause: err }, @@ -743,7 +905,7 @@ export async function processMediaForOmniDelivery( if (deliveries.length !== 1) { // Same guard-error class as the pass failure above: the violation // verdict stands, so inline fallbacks must withhold. - throw new OmniTransportGuardError( + throw guardRejection( `Transport-guard policies produced ${deliveries.length} media deliverables for ${displayName}; exactly one is supported.`, ); } @@ -766,12 +928,13 @@ export async function processMediaForOmniDelivery( } else if (priorDisclosure) { final = { ...final, disclosure: priorDisclosure }; } + bindSessionResource(final); guard = evaluateTransportLimits(config, final.recognized, displayName); } } if (guard.violation) { if (!processingConfig) { - throw new OmniTransportGuardError(guard.violation); + throw guardRejection(guard.violation); } debugLogger.debug( `omni ${final.recognized.modality} explicitly omitted (transport guard): ${guard.violation}`, @@ -789,6 +952,7 @@ export async function processMediaForOmniDelivery( omission: { reason: guard.violation }, transcripts: transcripts.length > 0 ? transcripts : undefined, additionalMedia: await processAdditionalMedia(), + resourceId: sessionResourceId, }; } const tokenEstimate = guard.estimate; @@ -805,6 +969,7 @@ export async function processMediaForOmniDelivery( degraded: final.degraded, transcripts: transcripts.length > 0 ? transcripts : undefined, additionalMedia: await processAdditionalMedia(), + resourceId: sessionResourceId, }; } @@ -871,6 +1036,13 @@ export async function readMediaViaOmniDelivery(params: { displayName, delivery.additionalMedia, ); + // Session resource handle (M §5.2): leads the part group in every + // branch — even an omitted/transcript-only delivery leaves the model + // a handle to recall or reprocess the source. Placed FIRST so the + // disclosure keeps its D8 adjacency to the media part. + const handleParts = delivery.resourceId + ? [{ text: formatResourceHandleText(displayName, delivery.resourceId) }] + : []; if (delivery.omission) { // Explicit omission (policy design §10.2): the media is withheld and // the omission notice text stands in its place. Not an error — the @@ -880,8 +1052,15 @@ export async function readMediaViaOmniDelivery(params: { }; return { llmContent: - transcriptParts.length > 0 || additionalParts.length > 0 - ? [omissionPart, ...additionalParts, ...transcriptParts] + transcriptParts.length > 0 || + additionalParts.length > 0 || + handleParts.length > 0 + ? [ + ...handleParts, + omissionPart, + ...additionalParts, + ...transcriptParts, + ] : omissionPart.text, returnDisplay: `Media omitted by the omni transport guard: ${relativePathForDisplay}`, tokenEstimate: delivery.tokenEstimate, @@ -898,6 +1077,7 @@ export async function readMediaViaOmniDelivery(params: { : []; return { llmContent: [ + ...handleParts, ...disclosureParts, ...additionalParts, ...transcriptParts, @@ -913,7 +1093,9 @@ export async function readMediaViaOmniDelivery(params: { displayName, }, }; - const parts: Array<{ text: string } | typeof fileDataPart> = []; + const parts: Array<{ text: string } | typeof fileDataPart> = [ + ...handleParts, + ]; const { width, height } = delivery.recognized.metadata; if ( delivery.recognized.modality === 'image' && @@ -984,3 +1166,11 @@ export function effectiveMaxDownloadFileBytes(config: Config): number { } return uploadCap; } + +// Re-anchoring is reached from the CLI's `@`-reference funnel, which already +// loads this module dynamically for every other omni call it makes. Kept off +// the ROOT barrel deliberately: that barrel is statically imported across the +// CLI, and every module added to its graph regroups esbuild's chunks — which +// is how the ACP agent's static closure acquired iconv-lite's 550 KB of +// encoding tables (scripts/check-serve-fast-path-bundle.js caught it). +export { reanchorRememberedMedia } from './memory-recall.js'; diff --git a/packages/core/src/omni/json-cache-file.ts b/packages/core/src/omni/json-cache-file.ts index 4cfed9375a9..3fd6ae6e687 100644 --- a/packages/core/src/omni/json-cache-file.ts +++ b/packages/core/src/omni/json-cache-file.ts @@ -13,10 +13,15 @@ import { createDebugLogger, type DebugLogger } from '../utils/debugLogger.js'; * per use site, and safe-tool batches run deliveries concurrently in one * process — unserialized load-modify-save would drop entries. Module * scope is deliberate: two instances on the same file must share the - * chain. Cross-process writes remain last-writer-wins (documented). */ + * chain. Cross-process writes remain last-writer-wins (documented). + * Exported for other omni-owned JSON documents with the same + * load-modify-save discipline (media-memory store). */ const fileOps = new Map>(); -function serialize(key: string, fn: () => Promise): Promise { +export function serializeFileOperation( + key: string, + fn: () => Promise, +): Promise { const prev = fileOps.get(key) ?? Promise.resolve(); const run = prev.then(fn, fn); const settled = run.then( @@ -82,7 +87,7 @@ export class OmniJsonCacheFile { | { result: R; changed?: boolean } | Promise<{ result: R; changed?: boolean }>, ): Promise { - return serialize(this.filePath, async () => { + return serializeFileOperation(this.filePath, async () => { const data = await this.load(); if (!data) return unreadableResult; const { result, changed } = await fn(data.entries); diff --git a/packages/core/src/omni/media-guidance.test.ts b/packages/core/src/omni/media-guidance.test.ts index b655584fed8..a5e7e771a0f 100644 --- a/packages/core/src/omni/media-guidance.test.ts +++ b/packages/core/src/omni/media-guidance.test.ts @@ -11,6 +11,7 @@ import { ToolNames } from '../tools/tool-names.js'; import { buildOmniMediaGuidanceSection } from './media-guidance.js'; import { OMNI_DISCLOSURE_TEXT_PREFIX, + OMNI_RESOURCE_HANDLE_TEXT_PREFIX, OMNI_OMISSION_TEXT_PREFIX, OMNI_TRANSCRIPT_TEXT_PREFIX, } from './disclosure.js'; @@ -25,6 +26,7 @@ function stubConfig(overrides?: { omniEnabled?: boolean; cgc?: Record; policyTools?: Record; + recallMode?: 'active' | 'sideQuery'; }): Config { return { isOmniEnabled: vi.fn().mockReturnValue(overrides?.omniEnabled ?? true), @@ -33,6 +35,13 @@ function stubConfig(overrides?: { .fn() .mockReturnValue(overrides?.cgc ?? DASHSCOPE_CGC), getOmniPolicyToolsSettings: vi.fn().mockReturnValue(overrides?.policyTools), + getOmniMemoryConfig: vi + .fn() + .mockReturnValue( + overrides?.recallMode + ? { recall: { mode: overrides.recallMode } } + : undefined, + ), } as unknown as Config; } @@ -107,3 +116,34 @@ describe('buildOmniMediaGuidanceSection', () => { expect(section).toContain('No media tools are enabled'); }); }); + +describe('buildOmniMediaGuidanceSection — recall guidance', () => { + it('explains the handle marker and the recall-before-reprocessing contract in active mode', () => { + const section = buildOmniMediaGuidanceSection( + stubConfig({ recallMode: 'active' }), + )!; + // The annotation ships with every memory-known delivery, but the tool + // that consumes it is deferred — so without this the model sees the + // marker with no explanation and reprocesses what memory already holds. + expect(section).toContain(OMNI_RESOURCE_HANDLE_TEXT_PREFIX); + expect(section).toContain('omni_recall_media_memory'); + expect(section).toMatch(/BEFORE reprocessing/); + // And it must say the handle is the only identity available. + expect(section).toMatch(/never be given its real path/); + }); + + it('says nothing about the recall tool in sideQuery mode', () => { + // D10: in sideQuery mode the harness injects recalled memory itself and + // the tool is not even registered — telling the model to call it would + // invite a guaranteed unknown-tool error. + const section = buildOmniMediaGuidanceSection( + stubConfig({ recallMode: 'sideQuery' }), + )!; + expect(section).not.toContain('omni_recall_media_memory'); + }); + + it('says nothing about recall when memory is not configured', () => { + const section = buildOmniMediaGuidanceSection(stubConfig())!; + expect(section).not.toContain('omni_recall_media_memory'); + }); +}); diff --git a/packages/core/src/omni/media-guidance.ts b/packages/core/src/omni/media-guidance.ts index e9ccccf5d58..a207761fbec 100644 --- a/packages/core/src/omni/media-guidance.ts +++ b/packages/core/src/omni/media-guidance.ts @@ -32,6 +32,7 @@ import { isOmniDeliveryActive } from './delivery-gate.js'; import { resolveMediaPolicyModelAccess } from './policy/model-access.js'; import { OMNI_DISCLOSURE_TEXT_PREFIX, + OMNI_RESOURCE_HANDLE_TEXT_PREFIX, OMNI_OMISSION_TEXT_PREFIX, OMNI_TRANSCRIPT_TEXT_PREFIX, } from './disclosure.js'; @@ -86,6 +87,22 @@ export function buildOmniMediaGuidanceSection(config: Config): string | null { ([name]) => resolveMediaPolicyModelAccess(config, name).enabled, ); + // Active-mode recall guidance. The handle annotation is emitted with + // every memory-known delivery, but the tool that consumes it is deferred + // (it surfaces via ToolSearch), so without this the model can receive + // 【媒体资源】 markers with no explanation anywhere in context — and the + // rest of this section actively tells it to gather evidence with the + // policy tools, i.e. to reprocess from scratch what memory already holds. + // Only stated when recall.mode is 'active': in sideQuery mode the harness + // injects recalled memory itself and the model must not call the tool. + const recallGuidance = + config.getOmniMemoryConfig?.()?.recall.mode === 'active' + ? ` +- ${OMNI_RESOURCE_HANDLE_TEXT_PREFIX}: an opaque session handle for that media. It is the ONLY identity you can use to reference the file — you will never be given its real path. +- BEFORE reprocessing media (extracting frames, transcribing, clipping), call \`${ToolNames.OMNI_RECALL_MEDIA_MEMORY}\` with that handle: earlier sessions may already have produced the transcript, keyframes or excerpt you need, and it returns instantly. It also reports honest gaps — which channels were never processed — and can suggest which tool closes them. +- Handles also work as the \`resourceId\` argument of the media tools below, in place of a path.` + : ''; + const toolGuidance = enabledTools.length > 0 ? `- When the task needs evidence beyond what was delivered — later time ranges, finer visual detail, more frames, a fuller transcript — do not stop at the delivered subset: fetch the evidence yourself with the media tools below, then read the produced file(s) to bring them into context. Work in targeted excerpts (a specific time range or region at a time) so each request stays within limits, and iterate until you have seen enough to complete the task. @@ -101,6 +118,8 @@ Media files in this session reach you through a preprocessing pipeline that must - ${OMNI_OMISSION_TEXT_PREFIX}: the media could not be delivered at all; the notice stands in its place. - ${OMNI_TRANSCRIPT_TEXT_PREFIX}: text derived from the media (e.g. a speech transcript), possibly delivered instead of the media itself. +${recallGuidance} + Interpret delivered media under this contract: - A degraded delivery is an OVERVIEW or entry point, not the complete content. The original file on disk is untouched and remains fully available for further processing. diff --git a/packages/core/src/omni/memory-recall.ts b/packages/core/src/omni/memory-recall.ts new file mode 100644 index 00000000000..288a7a87192 --- /dev/null +++ b/packages/core/src/omni/memory-recall.ts @@ -0,0 +1,163 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import path from 'node:path'; +import type { Config } from '../config/config.js'; +import { ToolNames } from '../tools/tool-names.js'; +import { + MediaMemoryService, + MediaMemoryRecallService, + type MediaMemoryNextPolicyAction, + type MediaMemoryRecallAdvisor, +} from '../services/media-memory/index.js'; +import { formatResourceHandleText } from './disclosure.js'; +import { resolveMediaPolicyModelAccess } from './policy/model-access.js'; +import { OmniObjectStore } from './storage.js'; +import type { MediaChannel } from '../services/media-memory/index.js'; +import type { OmniModality } from './recognition.js'; + +/** + * Session wiring for media-memory recall (M §9): both recall surfaces — + * the active `omni_recall_media_memory` tool and the passive sideQuery + * selector — build their service through this one module, so the advisor + * and root-dir derivation cannot drift between them. + */ + +/** First evidence-gathering step for one uncovered channel of a media + * type. Only DIRECT steps are suggested (a video's speech_text gap + * suggests extracting the audio track, not the audio-input transcribe + * tool — the model chains from there). */ +const GAP_STEP: ReadonlyArray<{ + mediaType: OmniModality; + channels: readonly MediaChannel[]; + toolName: string; + reason: string; +}> = [ + { + mediaType: 'video', + channels: ['visual'], + toolName: ToolNames.OMNI_EXTRACT_KEYFRAMES, + reason: 'no visual evidence collected yet: extract keyframes', + }, + { + mediaType: 'video', + // 'acoustic' ONLY. Extracting the track covers `acoustic`, which + // leaves `speech_text` open — so matching on speech_text too made the + // advisor re-suggest extraction in the very payload that returns the + // extracted audio. A wholly unprocessed video still matches (its gap + // contains acoustic) and the model chains to transcription from there. + channels: ['acoustic'], + toolName: ToolNames.OMNI_EXTRACT_AUDIO, + reason: 'no audio-track evidence collected yet: extract the audio track', + }, + { + mediaType: 'audio', + channels: ['speech_text'], + toolName: ToolNames.OMNI_TRANSCRIBE_AUDIO, + reason: 'no transcript collected yet: transcribe the audio', + }, +]; + +/** + * Advisor mapping recall gaps to `nextPolicyActions` (M §9.4): only tools + * that are BOTH registered in this session and opened to the model via + * `modelAccess.enabled` are ever suggested — recall must not steer the + * model into calls the media-policy gate would reject. + */ +export function buildMediaMemoryRecallAdvisor( + config: Config, +): MediaMemoryRecallAdvisor { + const callable = (toolName: string): boolean => + config.getToolRegistry().getTool(toolName) !== undefined && + resolveMediaPolicyModelAccess(config, toolName).enabled; + + return ({ resourceId, mediaType, gap }) => { + // Nothing can be gathered from a file that is gone. + if (gap.reason === 'artifact_unavailable') return []; + // `partial_coverage` means sampled evidence already exists, and by + // design it stays sampled: keyframes deliberately never report + // complete visual coverage. Suggesting the step that produced it would + // advise work that CANNOT close the gap, forever. + if (gap.reason === 'partial_coverage') return []; + const actions: MediaMemoryNextPolicyAction[] = []; + for (const step of GAP_STEP) { + if (step.mediaType !== mediaType) continue; + if (!gap.channels.some((c) => step.channels.includes(c))) continue; + if (!callable(step.toolName)) continue; + actions.push({ + toolName: step.toolName, + resourceId, + arguments: {}, + reason: step.reason, + }); + } + return actions; + }; +} + +/** + * Build the session recall service, or undefined when media memory is not + * configured on this Config (omni off, initialize skipped, stub configs). + */ +export function createMediaMemoryRecallService( + config: Config, +): MediaMemoryRecallService | undefined { + const memoryConfig = config.getOmniMemoryConfig?.(); + if (!memoryConfig) return undefined; + const omniRootDir = new OmniObjectStore( + config.storage.getQwenDir(), + ).getOmniRootDir(); + return new MediaMemoryRecallService( + omniRootDir, + memoryConfig.recall, + config.getOmniMediaResourceRegistry(), + { advise: buildMediaMemoryRecallAdvisor(config) }, + ); +} + +/** + * Re-anchor a remembered file into this session (design M §9.2.1). + * + * A handle is only minted at delivery, and delivery needs the bytes — so a + * file memory knows about but that is gone from disk had no way back into + * a session, and its transcripts and keyframes were stranded. The same + * friction hits an audit that does not want to re-deliver a 2.4 GB film + * just to ask what work was recorded: without a handle, recall rejects + * everything, and the natural next move (passing the filename as a + * resourceId) is correctly refused. + * + * The user's own `@`-reference is the authorization here — the same + * authorization a normal delivery carries — so this mints a handle from + * the RECORDED identity without needing the bytes. Returns undefined when + * memory is off or has never seen this locator. + */ +export function reanchorRememberedMedia( + config: Config, + absolutePath: string, +): Promise<{ resourceId: string; annotation: string } | undefined> { + const memoryConfig = config.getOmniMemoryConfig?.(); + if (!memoryConfig) return Promise.resolve(undefined); + const registry = config.getOmniMediaResourceRegistry?.(); + if (!registry) return Promise.resolve(undefined); + const store = new OmniObjectStore(config.storage.getQwenDir()); + return new MediaMemoryService(store.getOmniRootDir()) + .findBindingByFileRef(absolutePath) + .then((found) => { + if (!found) return undefined; + const { resourceId } = registry.bind({ + ...found.binding, + fileRef: absolutePath, + mediaType: found.mediaType, + }); + return { + resourceId, + annotation: formatResourceHandleText( + path.basename(absolutePath), + resourceId, + ), + }; + }); +} diff --git a/packages/core/src/omni/memory-side-query.test.ts b/packages/core/src/omni/memory-side-query.test.ts new file mode 100644 index 00000000000..bf29ef727ad --- /dev/null +++ b/packages/core/src/omni/memory-side-query.test.ts @@ -0,0 +1,484 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import type { Content } from '@google/genai'; +import type { Config } from '../config/config.js'; +import { + DEFAULT_OMNI_MEMORY_CONFIG, + MediaMemoryService, + MediaResourceRegistry, +} from '../services/media-memory/index.js'; +import { formatResourceHandleText } from './disclosure.js'; +import { + extractRequestResourceIds, + formatOmniMemorySideQueryReminder, + runOmniMemorySideQuery, +} from './memory-side-query.js'; + +vi.mock('../utils/sideQuery.js', () => ({ runSideQuery: vi.fn() })); +import { runSideQuery } from '../utils/sideQuery.js'; + +const runSideQueryMock = vi.mocked(runSideQuery); + +/** The slice of the selector's `runSideQuery` options these tests inspect — + * everything the production caller is responsible for composing. */ +interface SelectorOptions { + contents: Content[]; + abortSignal: AbortSignal; + promptId?: string; + validate?: (response: { entryIds: unknown }) => string | null; +} + +/** What the harness puts in front of the selector model. */ +interface SelectorPayload { + request: string; + candidates: Array<{ entryId: string }>; +} + +function parseSelectorPayload(options: SelectorOptions): SelectorPayload { + return JSON.parse( + (options.contents[0]!.parts![0] as { text: string }).text, + ) as SelectorPayload; +} + +/** Stand in for `runSideQuery` the way production behaves: the parsed model + * response is run through the caller's `validate` closure and its message + * is THROWN (sideQuery.ts). Mocks that only read `options.contents` and + * return a selection never touch `validate`, so a broken membership/budget + * check would go unnoticed even though it rejects every real selection. */ +function mockSelector( + respond: (payload: SelectorPayload) => { entryIds: unknown[] }, +): void { + runSideQueryMock.mockImplementation((async ( + _config: unknown, + options: SelectorOptions, + ) => { + const response = respond(parseSelectorPayload(options)); + const rejection = options.validate?.(response); + if (rejection) throw new Error(rejection); + return response; + }) as never); +} + +describe('omni memory sideQuery selector', () => { + let tmpDir: string; + let registry: MediaResourceRegistry; + + beforeEach(async () => { + tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'omni-sidequery-')); + registry = new MediaResourceRegistry(); + runSideQueryMock.mockReset(); + }); + + afterEach(async () => { + await fs.rm(tmpDir, { recursive: true, force: true }); + }); + + type SideQuerySettings = + (typeof DEFAULT_OMNI_MEMORY_CONFIG)['recall']['sideQuery']; + + function memoryConfig( + mode: 'active' | 'sideQuery', + sideQuery?: Partial, + ) { + return { + ...DEFAULT_OMNI_MEMORY_CONFIG, + recall: { + ...DEFAULT_OMNI_MEMORY_CONFIG.recall, + mode, + sideQuery: { + ...DEFAULT_OMNI_MEMORY_CONFIG.recall.sideQuery, + ...sideQuery, + }, + }, + }; + } + + function sideQueryConfig( + mode: 'active' | 'sideQuery' = 'sideQuery', + sideQuery?: Partial, + ): Config { + return { + getOmniMemoryConfig: () => memoryConfig(mode, sideQuery), + getOmniMediaResourceRegistry: () => registry, + storage: { getQwenDir: () => tmpDir }, + getToolRegistry: () => ({ getTool: () => undefined }), + getOmniPolicyToolsSettings: () => undefined, + } as unknown as Config; + } + + /** Record one image into the store and bind its session handle. */ + async function recordAndBind(): Promise { + const memory = new MediaMemoryService(path.join(tmpDir, 'omni')); + const binding = await memory.recordFileRecognized({ + fileRef: path.join(tmpDir, 'pic.png'), + sha256: 'a'.repeat(64), + mediaType: 'image', + metadata: { width: 32, height: 32 }, + sizeBytes: 1234, + mimeType: 'image/png', + origin: 'user', + source: { protocol: 'local', locator: 'pic.png' }, + recognition: { + ingestionConfigHash: '', + detectorVersion: 'omni-sniff-ffprobe/1', + probeStatus: 'complete', + }, + }); + return registry.bind({ + ...binding!, + fileRef: path.join(tmpDir, 'pic.png'), + mediaType: 'image', + }).resourceId; + } + + describe('extractRequestResourceIds', () => { + it('collects issued handles from annotation lines, deduplicated', async () => { + const resourceId = await recordAndBind(); + const parts = [ + 'plain user text', + { text: formatResourceHandleText('pic.png', resourceId) }, + { + text: + 'context\n' + + formatResourceHandleText('pic.png', resourceId) + + '\nmore', + }, + ]; + expect(extractRequestResourceIds(sideQueryConfig(), parts)).toEqual([ + resourceId, + ]); + }); + + it('ignores handles this session never issued', () => { + const parts = [ + { text: formatResourceHandleText('ghost.png', 'media-7-abcdef01') }, + ]; + expect(extractRequestResourceIds(sideQueryConfig(), parts)).toEqual([]); + }); + }); + + describe('runOmniMemorySideQuery', () => { + it('is a no-op in active mode (D10 mutual exclusion)', async () => { + const resourceId = await recordAndBind(); + const outcome = await runOmniMemorySideQuery({ + config: sideQueryConfig('active'), + requestParts: [ + { text: formatResourceHandleText('pic.png', resourceId) }, + ], + }); + expect(outcome).toBeNull(); + expect(runSideQueryMock).not.toHaveBeenCalled(); + }); + + it('is a no-op when the request carries no handles', async () => { + const outcome = await runOmniMemorySideQuery({ + config: sideQueryConfig(), + requestParts: ['just text'], + }); + expect(outcome).toBeNull(); + expect(runSideQueryMock).not.toHaveBeenCalled(); + }); + + it('materializes the selector picks and formats the reminder', async () => { + const resourceId = await recordAndBind(); + // Capture, assert AFTER: production wraps the selector call in + // try/catch and turns any throw into `selector_failed` degradation, so + // a failed expect() INSIDE the mock would surface as a null result + // instead of naming what leaked. + let seenPayload: SelectorPayload | undefined; + mockSelector((payload) => { + seenPayload = payload; + return { entryIds: [payload.candidates[0]!.entryId] }; + }); + + const outcome = await runOmniMemorySideQuery({ + config: sideQueryConfig(), + requestParts: [ + 'what size is this image?', + { text: formatResourceHandleText('pic.png', resourceId) }, + ], + }); + + expect(seenPayload?.request).toContain('what size is this image'); + // Selector-visible manifest: summaries only, no paths. + expect(JSON.stringify(seenPayload?.candidates)).not.toContain(tmpDir); + expect(outcome?.result).not.toBeNull(); + expect(outcome!.result!.entries).toHaveLength(1); + expect(outcome!.result!.entries[0]!.kind).toBe('metadata'); + const reminder = formatOmniMemorySideQueryReminder(outcome!.result!); + expect(reminder).toContain('【媒体记忆】'); + expect(reminder).toContain(outcome!.result!.entries[0]!.entryId); + expect(reminder).not.toContain(tmpDir); + }); + + it('shows the selector the question even when IDE context is merged in', async () => { + const resourceId = await recordAndBind(); + let seenRequest = ''; + runSideQueryMock.mockImplementation((async ( + _config: unknown, + options: { contents: Content[] }, + ) => { + const payload = JSON.parse( + (options.contents[0]!.parts![0] as { text: string }).text, + ); + seenRequest = payload.request; + return { entryIds: [] }; + }) as never); + + // Exactly how client.ts builds the parts in IDE mode: wrapIdeContext + // output is PREPENDED INTO the user's own text part, before the + // passive-recall pass runs — so the question lives in a part that + // STARTS with . Dropping such parts wholesale would + // make the selector pick relevance-blind. + await runOmniMemorySideQuery({ + config: sideQueryConfig(), + requestParts: [ + { + text: + '\nActive file: /x/y.ts\n' + + 'what size is this image?', + }, + { text: formatResourceHandleText('pic.png', resourceId) }, + ], + }); + + expect(seenRequest).toContain('what size is this image?'); + // The reminder itself is stripped, not forwarded. + expect(seenRequest).not.toContain('system-reminder'); + expect(seenRequest).not.toContain('/x/y.ts'); + }); + + it('degrades to an empty recall with a reason when the selector fails', async () => { + const resourceId = await recordAndBind(); + runSideQueryMock.mockRejectedValue(new Error('boom')); + + const outcome = await runOmniMemorySideQuery({ + config: sideQueryConfig(), + requestParts: [ + { text: formatResourceHandleText('pic.png', resourceId) }, + ], + }); + + expect(outcome).toMatchObject({ + result: null, + reason: expect.stringContaining('selector_failed'), + }); + }); + + it('rejects a selection the candidate manifest does not authorize', async () => { + // The `validate` closure is the only thing standing between the + // selector and an arbitrary entryId: a wrong verdict either lets a + // forged/cross-root id through to materialization, or (inverted) + // rejects every legitimate selection — and with the default + // `maxAttempts: 1` that turns every passive recall into + // `selector_failed`, silently killing the whole feature. + const resourceId = await recordAndBind(); + let validate: SelectorOptions['validate']; + let manifestEntryId = ''; + runSideQueryMock.mockImplementation((async ( + _config: unknown, + options: SelectorOptions, + ) => { + validate = options.validate; + manifestEntryId = parseSelectorPayload(options).candidates[0]!.entryId; + return { entryIds: [] }; + }) as never); + + await runOmniMemorySideQuery({ + config: sideQueryConfig('sideQuery', { maxSelectedEntries: 1 }), + requestParts: [ + { text: formatResourceHandleText('pic.png', resourceId) }, + ], + }); + + expect(validate).toBeDefined(); + expect(validate!({ entryIds: [manifestEntryId] })).toBeNull(); + expect(validate!({ entryIds: ['media-entry-forged'] })).toContain( + 'not in the candidate manifest', + ); + // Two copies of a legal id are still two picks: the budget cap is + // judged before membership, so the manifest needs only one entry. + expect( + validate!({ entryIds: [manifestEntryId, manifestEntryId] }), + ).toContain('at most 1 entryIds'); + expect(validate!({ entryIds: 'everything' })).toContain( + 'must be an array', + ); + }); + + it('degrades when the selector names an entryId outside the manifest', async () => { + const resourceId = await recordAndBind(); + mockSelector(() => ({ entryIds: ['media-entry-forged'] })); + + const outcome = await runOmniMemorySideQuery({ + config: sideQueryConfig(), + requestParts: [ + { text: formatResourceHandleText('pic.png', resourceId) }, + ], + }); + + // Whole-selection rejection, surfaced through the same degradation + // path as a generation failure — never a partial materialization. + expect(outcome).toMatchObject({ + result: null, + reason: expect.stringContaining('selector_failed'), + }); + expect(outcome!.reason).toContain('not in the candidate manifest'); + }); + + it('caps the request text handed to the selector', async () => { + // The selector is a bounded pre-flight call on the critical path of + // every request carrying media: an unbounded request text (a pasted + // log, a huge diff) would put the main request's latency and cost at + // the mercy of whatever the user happened to paste. + const resourceId = await recordAndBind(); + const longQuestion = 'q'.repeat(5000); + let seenRequest = ''; + runSideQueryMock.mockImplementation((async ( + _config: unknown, + options: SelectorOptions, + ) => { + seenRequest = parseSelectorPayload(options).request; + return { entryIds: [] }; + }) as never); + + await runOmniMemorySideQuery({ + config: sideQueryConfig(), + requestParts: [ + longQuestion, + { text: formatResourceHandleText('pic.png', resourceId) }, + ], + }); + + expect(seenRequest).toHaveLength(4000); + expect(seenRequest).toBe(longQuestion.slice(0, 4000)); + }); + + it('cancels the selector when the caller aborts', async () => { + // A Ctrl-C landing inside the selector window must reach the selector + // call: composed out of the request signal, the interrupted main + // request would sit through the whole sideQuery.timeoutMs waiting for + // a selection nobody will use. + const resourceId = await recordAndBind(); + const controller = new AbortController(); + let seenSignal: AbortSignal | undefined; + runSideQueryMock.mockImplementation((async ( + _config: unknown, + options: SelectorOptions, + ) => { + seenSignal = options.abortSignal; + controller.abort(); + throw new Error('aborted'); + }) as never); + + const outcome = await runOmniMemorySideQuery({ + config: sideQueryConfig(), + requestParts: [ + { text: formatResourceHandleText('pic.png', resourceId) }, + ], + signal: controller.signal, + }); + + expect(seenSignal?.aborted).toBe(true); + // A user abort is not the bounded window elapsing. + expect(outcome?.reason).toContain('selector_failed'); + }); + + it('reports selector_timeout when the bounded window elapses', async () => { + const resourceId = await recordAndBind(); + runSideQueryMock.mockImplementation((async ( + _config: unknown, + options: SelectorOptions, + ) => { + await new Promise((_resolve, reject) => { + options.abortSignal.addEventListener('abort', () => + reject(new Error('The operation was aborted')), + ); + }); + }) as never); + + const outcome = await runOmniMemorySideQuery({ + config: sideQueryConfig('sideQuery', { timeoutMs: 5 }), + requestParts: [ + { text: formatResourceHandleText('pic.png', resourceId) }, + ], + }); + + expect(outcome).toMatchObject({ + result: null, + reason: 'selector_timeout', + }); + }); + + it('attributes the selector call to the originating prompt', async () => { + // Without the promptId the selector's model traffic is logged under a + // synthetic id, detaching this pre-flight call's cost and failures + // from the request that caused them. + const resourceId = await recordAndBind(); + let seenPromptId: string | undefined; + runSideQueryMock.mockImplementation((async ( + _config: unknown, + options: SelectorOptions, + ) => { + seenPromptId = options.promptId; + return { entryIds: [] }; + }) as never); + + await runOmniMemorySideQuery({ + config: sideQueryConfig(), + requestParts: [ + { text: formatResourceHandleText('pic.png', resourceId) }, + ], + promptId: 'session-abc########3', + }); + + expect(seenPromptId).toBe('session-abc########3'); + }); + + it('treats an empty selection as nothing to inject', async () => { + const resourceId = await recordAndBind(); + runSideQueryMock.mockResolvedValue({ entryIds: [] } as never); + + const outcome = await runOmniMemorySideQuery({ + config: sideQueryConfig(), + requestParts: [ + { text: formatResourceHandleText('pic.png', resourceId) }, + ], + }); + + expect(outcome).toMatchObject({ + result: null, + reason: 'selector_selected_nothing', + }); + }); + + it('skips the selector entirely when memory has no candidates', async () => { + // Bind a handle whose version was never persisted (empty store). + const resourceId = registry.bind({ + fileId: 'f1', + fileVersionId: 'v1', + rootFileId: 'f1', + fileRef: path.join(tmpDir, 'ghost.png'), + mediaType: 'image', + }).resourceId; + + const outcome = await runOmniMemorySideQuery({ + config: sideQueryConfig(), + requestParts: [ + { text: formatResourceHandleText('ghost.png', resourceId) }, + ], + }); + + expect(outcome).toMatchObject({ result: null, reason: 'no_candidates' }); + expect(runSideQueryMock).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/packages/core/src/omni/memory-side-query.ts b/packages/core/src/omni/memory-side-query.ts new file mode 100644 index 00000000000..828f7225072 --- /dev/null +++ b/packages/core/src/omni/memory-side-query.ts @@ -0,0 +1,281 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import type { Content, Part, PartUnion } from '@google/genai'; +import type { Config } from '../config/config.js'; +import { createDebugLogger } from '../utils/debugLogger.js'; +import { runSideQuery } from '../utils/sideQuery.js'; +import { + MediaMemoryRecallRejection, + type MediaMemoryCandidateSummary, + type MediaMemoryRecallResult, +} from '../services/media-memory/index.js'; +import { parseResourceHandleText } from './disclosure.js'; +import { createMediaMemoryRecallService } from './memory-recall.js'; + +const debugLogger = createDebugLogger('omni:memory'); + +/** + * Passive sideQuery recall (memory design M §9.3, D10 sideQuery mode): + * before the main model request is sent, a bounded selector model reads a + * candidate manifest of what memory knows about the media the CURRENT + * request explicitly references, picks the relevant entryIds, and the + * harness materializes exactly those through the unified recall protocol. + * + * Constitutional bounds, enforced here and in the recall service: + * - only resources the request explicitly carries (parsed from the + * 【媒体资源】 handle annotations) are consulted — never a project-wide + * scan; + * - the selector sees summaries (no raw media, full text, paths, or + * secrets) and may ONLY return manifest entryIds — an unknown id, + * cross-root id, or over-budget selection rejects the WHOLE selection; + * - any failure (timeout, selector error, rejection) degrades to an empty + * recall with a recorded reason; the main request always proceeds. + */ + +/** Outcome of one passive recall attempt (the selector ran or was + * legitimately skipped after handles were found). */ +export interface OmniMemorySideQueryOutcome { + /** Materialized recall to inject, or null when there is nothing. */ + result: MediaMemoryRecallResult | null; + /** Recorded reason when `result` is null (M §17: 超时/非法 → 空召回 + + * 记录原因). */ + reason?: string; + /** Session handles the selector consulted. */ + resourceIds: string[]; +} + +/** JSON face the selector must answer with: entryIds only (M §9.3 — no + * free text, no new conclusions). */ +const SELECTION_SCHEMA: Record = { + type: 'object', + properties: { + entryIds: { + type: 'array', + items: { type: 'string' }, + description: 'entryIds chosen from the candidate manifest.', + }, + }, + required: ['entryIds'], + additionalProperties: false, +}; + +/** Bound on the request text handed to the selector for relevance. */ +const MAX_SELECTOR_REQUEST_CHARS = 4000; + +/** Extract the session resource handles a request explicitly carries: + * every 【媒体资源】 annotation part whose handle this session's registry + * actually issued (M §9.3 — passive recall never guesses beyond what the + * request references). Deduplicated, in first-appearance order. */ +export function extractRequestResourceIds( + config: Config, + parts: readonly PartUnion[], +): string[] { + const registry = config.getOmniMediaResourceRegistry?.(); + if (!registry) return []; + const found: string[] = []; + const seen = new Set(); + for (const part of parts) { + const text = + typeof part === 'string' + ? part + : typeof part === 'object' && part !== null && 'text' in part + ? (part as { text?: string }).text + : undefined; + if (!text) continue; + // Annotations may be embedded per-line inside a larger flattened part. + for (const line of text.split('\n')) { + const resourceId = parseResourceHandleText(line.trim()); + if (!resourceId || seen.has(resourceId)) continue; + if (!registry.resolve(resourceId)) continue; + seen.add(resourceId); + found.push(resourceId); + } + } + return found; +} + +/** Strip every `…` block from a text + * part. Harness reminders are not the user's question, but they can be + * PREPENDED INTO the user's own text part (IDE context does exactly that, + * before the passive-recall pass runs) — dropping the whole part on a + * leading tag would hide the question from the selector entirely. */ +function stripSystemReminders(text: string): string { + return text.replace(/[\s\S]*?<\/system-reminder>/g, ''); +} + +/** Plain request text (system reminders removed, bounded) — the selector's + * only view of what the user is asking. */ +function selectorRequestText(parts: readonly PartUnion[]): string { + const texts: string[] = []; + for (const part of parts) { + const text = + typeof part === 'string' + ? part + : typeof part === 'object' && part !== null && 'text' in part + ? (part as { text?: string }).text + : undefined; + if (!text) continue; + const stripped = stripSystemReminders(text).trim(); + if (stripped) texts.push(stripped); + } + let joined = texts.join('\n'); + if (joined.length > MAX_SELECTOR_REQUEST_CHARS) { + joined = joined.slice(0, MAX_SELECTOR_REQUEST_CHARS); + } + return joined; +} + +/** + * Run the passive selector for one outgoing request. Returns null when + * the passive path does not apply at all (memory off, mode !== sideQuery, + * or the request references no session media); otherwise an outcome whose + * `result` is the recall to inject (or null with a recorded reason). + * + * Latency is bounded by `sideQuery.timeoutMs` — the caller awaits this + * BEFORE sending the main request (M §9.3: 选中结果必须在主请求发送前注入). + */ +export async function runOmniMemorySideQuery(params: { + config: Config; + requestParts: readonly PartUnion[]; + promptId?: string; + signal?: AbortSignal; +}): Promise { + const { config } = params; + const memoryConfig = config.getOmniMemoryConfig?.(); + if (!memoryConfig || memoryConfig.recall.mode !== 'sideQuery') return null; + const resourceIds = extractRequestResourceIds(config, params.requestParts); + if (resourceIds.length === 0) return null; + const service = createMediaMemoryRecallService(config); + if (!service) return null; + const sideQuery = memoryConfig.recall.sideQuery; + + let manifest: MediaMemoryCandidateSummary[]; + try { + manifest = await service.candidateSummaries(resourceIds); + } catch (err) { + const reason = `manifest_failed: ${err instanceof Error ? err.message : err}`; + debugLogger.debug(`omni sideQuery recall degraded: ${reason}`); + return { result: null, reason, resourceIds }; + } + if (manifest.length === 0) { + return { result: null, reason: 'no_candidates', resourceIds }; + } + + const manifestIds = new Set(manifest.map((c) => c.entryId)); + const timeoutSignal = AbortSignal.timeout(sideQuery.timeoutMs); + const abortSignal = params.signal + ? AbortSignal.any([params.signal, timeoutSignal]) + : timeoutSignal; + + const contents: Content[] = [ + { + role: 'user', + parts: [ + { + text: JSON.stringify( + { + request: selectorRequestText(params.requestParts), + candidates: manifest, + }, + null, + 1, + ), + } as Part, + ], + }, + ]; + + let selection: { entryIds: string[] }; + try { + selection = await runSideQuery<{ entryIds: string[] }>(config, { + contents, + schema: SELECTION_SCHEMA, + abortSignal, + ...(sideQuery.model !== null ? { model: sideQuery.model } : {}), + systemInstruction: + 'You select which persisted media-memory entries are relevant to ' + + 'the current request. Input: {request, candidates}. Return ONLY ' + + `{"entryIds": [...]}: at most ${sideQuery.maxSelectedEntries} ids, ` + + 'each copied verbatim from the candidates. Return an empty array ' + + 'when nothing is relevant. Never invent ids, never add other keys.', + ...(params.promptId !== undefined ? { promptId: params.promptId } : {}), + purpose: 'omni-memory-sidequery-selector', + maxAttempts: sideQuery.maxAttempts, + skipOutputLanguagePreference: true, + validate: (response) => { + if (!Array.isArray(response.entryIds)) + return 'entryIds must be an array'; + if (response.entryIds.length > sideQuery.maxSelectedEntries) { + return `at most ${sideQuery.maxSelectedEntries} entryIds`; + } + const unknown = response.entryIds.find( + (id) => typeof id !== 'string' || !manifestIds.has(id), + ); + return unknown !== undefined + ? `entryId ${String(unknown)} is not in the candidate manifest` + : null; + }, + }); + } catch (err) { + // Timeout, generation failure, or a selection our `validate` refused. + // Note the asymmetry: `maxAttempts` governs the client's own retry loop + // (unparseable output, schema violations), while `validate` runs once + // afterwards — so a selection that parses but names an id outside the + // manifest throws on the first offense and is not retried. Either way: + // empty recall, main request proceeds (M §17). + const reason = timeoutSignal.aborted + ? 'selector_timeout' + : `selector_failed: ${err instanceof Error ? err.message : err}`; + debugLogger.debug(`omni sideQuery recall degraded: ${reason}`); + return { result: null, reason, resourceIds }; + } + + if (selection.entryIds.length === 0) { + return { result: null, reason: 'selector_selected_nothing', resourceIds }; + } + + try { + const result = await service.recallSelection( + resourceIds, + selection.entryIds, + ); + // Every pick can drop out in the availability pass (the artifacts were + // deleted between the walk and now). Injecting the empty shell would + // spend the reminder on a payload that says nothing. + if (result.entries.length === 0) { + return { result: null, reason: 'materialized_nothing', resourceIds }; + } + return { result, resourceIds }; + } catch (err) { + // Defense in depth: validate() should have caught any bad selection, + // but a whole-rejection here still degrades to an empty recall. + const reason = + err instanceof MediaMemoryRecallRejection + ? `selection_rejected: ${err.reason}` + : `materialize_failed: ${err instanceof Error ? err.message : err}`; + debugLogger.debug(`omni sideQuery recall degraded: ${reason}`); + return { result: null, reason, resourceIds }; + } +} + +/** Model-facing injection block for a materialized passive recall. Framed + * as a system reminder so provider converters treat it as harness text; + * the JSON body is the same unified protocol shape the active tool + * returns (M §9.4). */ +export function formatOmniMemorySideQueryReminder( + result: MediaMemoryRecallResult, +): string { + return ( + '\n' + + '【媒体记忆】Recalled media memory for the resources referenced in ' + + 'this request (passive mode). Entries below were persisted by ' + + 'earlier processing; resourceIds in them are session handles you may ' + + 'pass to omni media tools.\n' + + `${JSON.stringify(result, null, 1)}\n` + + '' + ); +} diff --git a/packages/core/src/omni/policy/degradation-cache.ts b/packages/core/src/omni/policy/degradation-cache.ts index d114ba47678..61c2df4f18a 100644 --- a/packages/core/src/omni/policy/degradation-cache.ts +++ b/packages/core/src/omni/policy/degradation-cache.ts @@ -41,8 +41,15 @@ export const MAX_CACHED_DISCLOSURE_LENGTH = 2048; /** Io params are per-invocation plumbing, never policy identity: the same * policy applied to the same object must hit regardless of where the - * source file sat or which staging dir the run used. */ -const FINGERPRINT_EXCLUDED_KEYS = new Set(['inputPath', 'outputDir']); + * source file sat or which staging dir the run used. `resourceId` joins + * them for gated model calls — the session handle names the same bytes the + * hash already keys, and it is freshly minted every session, so leaving it + * in would make every session re-derive identical work. */ +const FINGERPRINT_EXCLUDED_KEYS = new Set([ + 'inputPath', + 'outputDir', + 'resourceId', +]); /** Deterministic JSON: objects serialized with sorted keys at every * depth, so `{a,b}` and `{b,a}` fingerprint identically. */ diff --git a/packages/core/src/omni/policy/model-access.test.ts b/packages/core/src/omni/policy/model-access.test.ts index 1ec778878d9..c842e3729dd 100644 --- a/packages/core/src/omni/policy/model-access.test.ts +++ b/packages/core/src/omni/policy/model-access.test.ts @@ -608,4 +608,166 @@ describe('evaluateMediaPolicyToolCall', () => { }); expect(fixed).toEqual({ outcome: 'pass', args }); }); + + describe('resourceId input resolution (M §5.2)', () => { + const registryWith = (bindings: Record) => ({ + resolve: (id: string) => + bindings[id] !== undefined + ? { + resourceId: id, + fileId: 'f1', + fileVersionId: 'v1', + rootFileId: 'f1', + fileRef: bindings[id], + mediaType: 'image' as const, + } + : undefined, + }); + const enabledConfig = (bindings: Record) => ({ + ...configWith({ + omni_compress_image: { modelAccess: { enabled: true } }, + }), + getOmniMediaResourceRegistry: () => registryWith(bindings) as never, + }); + + it('resolves a session handle to inputPath and drops resourceId', () => { + const result = evaluateMediaPolicyToolCall({ + config: enabledConfig({ 'media-1-ab': '/media/movie.mkv' }), + tool: policyTool(), + args: { resourceId: 'media-1-ab', outputDir: '/out' }, + executionOrigin: { kind: 'model' }, + }); + expect(result).toEqual({ + outcome: 'pass', + args: { inputPath: '/media/movie.mkv', outputDir: '/out' }, + }); + }); + + it('rejects a handle this session never issued', () => { + const result = evaluateMediaPolicyToolCall({ + config: enabledConfig({}), + tool: policyTool(), + args: { resourceId: 'media-9-zz', outputDir: '/out' }, + executionOrigin: { kind: 'model' }, + }); + expect(result).toMatchObject({ + outcome: 'reject', + reason: 'invalid_params', + }); + expect((result as { message: string }).message).toContain( + 'not issued in this session', + ); + }); + + it('rejects a call on a config with no session registry', () => { + const result = evaluateMediaPolicyToolCall({ + config: configWith({ + omni_compress_image: { modelAccess: { enabled: true } }, + }), + tool: policyTool(), + args: { resourceId: 'media-1-ab', outputDir: '/out' }, + executionOrigin: { kind: 'model' }, + }); + expect(result).toMatchObject({ + outcome: 'reject', + reason: 'invalid_params', + }); + }); + + it('rejects naming both inputPath and resourceId', () => { + const result = evaluateMediaPolicyToolCall({ + config: enabledConfig({ 'media-1-ab': '/media/movie.mkv' }), + tool: policyTool(), + args: { + resourceId: 'media-1-ab', + inputPath: '/elsewhere.png', + outputDir: '/out', + }, + executionOrigin: { kind: 'model' }, + }); + expect(result).toMatchObject({ + outcome: 'reject', + reason: 'invalid_params', + }); + expect((result as { message: string }).message).toContain('exactly one'); + }); + + it('cannot sidestep an operator-pinned inputPath via a handle', () => { + const result = evaluateMediaPolicyToolCall({ + config: { + ...configWith({ + omni_compress_image: { + modelAccess: { + enabled: true, + lockedArguments: { inputPath: '/pinned.png' }, + }, + }, + }), + getOmniMediaResourceRegistry: () => + registryWith({ 'media-1-ab': '/media/movie.mkv' }) as never, + }, + tool: policyTool(), + args: { resourceId: 'media-1-ab', outputDir: '/out' }, + executionOrigin: { kind: 'model' }, + }); + // The resolved inputPath collides with the locked key — rejected + // exactly like naming the locked key directly. + expect(result).toMatchObject({ + outcome: 'reject', + reason: 'invalid_params', + }); + }); + + it('rejects a handle whose modality the tool does not accept', () => { + // The fixed-policy path validates modality at startup; a gated caller + // holds only opaque handles, so mixing two up must fail as a + // correctable parameter error rather than as a spawned ffmpeg that + // burns the tool timeout and returns an opaque stderr tail. + const result = evaluateMediaPolicyToolCall({ + config: { + ...configWith({ + omni_compress_image: { modelAccess: { enabled: true } }, + }), + getOmniMediaResourceRegistry: () => + ({ + resolve: () => ({ + resourceId: 'media-1-ab', + fileId: 'f1', + fileVersionId: 'v1', + rootFileId: 'f1', + fileRef: '/media/movie.mkv', + mediaType: 'video' as const, + }), + }) as never, + }, + tool: policyTool(), + args: { resourceId: 'media-1-ab', outputDir: '/out' }, + executionOrigin: { kind: 'model' }, + }); + expect(result).toMatchObject({ + outcome: 'reject', + reason: 'invalid_params', + }); + expect((result as { message: string }).message).toContain( + 'names video media', + ); + }); + + it('never resolves handles for fixed_policy calls (args pass untouched)', () => { + const args = { resourceId: 'media-1-ab', outputDir: '/out' }; + const result = evaluateMediaPolicyToolCall({ + config: enabledConfig({ 'media-1-ab': '/media/movie.mkv' }), + tool: policyTool(), + args, + executionOrigin: { + kind: 'fixed_policy', + policyId: 'p1', + stage: 'preprocessing', + }, + }); + // RESERVED_ARGUMENT_KEYS already bans resourceId in fixed-policy + // arguments; the gate's job is only to leave fixed calls alone. + expect(result).toEqual({ outcome: 'pass', args }); + }); + }); }); diff --git a/packages/core/src/omni/policy/model-access.ts b/packages/core/src/omni/policy/model-access.ts index 8bb6d7c5ef7..9e2a4c2c320 100644 --- a/packages/core/src/omni/policy/model-access.ts +++ b/packages/core/src/omni/policy/model-access.ts @@ -7,6 +7,7 @@ import type { FunctionDeclaration } from '@google/genai'; import type { ToolExecutionOrigin } from '../../core/turn.js'; import type { MediaPolicyToolDescriptor } from '../../tools/tools.js'; +import type { OmniMediaRegistryView } from '../../services/media-memory/registry.js'; import { isPlainRecord } from './types.js'; import type { MediaPolicyToolConfigView, @@ -195,19 +196,23 @@ export type MediaPolicyCallGateResult = * orchestrator already resolved its own `arguments`; modelAccess does * not apply to fixed calls); * - a model/client-origin call of a media-policy tool requires - * `modelAccess.enabled`, must not name any lockedArguments key - * explicitly, and gets defaults + lockedArguments merged in; + * `modelAccess.enabled`, may reference its input by opaque session + * `resourceId` INSTEAD of `inputPath` (resolved here to the real + * locator — memory design M §5.2; the model never learns the path), + * must not name any lockedArguments key explicitly, and gets + * defaults + lockedArguments merged in; * - everything else passes untouched. * * A missing origin fails closed as `{ kind: 'model' }`. */ export function evaluateMediaPolicyToolCall(params: { - config: MediaPolicyConfigView; + config: MediaPolicyConfigView & OmniMediaRegistryView; tool: { name: string; mediaPolicyDescriptor?: MediaPolicyToolDescriptor }; args: Record; executionOrigin: ToolExecutionOrigin | undefined; }): MediaPolicyCallGateResult { - const { config, tool, args } = params; + const { config, tool } = params; + let args = params.args; const origin = params.executionOrigin ?? { kind: 'model' }; if (origin.kind === 'fixed_policy') { @@ -239,6 +244,57 @@ export function evaluateMediaPolicyToolCall(params: { }; } + // Session-handle input (M §5.2): a gated caller may name its source by + // the opaque resourceId minted at delivery/recall instead of a real + // path. Resolution happens BEFORE the lockedArguments check so a + // handle-resolved inputPath cannot sidestep an operator-pinned input. + if (typeof args['resourceId'] === 'string') { + if (args['inputPath'] !== undefined) { + return { + outcome: 'reject', + reason: 'invalid_params', + message: + `Invalid parameters for tool "${tool.name}": provide exactly ` + + `one of "inputPath" or "resourceId", not both.`, + }; + } + const binding = config + .getOmniMediaResourceRegistry?.() + ?.resolve(args['resourceId']); + if (!binding) { + // Unknown = never issued in this session (fabricated or stale + // cross-session handle) — reject, never guess (M §9.2 stance). + return { + outcome: 'reject', + reason: 'invalid_params', + message: + `Invalid parameters for tool "${tool.name}": resourceId ` + + `"${args['resourceId']}" was not issued in this session. Use a ` + + `handle from a 【媒体资源】 annotation or a recall result.`, + }; + } + // The handle's modality must be one this tool declares consuming. The + // fixed-policy path validates this at startup; a gated caller holds + // only opaque handles, so mixing two up is easy — and without this the + // mistake becomes a spawned ffmpeg that burns the tool timeout and + // returns an opaque stderr tail, instead of an instant parameter error + // the caller can correct. + const accepted = tool.mediaPolicyDescriptor.inputMediaTypes; + if (accepted !== undefined && !accepted.includes(binding.mediaType)) { + return { + outcome: 'reject', + reason: 'invalid_params', + message: + `Invalid parameters for tool "${tool.name}": resourceId ` + + `"${args['resourceId']}" names ${binding.mediaType} media, but ` + + `this tool accepts ${accepted.join(', ')}. Pass a handle whose ` + + `media type matches.`, + }; + } + const { resourceId: _resourceId, ...rest } = args; + args = { ...rest, inputPath: binding.fileRef }; + } + const lockedKeys = Object.keys(access.lockedArguments); const violations = lockedKeys.filter((key) => Object.prototype.hasOwnProperty.call(args, key), diff --git a/packages/core/src/omni/policy/model-call-collection.test.ts b/packages/core/src/omni/policy/model-call-collection.test.ts new file mode 100644 index 00000000000..0d719b6c37f --- /dev/null +++ b/packages/core/src/omni/policy/model-call-collection.test.ts @@ -0,0 +1,218 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import type { Config } from '../../config/config.js'; +import type { PolicyArtifactBatch } from '../../core/turn.js'; +import type { MediaPolicyToolDescriptor } from '../../tools/tools.js'; +import { + DEFAULT_OMNI_MEMORY_CONFIG, + MediaResourceRegistry, + type MediaMemorySnapshot, +} from '../../services/media-memory/index.js'; +import { OmniObjectStore } from '../storage.js'; +import { collectModelPolicyCall } from './model-call-collection.js'; + +/** A 1x1 JPEG — recognizeMediaFile sniffs real bytes, so the artifact has + * to be a genuine image. */ +const JPEG_1X1 = Buffer.from( + '/9j/4AAQSkZJRgABAQEAYABgAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0a' + + 'HBwcJC4nHywjLBwcKDcpLDA1NTU1HyU5PTgyNTL/wAALCAABAAEBAREA/8QAFAABAQAAAAAAAAAA' + + 'AAAAAAAAAAP/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/8QAFAEBAAAAAAAAAAAAAAAAAAAAAP/EABQR' + + 'AQAAAAAAAAAAAAAAAAAAAAD/2gAMAwEAAhEDEQA/AKAA/9k=', + 'base64', +); + +const DESCRIPTOR: MediaPolicyToolDescriptor = { + kind: 'media_policy', + version: '3', + inputMediaTypes: ['image'], + outputs: [ + { kind: 'media', required: true, mimeTypes: ['image/jpeg'], lossy: true }, + ], +}; + +describe('collectModelPolicyCall', () => { + let tmpDir: string; + let outputDir: string; + let inputPath: string; + let registry: MediaResourceRegistry; + + beforeEach(async () => { + tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'omni-model-commit-')); + outputDir = path.join(tmpDir, 'out'); + await fs.mkdir(outputDir, { recursive: true }); + inputPath = path.join(tmpDir, 'source.jpg'); + await fs.writeFile(inputPath, JPEG_1X1); + await fs.writeFile(path.join(outputDir, 'derived.jpg'), JPEG_1X1); + registry = new MediaResourceRegistry(); + }); + + afterEach(async () => { + await fs.rm(tmpDir, { recursive: true, force: true }); + }); + + function bridgeConfig(overrides?: Record): Config { + return { + storage: { getQwenDir: () => tmpDir }, + getOmniMemoryConfig: () => DEFAULT_OMNI_MEMORY_CONFIG, + getOmniMediaResourceRegistry: () => registry, + ...overrides, + } as unknown as Config; + } + + function batchOf( + origin: PolicyArtifactBatch['executionOrigin'] = { kind: 'model' }, + ): PolicyArtifactBatch { + return { + toolName: 'omni_downsample_image', + invocationId: 'call-abc', + executionOrigin: origin, + artifacts: [ + { + kind: 'image', + storage: 'workspace', + title: 'derived', + workspacePath: 'derived.jpg', + mimeType: 'image/jpeg', + sizeBytes: JPEG_1X1.length, + metadata: { + omniDisclosure: 'downsampled to 1x1', + omniRole: 'degraded', + }, + }, + ], + }; + } + + async function readSnapshot(): Promise { + const file = path.join( + new OmniObjectStore(tmpDir).getOmniRootDir(), + 'memory.json', + ); + try { + return JSON.parse(await fs.readFile(file, 'utf8')) as MediaMemorySnapshot; + } catch { + return { + schemaVersion: 1, + files: {}, + versions: {}, + executions: {}, + entries: {}, + }; + } + } + + it('commits a model-origin success through the same boundary', async () => { + await collectModelPolicyCall({ + config: bridgeConfig(), + batch: batchOf(), + descriptor: DESCRIPTOR, + args: { inputPath, outputDir, quality: 70 }, + }); + + const snapshot = await readSnapshot(); + const executions = Object.values(snapshot.executions); + expect(executions).toHaveLength(1); + expect(executions[0]).toMatchObject({ + toolName: 'omni_downsample_image', + invocationId: 'call-abc', + // The REAL origin is preserved — not rewritten as fixed_policy. + executionOrigin: { kind: 'model' }, + toolVersion: '3', + // Per-invocation plumbing is excluded from reproducible config. + finalArguments: { quality: 70 }, + }); + + // One entry per declared output, and the derivative was promoted into + // the object store BEFORE the record referencing it was written. + const entries = Object.values(snapshot.entries); + expect(entries).toHaveLength(1); + expect(entries[0]).toMatchObject({ + kind: 'derived_media', + role: 'degraded', + disclosure: 'downsampled to 1x1', + }); + const derivedVersion = snapshot.versions[entries[0]!.derivedVersionId!]!; + const objectRef = snapshot.files[derivedVersion.fileId]!.fileRef; + expect(objectRef.startsWith(path.join(tmpDir, 'omni'))).toBe(true); + await expect(fs.stat(objectRef)).resolves.toBeDefined(); + }); + + it('reuses the session binding for a handle-driven call', async () => { + // The gate turned the model's resourceId into inputPath; the registry + // still holds the identity, so the source is not re-recognized. + const bound = registry.bind({ + fileId: 'f-src', + fileVersionId: 'v-src', + rootFileId: 'f-root', + fileRef: inputPath, + mediaType: 'image', + }); + + await collectModelPolicyCall({ + config: bridgeConfig(), + batch: batchOf(), + descriptor: DESCRIPTOR, + args: { inputPath, outputDir, resourceId: bound.resourceId }, + }); + + const snapshot = await readSnapshot(); + const execution = Object.values(snapshot.executions)[0]!; + expect(execution.sourceVersionId).toBe('v-src'); + expect(execution.rootFileId).toBe('f-root'); + // resourceId is per-invocation plumbing, never reproducible config. + expect(execution.finalArguments).not.toHaveProperty('resourceId'); + }); + + it('leaves fixed-policy successes to the orchestrator', async () => { + await collectModelPolicyCall({ + config: bridgeConfig(), + batch: batchOf({ + kind: 'fixed_policy', + policyId: 'img', + stage: 'preprocessing', + }), + descriptor: DESCRIPTOR, + args: { inputPath, outputDir }, + }); + + const snapshot = await readSnapshot(); + expect(Object.keys(snapshot.executions)).toHaveLength(0); + }); + + it('does nothing when memory is not configured', async () => { + await collectModelPolicyCall({ + config: bridgeConfig({ getOmniMemoryConfig: () => undefined }), + batch: batchOf(), + descriptor: DESCRIPTOR, + args: { inputPath, outputDir }, + }); + + const snapshot = await readSnapshot(); + expect(Object.keys(snapshot.executions)).toHaveLength(0); + }); + + it('never throws when an artifact escapes the declared output directory', async () => { + const escaping = batchOf(); + escaping.artifacts[0]!.workspacePath = '../source.jpg'; + + await expect( + collectModelPolicyCall({ + config: bridgeConfig(), + batch: escaping, + descriptor: DESCRIPTOR, + args: { inputPath, outputDir }, + }), + ).resolves.toBeUndefined(); + + const snapshot = await readSnapshot(); + expect(Object.keys(snapshot.executions)).toHaveLength(0); + }); +}); diff --git a/packages/core/src/omni/policy/model-call-collection.ts b/packages/core/src/omni/policy/model-call-collection.ts new file mode 100644 index 00000000000..a473d42a8ca --- /dev/null +++ b/packages/core/src/omni/policy/model-call-collection.ts @@ -0,0 +1,226 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import path from 'node:path'; +import type { Config } from '../../config/config.js'; +import type { PolicyArtifactBatch } from '../../core/turn.js'; +import type { MediaPolicyToolDescriptor } from '../../tools/tools.js'; +import { createDebugLogger } from '../../utils/debugLogger.js'; +import { + MediaMemoryService, + MEDIA_DETECTOR_VERSION, + type MediaMemoryBinding, + type OmniMediaRegistryView, + type OmniMemoryConfigView, + type PolicyOutputInput, +} from '../../services/media-memory/index.js'; +import { + extensionForMime, + hashFileSha256, + recognizeMediaFile, +} from '../recognition.js'; +import { OmniObjectStore } from '../storage.js'; +import { computePolicyFingerprint } from './degradation-cache.js'; +import { + assertRequiredOutputsPresent, + validateArtifact, +} from './orchestrator.js'; + +const debugLogger = createDebugLogger('omni:memory'); + +/** + * Model/client-origin half of the `OmniPolicySucceeded` boundary (memory + * design M §7.1: «固定调用、模型 ToolCall 和 direct client 调用在成功后 + * 都进入同一个 OmniPolicySucceeded 逻辑边界»). + * + * The fixed-policy orchestrator commits its own successes inline, where it + * already holds the staging dir, the promoted objects and the source + * binding. A gated model call has no orchestrator around it: the scheduler + * (and ACP's own executor) capture the raw `PolicyArtifactBatch` and hand + * it here, which walks the same gates in the same order — validate against + * the descriptor, require declared outputs, promote to the object store, + * THEN commit (M §6.4 / S §5: no record may reference bytes that are not + * yet in `objects/`). + * + * Without this, evidence gathering never accumulated: recall reported a + * gap, the advisor suggested a tool, the model called it, the tool + * succeeded — and the next session's recall reported the identical gap + * because nothing was ever written. + * + * Collection failure never affects the tool call (D12): everything is + * caught and logged. + */ +export async function collectModelPolicyCall(params: { + config: Config; + batch: PolicyArtifactBatch; + descriptor: MediaPolicyToolDescriptor; + /** Resolved arguments the tool actually ran with (post-gate: a + * `resourceId` the caller passed has already become `inputPath`). */ + args: Record; + /** Epoch ms when the tool actually began executing. The caller owns this + * — measuring it here would time the COLLECTION, not the work. */ + startedAt?: number; + signal?: AbortSignal; +}): Promise { + const { config, batch, descriptor, args, signal } = params; + // Fixed-policy successes are committed by the orchestrator itself; a + // second commit here would be a duplicate write of the same execution. + if (batch.executionOrigin.kind === 'fixed_policy') return; + const memoryConfig = (config as OmniMemoryConfigView).getOmniMemoryConfig?.(); + if (!memoryConfig) return; + + const inputPath = args['inputPath']; + const outputDir = args['outputDir']; + if (typeof inputPath !== 'string' || typeof outputDir !== 'string') return; + + try { + const store = new OmniObjectStore(config.storage.getQwenDir()); + const memory = new MediaMemoryService(store.getOmniRootDir(), { + maxInlineTextBytes: memoryConfig.collection.maxInlineTextBytes, + }); + + const source = await resolveSourceBinding({ + config, + memory, + inputPath, + signal, + }); + if (!source) return; + + // Fall back to "now" only when the caller could not supply the real + // start; that degrades the recorded duration, never correctness. + const startedAt = new Date(params.startedAt ?? Date.now()).toISOString(); + // Same containment rule the orchestrator applies to its staging dir: + // an artifact must live inside the output directory the call declared. + const validated = await Promise.all( + batch.artifacts.map((artifact) => + validateArtifact(artifact, descriptor, path.resolve(outputDir), signal), + ), + ); + assertRequiredOutputsPresent(descriptor, validated, batch.toolName); + + const outputs: PolicyOutputInput[] = await Promise.all( + validated.map(async (artifact) => { + const mimeType = + artifact.kind === 'media' + ? artifact.recognized.detectedMimeType + : artifact.mimeType; + const put = await store.putFile( + artifact.absolutePath, + artifact.sha256, + extensionForMime(mimeType), + signal, + ); + return artifact.kind === 'media' + ? { + kind: 'media' as const, + objectPath: put.objectPath, + sha256: artifact.sha256, + mediaType: artifact.recognized.modality, + metadata: artifact.recognized.metadata, + sizeBytes: artifact.recognized.sizeBytes, + mimeType: artifact.recognized.detectedMimeType, + role: artifact.role, + disclosure: artifact.disclosure, + } + : { + kind: 'text' as const, + objectPath: put.objectPath, + sha256: artifact.sha256, + mimeType: artifact.mimeType, + text: artifact.text, + sizeBytes: artifact.sizeBytes, + role: artifact.role, + disclosure: artifact.disclosure, + }; + }), + ); + + // Reserved runtime keys are per-invocation plumbing, not reproducible + // configuration — the same exclusion the fingerprint applies. + const finalArguments: Record = { ...args }; + delete finalArguments['inputPath']; + delete finalArguments['outputDir']; + delete finalArguments['resourceId']; + + const commit = await memory.commitPolicySucceeded({ + invocationId: batch.invocationId, + source, + executionOrigin: batch.executionOrigin, + toolName: batch.toolName, + ...(descriptor.version !== undefined + ? { toolVersion: descriptor.version } + : {}), + finalArguments, + omniConfigHash: computePolicyFingerprint( + batch.toolName, + args, + descriptor.version, + ), + startedAt, + completedAt: new Date().toISOString(), + outputs, + }); + // `created: false` is a content-identity replay: the same file already + // recorded this execution (a same-invocation retry, or a degradation + // cache hit). Worth distinguishing in the log, because "no new + // execution appeared in memory.json" otherwise reads identically to a + // collection that silently did nothing. + debugLogger.debug( + `omni memory: ${commit?.created === false ? 'replayed' : 'recorded'} ` + + `${batch.toolName} (${batch.executionOrigin.kind}) with ` + + `${outputs.length} output(s)`, + ); + } catch (err) { + if (signal?.aborted) return; + debugLogger.debug( + `omni memory: collecting ${batch.toolName} (${batch.executionOrigin.kind}) ` + + `failed, skipping: ${err instanceof Error ? err.message : err}`, + ); + } +} + +/** + * Memory identity of the source the call ran against. A handle-driven call + * already has one in the session registry (the gate resolved that handle + * into `inputPath`), so the file is not re-hashed; a call that named a raw + * path falls back to recognizing it, which is an idempotent upsert. + */ +async function resolveSourceBinding(params: { + config: Config; + memory: MediaMemoryService; + inputPath: string; + signal?: AbortSignal; +}): Promise { + const { config, memory, inputPath, signal } = params; + const bound = (config as OmniMediaRegistryView) + .getOmniMediaResourceRegistry?.() + ?.resolveByFileRef(inputPath); + if (bound) { + return { + fileId: bound.fileId, + fileVersionId: bound.fileVersionId, + rootFileId: bound.rootFileId, + }; + } + const recognized = await recognizeMediaFile(inputPath, { signal }); + const sha256 = await hashFileSha256(inputPath, signal); + return memory.recordFileRecognized({ + fileRef: inputPath, + sha256, + mediaType: recognized.modality, + metadata: recognized.metadata, + sizeBytes: recognized.sizeBytes, + mimeType: recognized.detectedMimeType, + origin: 'user', + source: { protocol: 'local', locator: path.basename(inputPath) }, + recognition: { + ingestionConfigHash: '', + detectorVersion: MEDIA_DETECTOR_VERSION, + probeStatus: 'complete', + }, + }); +} diff --git a/packages/core/src/omni/policy/orchestrator.test.ts b/packages/core/src/omni/policy/orchestrator.test.ts index 0c906e0a6b7..c6787bb6195 100644 --- a/packages/core/src/omni/policy/orchestrator.test.ts +++ b/packages/core/src/omni/policy/orchestrator.test.ts @@ -12,6 +12,9 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import type { Config } from '../../config/config.js'; import type { ToolCallRequestInfo } from '../../core/turn.js'; import type { MediaPolicyToolDescriptor } from '../../tools/tools.js'; +import type { MediaMemoryService } from '../../services/media-memory/index.js'; +import { MEDIA_MEMORY_FILE_NAME } from '../../services/media-memory/store.js'; +import type { MediaMemorySnapshot } from '../../services/media-memory/types.js'; import type { RecognizedMedia } from '../recognition.js'; import { OmniObjectStore } from '../storage.js'; import { @@ -1442,6 +1445,52 @@ describe('runFixedPolicies', () => { config = makeConfig({ omni_transcribe_stub: TRANSCRIPT_DESCRIPTOR }); }); + it('reuses a TEXT product on the second run without re-running the tool', async () => { + // Text products carry no derived version node, so their object path + // must be reconstructed from the content hash. A real-run probe caught + // this: audio/keyframe reuse hit while the transcript re-ran the entire + // ASR pass — the single most expensive thing #8189 exists to avoid. + const { MediaMemoryService } = await import( + '../../services/media-memory/index.js' + ); + const service = new MediaMemoryService(store.getOmniRootDir()); + const sha256 = createHash('sha256') + .update(await fs.readFile(sourcePath)) + .digest('hex'); + const sourceBinding = await service.recordFileRecognized({ + fileRef: sourcePath, + sha256, + mediaType: 'image', + metadata: recognizedImage().metadata, + sizeBytes: recognizedImage().sizeBytes, + mimeType: 'image/png', + origin: 'user', + source: { protocol: 'local', locator: 'photo.png' }, + recognition: { + ingestionConfigHash: '', + detectorVersion: 'omni-sniff-ffprobe/1', + probeStatus: 'complete', + }, + }); + const options = { + store, + policies: [transcriptPolicy()], + memory: { service, sourceBinding }, + }; + + mockFileArtifact(); + const first = await runFixedPolicies(config, source, options); + expect(first.fileDeliveries[0]?.text).toBe(TRANSCRIPT_TEXT); + expect(executeToolCallMock).toHaveBeenCalledTimes(1); + + const second = await runFixedPolicies(config, source, options); + expect(executeToolCallMock).toHaveBeenCalledTimes(1); + // Full text comes back from the promoted object, not from the + // entry's truncated inlineText copy. + expect(second.fileDeliveries[0]?.text).toBe(TRANSCRIPT_TEXT); + expect(second.fileDeliveries[0]?.disclosure).toBe(TRANSCRIPT_DISCLOSURE); + }); + it('validates and promotes the transcript into fileDeliveries with text + disclosure', async () => { mockFileArtifact(); const { deliveries, fileDeliveries, records } = await runFixedPolicies( @@ -1802,4 +1851,295 @@ describe('runFixedPolicies', () => { expect(records[0]).toMatchObject({ outcome: 'succeeded' }); }); }); + describe('memory reuse skips execution (#8189)', () => { + it('reuses recorded outputs on the second run without calling the tool', async () => { + const { MediaMemoryService } = await import( + '../../services/media-memory/index.js' + ); + const service = new MediaMemoryService(store.getOmniRootDir()); + const sha256 = createHash('sha256') + .update(await fs.readFile(sourcePath)) + .digest('hex'); + const sourceBinding = await service.recordFileRecognized({ + fileRef: sourcePath, + sha256, + mediaType: 'image', + metadata: recognizedImage().metadata, + sizeBytes: recognizedImage().sizeBytes, + mimeType: 'image/png', + origin: 'user', + source: { protocol: 'local', locator: 'source.png' }, + recognition: { + ingestionConfigHash: '', + detectorVersion: 'omni-sniff-ffprobe/1', + probeStatus: 'complete', + }, + }); + const options = { + store, + policies: [makePolicy({})], + memory: { service, sourceBinding }, + }; + + mockToolSuccess(); + const first = await runFixedPolicies(config, source, options); + expect(first.records[0]).toMatchObject({ outcome: 'succeeded' }); + expect(executeToolCallMock).toHaveBeenCalledTimes(1); + + // Second delivery of the same bytes under the same configuration: + // the tool must NOT run again, and the derivative must be the very + // same content-addressed object. + const second = await runFixedPolicies(config, source, options); + expect(executeToolCallMock).toHaveBeenCalledTimes(1); + expect(second.records[0]).toMatchObject({ outcome: 'succeeded' }); + expect(second.deliveries[0]!.sha256).toBe(first.deliveries[0]!.sha256); + expect(second.deliveries[0]!.filePath).toBe( + first.deliveries[0]!.filePath, + ); + expect(second.deliveries[0]!.disclosure).toBe( + first.deliveries[0]!.disclosure, + ); + }); + }); + + describe('memory collection commits (S5)', () => { + let service: MediaMemoryService; + + /** Record the source bytes as a user file, the way the delivery + * pipeline does before it hands the root to the orchestrator. */ + async function recordSource(service: MediaMemoryService) { + const sha256 = createHash('sha256') + .update(await fs.readFile(sourcePath)) + .digest('hex'); + const binding = await service.recordFileRecognized({ + fileRef: sourcePath, + sha256, + mediaType: 'image', + metadata: recognizedImage().metadata, + sizeBytes: recognizedImage().sizeBytes, + mimeType: 'image/png', + origin: 'user', + source: { protocol: 'local', locator: 'photo.png' }, + recognition: { + ingestionConfigHash: '', + detectorVersion: 'omni-sniff-ffprobe/1', + probeStatus: 'complete', + }, + }); + return binding!; + } + + async function readSnapshot(): Promise { + return JSON.parse( + await fs.readFile( + path.join(store.getOmniRootDir(), MEDIA_MEMORY_FILE_NAME), + 'utf8', + ), + ) as MediaMemorySnapshot; + } + + beforeEach(async () => { + const { MediaMemoryService: Service } = await import( + '../../services/media-memory/index.js' + ); + service = new Service(store.getOmniRootDir()); + }); + + it('commits the execution and threads the derived binding onto the delivery', async () => { + // Everything memory can later recall about a policy derivative — + // reuse (#8189), recall payloads, the honesty of the lineage graph — + // hangs off this one commit at the orchestrator's success point. If + // it stops firing, delivery still looks perfectly healthy: the + // derivative ships, and memory quietly stays empty forever. + const sourceBinding = await recordSource(service); + mockToolSuccess(); + const { deliveries, records } = await runFixedPolicies(config, source, { + store, + policies: [makePolicy()], + memory: { service, sourceBinding }, + }); + expect(records[0]).toMatchObject({ outcome: 'succeeded' }); + + const snapshot = await readSnapshot(); + const executions = Object.values(snapshot.executions); + expect(executions).toHaveLength(1); + const execution = executions[0]; + expect(execution).toMatchObject({ + // EXECUTED_ON: the version the tool actually ran against, not + // whatever the root of the tree happens to be. + sourceVersionId: sourceBinding.fileVersionId, + rootFileId: sourceBinding.rootFileId, + executionOrigin: { + kind: 'fixed_policy', + policyId: 'img-downsample', + stage: 'preprocessing', + }, + toolName: 'omni_downsample_image', + // Effective arguments AND the fingerprint they hash to: this pair + // is the content-identity reuse key, so a commit that recorded + // either loosely would hand later runs a false reuse hit. + finalArguments: { maxDimension: 1568 }, + omniConfigHash: computePolicyFingerprint( + 'omni_downsample_image', + { maxDimension: 1568 }, + undefined, + ), + }); + expect(execution.outputRefs).toHaveLength(1); + // Recall reports "processed at" from these two fields, so both must + // be real ISO instants in the right order — a missing or reversed + // window turns into a nonsense duration in the payload. + expect( + Date.parse(execution.completedAt) - Date.parse(execution.startedAt), + ).toBeGreaterThanOrEqual(0); + + const degradedSha = sha256Of(DEGRADED_BYTES); + const derivedVersion = Object.values(snapshot.versions).find( + (version) => version.sha256 === degradedSha, + ); + expect(derivedVersion).toBeDefined(); + // DERIVED_FROM / PRODUCED_BY: without both edges the derivative is + // an orphan node and recall can never reach it from the user's file. + expect(derivedVersion!.parentVersionId).toBe(sourceBinding.fileVersionId); + expect(derivedVersion!.producedByExecutionId).toBe(execution.executionId); + expect(snapshot.files[derivedVersion!.fileId].rootFileId).toBe( + sourceBinding.rootFileId, + ); + + // The binding rides out on the delivery: the transport guard and the + // reactive ladder commit their own passes onto it, so a delivery that + // ships without it silently starts a second, disconnected lineage. + expect(deliveries).toHaveLength(1); + expect(deliveries[0].memoryBinding).toEqual({ + fileId: derivedVersion!.fileId, + fileVersionId: derivedVersion!.fileVersionId, + rootFileId: sourceBinding.rootFileId, + }); + }); + + it('threads each derived binding into the next pass as its own source', async () => { + // Two-stage lineage: the second policy runs on the FIRST policy's + // derivative. Committing that pass against the root binding instead + // of the derivative's own would flatten the chain into two siblings + // of the root — the graph would then claim the 800px image was + // produced from the original PNG, which is a lie about provenance + // and breaks reuse for the intermediate. + const sourceBinding = await recordSource(service); + // Distinct bytes per rung so each stage has its own content identity. + executeToolCallMock.mockImplementation( + async (_config: Config, request: ToolCallRequestInfo) => { + const outputDir = request.args['outputDir'] as string; + const bytes = `degraded-${request.args['maxDimension']}`; + await fs.writeFile(path.join(outputDir, 'out.jpg'), bytes); + return { + callId: request.callId, + responseParts: [], + resultDisplay: undefined, + error: undefined, + errorType: undefined, + policyArtifacts: { + toolName: request.name, + invocationId: request.callId, + executionOrigin: request.executionOrigin, + artifacts: [ + { + kind: 'image', + storage: 'workspace', + title: 'out.jpg', + workspacePath: 'out.jpg', + mimeType: 'image/jpeg', + metadata: { omniDisclosure: 'Downsampled.' }, + }, + ], + }, + }; + }, + ); + + const { deliveries } = await runFixedPolicies(config, source, { + store, + policies: [ + makePolicy({ + id: 'stage-1', + output: { + reprocessMedia: true, + source: 'omit', + artifacts: { '*': 'include' }, + }, + }), + makePolicy({ + id: 'stage-2', + origins: ['policy'], + arguments: { maxDimension: 800 }, + }), + ], + memory: { service, sourceBinding }, + }); + expect(executeToolCallMock).toHaveBeenCalledTimes(2); + + const snapshot = await readSnapshot(); + const versionBySha = (sha: string) => + Object.values(snapshot.versions).find((v) => v.sha256 === sha)!; + const stage1Version = versionBySha(sha256Of('degraded-1568')); + const stage2Version = versionBySha(sha256Of('degraded-800')); + const stage2Execution = Object.values(snapshot.executions).find( + (e) => + e.executionOrigin.kind === 'fixed_policy' && + e.executionOrigin.policyId === 'stage-2', + )!; + expect(stage2Execution.sourceVersionId).toBe(stage1Version.fileVersionId); + expect(stage2Version.parentVersionId).toBe(stage1Version.fileVersionId); + // Depth grows, but every node stays inside the ONE root's tree — + // that root bounds every recall traversal. + expect(snapshot.files[stage2Version.fileId].rootFileId).toBe( + sourceBinding.rootFileId, + ); + expect(deliveries).toHaveLength(1); + expect(deliveries[0].memoryBinding).toMatchObject({ + fileVersionId: stage2Version.fileVersionId, + }); + }); + + it('commits a degradation-cache hit onto the same lineage', async () => { + // The degradation cache is a workspace file that outlives a wiped + // memory.json (or predates memory being switched on at all). On that + // replay the tool never runs, so the post-promotion commit above + // never fires — without the cache-hit commit the delivered + // derivative would carry no binding, and every later pass in the + // pipeline would derive from nothing. + mockToolSuccess(); + const seeded = await runFixedPolicies(config, source, { + store, + policies: [makePolicy()], + }); + expect(seeded.records[0]).toMatchObject({ outcome: 'succeeded' }); + + const sourceBinding = await recordSource(service); + const { deliveries, records } = await runFixedPolicies(config, source, { + store, + policies: [makePolicy()], + memory: { service, sourceBinding }, + }); + expect(records[0]).toMatchObject({ outcome: 'cache_hit' }); + expect(executeToolCallMock).toHaveBeenCalledTimes(1); + + const snapshot = await readSnapshot(); + const executions = Object.values(snapshot.executions); + expect(executions).toHaveLength(1); + expect(executions[0]).toMatchObject({ + invocationId: 'cache-hit', + sourceVersionId: sourceBinding.fileVersionId, + toolName: 'omni_downsample_image', + }); + const derivedVersion = Object.values(snapshot.versions).find( + (version) => version.sha256 === sha256Of(DEGRADED_BYTES), + ); + expect(derivedVersion).toBeDefined(); + expect(deliveries[0].memoryBinding).toEqual({ + fileId: derivedVersion!.fileId, + fileVersionId: derivedVersion!.fileVersionId, + rootFileId: sourceBinding.rootFileId, + }); + }); + }); }); diff --git a/packages/core/src/omni/policy/orchestrator.ts b/packages/core/src/omni/policy/orchestrator.ts index e29f61f496d..43eabd8c8a1 100644 --- a/packages/core/src/omni/policy/orchestrator.ts +++ b/packages/core/src/omni/policy/orchestrator.ts @@ -34,6 +34,12 @@ import { } from './degradation-cache.js'; import { DEFAULT_OMNI_PROCESSING_LIMITS } from './config.js'; import { resolvePolicyToolSettings } from './tools/media-policy-tool.js'; +import type { + MediaMemoryBinding, + MediaMemoryService, + PolicyOutputInput, + ReusableExecutionOutputs, +} from '../../services/media-memory/index.js'; import type { FixedPolicyOrigin, NormalizedFixedPolicy, @@ -55,6 +61,10 @@ export interface PolicyDeliveryResource { disclosure?: string; /** True when the resource is a lossy derivative of the user's input. */ degraded?: boolean; + /** Media-memory identity of this resource, when memory collection is + * active (S5): threaded so downstream policy passes (transport guard, + * reactive ladder) commit onto the same lineage graph. */ + memoryBinding?: MediaMemoryBinding; } /** Size ceiling for non-media (`kind: 'file'`) policy artifacts — the @@ -119,6 +129,14 @@ export interface RunFixedPoliciesOptions { /** Per-root derivation budgets (decision D11); the system defaults * apply when the caller has no normalized processing config. */ limits?: NormalizedOmniProcessingLimits; + /** Media-memory collection (S5). When present, successful executions + * commit onto the memory graph; `sourceBinding` is the root resource's + * identity when the caller already recorded it. Collection failures + * never block delivery (design M §6.4). */ + memory?: { + service: MediaMemoryService; + sourceBinding?: MediaMemoryBinding; + }; } /** Root resource entering the orchestrator. */ @@ -128,6 +146,9 @@ export interface PolicySourceResource { /** User-recognizable name for records and error messages. */ displayName: string; origin: Extract; + /** Content hash when the caller already computed it (memory + * collection records it at recognition time); avoids re-hashing. */ + sha256?: string; } /** Thrown when a policy invocation fails and the failure must abort the @@ -152,6 +173,11 @@ interface WorkItem { sha256?: string; disclosure?: string; degraded?: boolean; + /** Media-memory identity of this item (S5). Set on the root from + * `options.memory.sourceBinding` and on derived items from the commit + * of the execution that produced them; absent when memory is off or + * the identity is unknown (commit failed). */ + memoryBinding?: MediaMemoryBinding; /** Per-derivation-chain run counts (policy id → runs). Copied — never * shared — on derivation, so sibling branches cap independently. */ lineageRuns: Map; @@ -173,6 +199,8 @@ interface PolicyExecution { degraded: boolean; /** `metadata.omniRole`, when the tool labeled the artifact. */ role?: string; + /** Media-memory identity, when collection committed this artifact. */ + memoryBinding?: MediaMemoryBinding; }>; /** Non-media file artifacts (transcripts). Never re-enter matching. */ derivedFiles: PolicyFileDelivery[]; @@ -396,6 +424,8 @@ async function runFixedPoliciesUnbounded( recognized: source.recognized, label: source.displayName, origin: source.origin, + sha256: source.sha256, + memoryBinding: options.memory?.sourceBinding, lineageRuns: new Map(), depth: 0, deliver: true, @@ -505,6 +535,9 @@ async function runFixedPoliciesUnbounded( options.store, cache, options.signal, + options.memory && item.memoryBinding + ? { service: options.memory.service, source: item.memoryBinding } + : undefined, ); records.push({ policyId: policy.id, @@ -612,6 +645,7 @@ async function runFixedPoliciesUnbounded( sha256: item.sha256, disclosure: item.disclosure, degraded: item.degraded, + memoryBinding: item.memoryBinding, })), fileDeliveries, records, @@ -619,7 +653,7 @@ async function runFixedPoliciesUnbounded( } /** Validated view of one media artifact after descriptor/staging checks. */ -interface ValidatedMediaArtifact { +export interface ValidatedMediaArtifact { kind: 'media'; absolutePath: string; recognized: RecognizedMedia; @@ -632,7 +666,7 @@ interface ValidatedMediaArtifact { /** Validated view of one non-media file artifact (transcript protocol, * upstream P §6.2): bounded UTF-8 text, never probed as media. */ -interface ValidatedFileArtifact { +export interface ValidatedFileArtifact { kind: 'file'; absolutePath: string; mimeType: string; @@ -644,7 +678,7 @@ interface ValidatedFileArtifact { role?: string; } -type ValidatedArtifact = ValidatedMediaArtifact | ValidatedFileArtifact; +export type ValidatedArtifact = ValidatedMediaArtifact | ValidatedFileArtifact; /** * Execute one policy against one work item: degradation-cache lookup, @@ -659,6 +693,9 @@ async function executePolicy( store: OmniObjectStore, cache: OmniDegradationCache, signal: AbortSignal | undefined, + memory: + | { service: MediaMemoryService; source: MediaMemoryBinding } + | undefined, ): Promise { const tool = config.getToolRegistry().getTool(policy.toolName); const descriptor = tool?.mediaPolicyDescriptor; @@ -684,6 +721,57 @@ async function executePolicy( effectiveArguments, descriptor.version, ); + // Memory collection needs wall-clock execution bounds; captured here so + // the cache-hit path (which re-materializes the same execution node) + // records honest, if near-zero, durations. + const startedAt = new Date().toISOString(); + // Execution-free reuse (#8189 «同文件同 settings 二次触发同一 policy： + // 直接复用，无重复执行»). Consulted BEFORE the degradation cache because + // memory covers every output shape — multi-output tools and text + // products (an 81-minute transcript!) that the cache deliberately never + // stores, and therefore used to re-transcode on every delivery. + if (memory) { + const reusable = await memory.service.findReusableOutputs( + item.sha256, + fingerprint, + ); + const rebuilt = reusable + ? await rebuildReusedOutputs(reusable, descriptor, store, signal) + : undefined; + if (rebuilt) { + debugLogger.debug( + `memory reuse hit: policy=${policy.id} sha256=${item.sha256.slice(0, 12)}… ` + + `outputs=${rebuilt.outputs.length} (tool not executed)`, + ); + // The reusing file records its OWN execution, stamped with + // `reusedExecutionId` by the service (M §11.3) — provenance stays + // per-file while the computation is shared. + const commit = await memory.service.commitPolicySucceeded({ + invocationId: 'memory-reuse', + source: memory.source, + executionOrigin: { + kind: 'fixed_policy', + policyId: policy.id, + stage: policy.stage, + }, + toolName: policy.toolName, + toolVersion: descriptor.version, + finalArguments: effectiveArguments, + omniConfigHash: fingerprint, + startedAt, + completedAt: new Date().toISOString(), + outputs: rebuilt.outputs, + }); + return { + outcome: 'succeeded', + derived: rebuilt.derived.map((d) => ({ + ...d, + memoryBinding: commit?.mediaBindings.get(d.sha256), + })), + derivedFiles: rebuilt.derivedFiles, + }; + } + } const hit = await cache.get(item.sha256, fingerprint); if (hit) { try { @@ -713,6 +801,41 @@ async function executePolicy( debugLogger.debug( `degradation cache hit: policy=${policy.id} sha256=${item.sha256.slice(0, 12)}…`, ); + // A cache hit converges on the same content-keyed execution node + // as the original run (design M §11: no duplicate nodes) — the + // commit is a no-op when memory already has it, and only writes + // when the memory store was wiped while the degradation cache + // survived. Never blocks delivery. + const commit = memory + ? await memory.service.commitPolicySucceeded({ + invocationId: 'cache-hit', + source: memory.source, + executionOrigin: { + kind: 'fixed_policy', + policyId: policy.id, + stage: policy.stage, + }, + toolName: policy.toolName, + toolVersion: descriptor.version, + finalArguments: effectiveArguments, + omniConfigHash: fingerprint, + startedAt, + completedAt: new Date().toISOString(), + outputs: [ + { + kind: 'media', + objectPath, + sha256: hit.degradedSha256, + mediaType: recognized.modality, + metadata: recognized.metadata, + sizeBytes: recognized.sizeBytes, + mimeType: recognized.detectedMimeType, + role: hit.role, + disclosure: hit.disclosure, + }, + ], + }) + : undefined; return { outcome: 'cache_hit', derived: [ @@ -723,6 +846,7 @@ async function executePolicy( disclosure: hit.disclosure, degraded: true, role: hit.role, + memoryBinding: commit?.mediaBindings.get(hit.degradedSha256), }, ], derivedFiles: [], @@ -855,6 +979,58 @@ async function executePolicy( }); } } + // Memory collection (S5, design M §6.4): commit AFTER promotion — the + // objects/ bytes the records reference already exist — and in one shot + // at the point where every commit input coexists. A collection failure + // logs inside the service and returns undefined; delivery proceeds. + if (memory) { + const outputs: PolicyOutputInput[] = promoted.map( + ({ artifact, objectPath }) => + artifact.kind === 'media' + ? { + kind: 'media' as const, + objectPath, + sha256: artifact.sha256, + mediaType: artifact.recognized.modality, + metadata: artifact.recognized.metadata, + sizeBytes: artifact.recognized.sizeBytes, + mimeType: artifact.recognized.detectedMimeType, + role: artifact.role, + disclosure: artifact.disclosure, + } + : { + kind: 'text' as const, + objectPath, + sha256: artifact.sha256, + mimeType: artifact.mimeType, + text: artifact.text, + sizeBytes: artifact.sizeBytes, + role: artifact.role, + disclosure: artifact.disclosure, + }, + ); + const commit = await memory.service.commitPolicySucceeded({ + invocationId, + source: memory.source, + executionOrigin: { + kind: 'fixed_policy', + policyId: policy.id, + stage: policy.stage, + }, + toolName: policy.toolName, + toolVersion: descriptor.version, + finalArguments: effectiveArguments, + omniConfigHash: fingerprint, + startedAt, + completedAt: new Date().toISOString(), + outputs, + }); + if (commit) { + for (const entry of derived) { + entry.memoryBinding = commit.mediaBindings.get(entry.sha256); + } + } + } // The cache maps one input to ONE media derivative; multi-output tools // and file artifacts (whose cache-hit path depends on media // re-recognition) are simply not cached — re-run instead of guessing. @@ -909,7 +1085,113 @@ async function executePolicy( * protocol), bounded strict-UTF-8 text matching a declared file output — * and, for lossy outputs, a non-empty `metadata.omniDisclosure`. */ -async function validateArtifact( +/** + * Rebuild the deliverables of a recorded execution without running the + * tool. Every reuse is verified exactly like a degradation-cache hit: the + * object must still be a regular file, its bytes must still hash to the + * recorded identity (memory.json is project-local and hand-editable), and + * a media object must re-recognize as a type this tool DECLARES producing + * — otherwise a crafted record could route arbitrary store content + * through a policy that never made it. Any doubt returns undefined and the + * caller re-derives from source. + */ +async function rebuildReusedOutputs( + reusable: ReusableExecutionOutputs, + descriptor: MediaPolicyToolDescriptor, + store: OmniObjectStore, + signal: AbortSignal | undefined, +): Promise< + | { + outputs: PolicyOutputInput[]; + derived: PolicyExecution['derived']; + derivedFiles: PolicyExecution['derivedFiles']; + } + | undefined +> { + const outputs: PolicyOutputInput[] = []; + const derived: PolicyExecution['derived'] = []; + const derivedFiles: PolicyExecution['derivedFiles'] = []; + try { + for (const record of reusable.outputs) { + // A media output carries its object path (its derived version's file + // record). A TEXT output has no version node, so its location is + // reconstructed from the content hash — the object store is + // content-addressed, which is what makes that sound. Without this, + // text products (transcripts — the most expensive thing to re-derive + // and the whole point of #8189) could never be reused. + const objectPath = + record.objectPath ?? + store.objectPathFor(record.sha256, extensionForMime(record.mimeType)); + const stat = await fs.lstat(objectPath).catch(() => undefined); + if (!stat?.isFile() || stat.isSymbolicLink()) return undefined; + if ((await hashFileSha256(objectPath, signal)) !== record.sha256) { + return undefined; + } + if (record.kind === 'media') { + const recognized = await recognizeMediaFile(objectPath, { signal }); + const declared = descriptor.outputs.some( + (o) => + o.kind === 'media' && + o.mimeTypes?.includes(recognized.detectedMimeType), + ); + if (!declared) return undefined; + outputs.push({ + kind: 'media', + objectPath, + sha256: record.sha256, + mediaType: recognized.modality, + metadata: recognized.metadata, + sizeBytes: recognized.sizeBytes, + mimeType: recognized.detectedMimeType, + role: record.role, + disclosure: record.disclosure, + }); + derived.push({ + filePath: objectPath, + recognized, + sha256: record.sha256, + disclosure: record.disclosure, + degraded: record.disclosure !== undefined, + role: record.role, + }); + } else { + // Read the FULL text back from the promoted object: the entry's + // inlineText is bounded by `collection.maxInlineTextBytes`, and a + // truncated transcript must never be delivered as the whole one. + const text = await fs.readFile(objectPath, 'utf8'); + outputs.push({ + kind: 'text', + objectPath, + sha256: record.sha256, + mimeType: record.mimeType, + text, + sizeBytes: record.sizeBytes, + role: record.role, + disclosure: record.disclosure, + }); + derivedFiles.push({ + filePath: objectPath, + role: record.role, + mimeType: record.mimeType, + text, + sha256: record.sha256, + sizeBytes: record.sizeBytes, + disclosure: record.disclosure, + }); + } + } + } catch (err) { + if (signal?.aborted) throw err; + debugLogger.debug( + `memory reuse could not be verified, re-deriving: ` + + `${err instanceof Error ? err.message : String(err)}`, + ); + return undefined; + } + return { outputs, derived, derivedFiles }; +} + +export async function validateArtifact( artifact: ToolArtifact, descriptor: MediaPolicyToolDescriptor, stagingDir: string, @@ -1028,7 +1310,7 @@ async function validateArtifact( /** Every required media/file output declared by the descriptor must have * been produced (§5 completeness check). */ -function assertRequiredOutputsPresent( +export function assertRequiredOutputsPresent( descriptor: MediaPolicyToolDescriptor, validated: ValidatedArtifact[], toolName: string, diff --git a/packages/core/src/omni/policy/tools/clip-video.test.ts b/packages/core/src/omni/policy/tools/clip-video.test.ts index 1f024d9f22e..3b5733cc8b6 100644 --- a/packages/core/src/omni/policy/tools/clip-video.test.ts +++ b/packages/core/src/omni/policy/tools/clip-video.test.ts @@ -77,7 +77,10 @@ describe('OmniClipVideoTool', () => { expect(tool.name).toBe(OMNI_CLIP_VIDEO_TOOL_NAME); expect(tool.mediaPolicyDescriptor).toEqual({ kind: 'media_policy', - version: '1', + // Bumped with the `omniRole: 'clip'` annotation so pre-role cache + // entries and recorded executions cannot converge onto this + // fingerprint and keep reporting an excerpt as complete coverage. + version: '2', inputMediaTypes: ['video'], outputs: [ { @@ -106,7 +109,7 @@ describe('OmniClipVideoTool', () => { 'video', signal, ); - const outputPath = path.join(outputDir, 'clip.mp4'); + const outputPath = path.join(outputDir, 'clip-clip-10s+15s.mp4'); expect(mocks.runFfmpeg).toHaveBeenCalledWith( [ '-y', @@ -141,11 +144,12 @@ describe('OmniClipVideoTool', () => { kind: 'video', storage: 'workspace', title: 'Clipped video', - workspacePath: 'clip.mp4', + workspacePath: 'clip-clip-10s+15s.mp4', mimeType: 'video/mp4', sizeBytes: OUTPUT_SIZE, metadata: { omniDisclosure: '原 63s → 片段 [10s–25s] 15s，片段外内容全部丢弃', + omniRole: 'clip', }, }, ]); diff --git a/packages/core/src/omni/policy/tools/clip-video.ts b/packages/core/src/omni/policy/tools/clip-video.ts index 214a8382f42..ec33d7cfc28 100644 --- a/packages/core/src/omni/policy/tools/clip-video.ts +++ b/packages/core/src/omni/policy/tools/clip-video.ts @@ -23,6 +23,7 @@ import { mediaPolicyToolError, mediaPolicyToolFailure, mediaPolicyToolSuccess, + policyOutputFileName, resolvePolicyToolTimeoutMs, type MediaPolicyIoParams, type MediaPolicyToolConfigView, @@ -39,8 +40,6 @@ export const CLIP_VIDEO_DEFAULTS = { audioBitrateKbps: 128, } as const; -const OUTPUT_FILE_NAME = 'clip.mp4'; - export interface ClipVideoParams extends MediaPolicyIoParams { /** Clip start in seconds (default 0). */ startSec?: number; @@ -63,7 +62,12 @@ const TUNABLE_SCHEMA_PROPERTIES = { const DESCRIPTOR: MediaPolicyToolDescriptor = { kind: 'media_policy', - version: '1', + // '2': outputs now carry `metadata.omniRole: 'clip'`, which memory maps + // to `partial` coverage. Pre-'2' cache entries and recorded executions + // hold role-less outputs whose coverage was derived as `complete`; + // sharing a version would let them converge onto the same fingerprint + // and keep reporting a temporal excerpt as complete footage. + version: '2', inputMediaTypes: ['video'], outputs: [ { @@ -137,7 +141,18 @@ class ClipVideoInvocation extends BaseMediaPolicyToolInvocation ); } - const outputPath = path.join(this.params.outputDir, OUTPUT_FILE_NAME); + // Self-describing name: two different spans of one source coexist + // instead of the later clip destroying the earlier one. + const outputFileName = policyOutputFileName({ + inputPath: this.params.inputPath, + operation: 'clip', + variant: + durationSec !== undefined + ? `${Math.round(startSec)}s+${Math.round(durationSec)}s` + : `${Math.round(startSec)}s-end`, + extension: '.mp4', + }); + const outputPath = path.join(this.params.outputDir, outputFileName); // Input-side -ss/-t plus a full re-encode: frame-accurate cuts // regardless of keyframe placement (`-c copy` snaps to keyframes). // The scale filter only forces even dimensions (libx264 hard @@ -193,12 +208,15 @@ class ClipVideoInvocation extends BaseMediaPolicyToolInvocation return mediaPolicyToolSuccess({ outputDir: this.params.outputDir, - outputFileName: OUTPUT_FILE_NAME, + outputFileName, artifactKind: 'video', title: 'Clipped video', mimeType: 'video/mp4', sizeBytes: outputSizeBytes, disclosure, + // Marks the artifact as a temporal excerpt for downstream role + // consumers (output routing selectors, memory coverage). + role: 'clip', }); } catch (error) { return mediaPolicyToolFailure(error); @@ -225,7 +243,7 @@ export class OmniClipVideoTool extends BaseMediaPolicyTool { ...MEDIA_POLICY_IO_SCHEMA_PROPERTIES, ...TUNABLE_SCHEMA_PROPERTIES, }, - required: ['inputPath', 'outputDir'], + required: ['outputDir'], additionalProperties: false, }, config, diff --git a/packages/core/src/omni/policy/tools/convert-image.test.ts b/packages/core/src/omni/policy/tools/convert-image.test.ts index 413fa9e8342..96ee3ccbccf 100644 --- a/packages/core/src/omni/policy/tools/convert-image.test.ts +++ b/packages/core/src/omni/policy/tools/convert-image.test.ts @@ -127,7 +127,9 @@ describe('OmniConvertImageTool', () => { }); expect(rotate).toHaveBeenCalledOnce(); expect(jpeg).toHaveBeenCalledWith({ quality: 90 }); - expect(toFile).toHaveBeenCalledWith(path.join(outputDir, 'converted.jpg')); + expect(toFile).toHaveBeenCalledWith( + path.join(outputDir, 'photo-converted.jpg'), + ); expect(result.error).toBeUndefined(); expect(result.artifacts).toEqual([ @@ -135,7 +137,7 @@ describe('OmniConvertImageTool', () => { kind: 'image', storage: 'workspace', title: 'Converted image', - workspacePath: 'converted.jpg', + workspacePath: 'photo-converted.jpg', mimeType: 'image/jpeg', sizeBytes: OUTPUT_SIZE, metadata: { @@ -169,9 +171,11 @@ describe('OmniConvertImageTool', () => { const { result } = await run({ format: 'png' }); expect(png).toHaveBeenCalledWith(); expect(jpeg).not.toHaveBeenCalled(); - expect(toFile).toHaveBeenCalledWith(path.join(outputDir, 'converted.png')); + expect(toFile).toHaveBeenCalledWith( + path.join(outputDir, 'photo-converted.png'), + ); expect(result.artifacts?.[0]).toMatchObject({ - workspacePath: 'converted.png', + workspacePath: 'photo-converted.png', mimeType: 'image/png', }); expect(result.artifacts?.[0]?.metadata?.['omniDisclosure']).toBe( @@ -184,7 +188,7 @@ describe('OmniConvertImageTool', () => { const { result } = await run({ format: 'webp', quality: 60 }); expect(webp).toHaveBeenCalledWith({ quality: 60 }); expect(result.artifacts?.[0]).toMatchObject({ - workspacePath: 'converted.webp', + workspacePath: 'photo-converted.webp', mimeType: 'image/webp', }); expect(result.artifacts?.[0]?.metadata?.['omniDisclosure']).toBe( diff --git a/packages/core/src/omni/policy/tools/convert-image.ts b/packages/core/src/omni/policy/tools/convert-image.ts index fde77e3036c..8198f5417c9 100644 --- a/packages/core/src/omni/policy/tools/convert-image.ts +++ b/packages/core/src/omni/policy/tools/convert-image.ts @@ -22,6 +22,7 @@ import { mediaPolicyToolError, mediaPolicyToolFailure, mediaPolicyToolSuccess, + policyOutputFileName, resolvePolicyToolTimeoutMs, sharpTimeoutSeconds, type MediaPolicyIoParams, @@ -38,7 +39,8 @@ export const CONVERT_IMAGE_DEFAULTS = { } as const; interface OutputFormat { - fileName: string; + /** Output extension, with the leading dot. */ + extension: string; mimeType: string; label: string; /** The disclosure's loss clause for this target format, given the @@ -54,7 +56,7 @@ const ALPHA_CAPABLE_CODECS = new Set(['png', 'webp', 'gif', 'tiff']); const OUTPUT_FORMATS: Record = { jpeg: { - fileName: 'converted.jpg', + extension: '.jpg', mimeType: 'image/jpeg', label: 'JPEG', lossNote: (codec) => @@ -66,14 +68,14 @@ const OUTPUT_FORMATS: Record = { encode: (p, quality) => p.jpeg({ quality }), }, png: { - fileName: 'converted.png', + extension: '.png', mimeType: 'image/png', label: 'PNG', lossNote: () => '元数据丢弃', encode: (p) => p.png(), }, webp: { - fileName: 'converted.webp', + extension: '.webp', mimeType: 'image/webp', label: 'WEBP', lossNote: () => '元数据丢弃', @@ -194,7 +196,12 @@ class ConvertImageInvocation extends BaseMediaPolicyToolInvocation { 'audio', signal, ); - const outputPath = path.join(outputDir, 'downsampled.m4a'); + const outputPath = path.join(outputDir, 'track-downsampled.m4a'); expect(mocks.runFfmpeg).toHaveBeenCalledTimes(1); expect(mocks.runFfmpeg).toHaveBeenCalledWith( [ @@ -136,7 +136,7 @@ describe('OmniDownsampleAudioTool', () => { kind: 'audio', storage: 'workspace', title: 'Downsampled audio', - workspacePath: 'downsampled.m4a', + workspacePath: 'track-downsampled.m4a', mimeType: 'audio/mp4', sizeBytes: OUTPUT_SIZE, metadata: { diff --git a/packages/core/src/omni/policy/tools/downsample-audio.ts b/packages/core/src/omni/policy/tools/downsample-audio.ts index 76b48a19a53..605b5edd5e0 100644 --- a/packages/core/src/omni/policy/tools/downsample-audio.ts +++ b/packages/core/src/omni/policy/tools/downsample-audio.ts @@ -25,6 +25,7 @@ import { mediaPolicyToolError, mediaPolicyToolFailure, mediaPolicyToolSuccess, + policyOutputFileName, resolvePolicyToolTimeoutMs, type MediaPolicyIoParams, type MediaPolicyToolConfigView, @@ -39,8 +40,6 @@ export const DOWNSAMPLE_AUDIO_DEFAULTS = { channels: 1, } as const; -const OUTPUT_FILE_NAME = 'downsampled.m4a'; - export interface DownsampleAudioParams extends MediaPolicyIoParams { /** Output bit rate in kbit/s. */ bitrateKbps?: number; @@ -137,7 +136,14 @@ class DownsampleAudioInvocation extends BaseMediaPolicyToolInvocation { }); expect(jpeg).toHaveBeenCalledWith({ quality: 75 }); expect(toFile).toHaveBeenCalledWith( - path.join(outputDir, 'downsampled.jpg'), + path.join(outputDir, 'photo-downsampled.jpg'), ); expect(result.error).toBeUndefined(); @@ -149,7 +149,7 @@ describe('OmniDownsampleImageTool', () => { kind: 'image', storage: 'workspace', title: 'Downsampled image', - workspacePath: 'downsampled.jpg', + workspacePath: 'photo-downsampled.jpg', mimeType: 'image/jpeg', sizeBytes: OUTPUT_SIZE, metadata: { diff --git a/packages/core/src/omni/policy/tools/downsample-image.ts b/packages/core/src/omni/policy/tools/downsample-image.ts index 596662c97e7..6d3124b1b8f 100644 --- a/packages/core/src/omni/policy/tools/downsample-image.ts +++ b/packages/core/src/omni/policy/tools/downsample-image.ts @@ -22,6 +22,7 @@ import { mediaPolicyToolError, mediaPolicyToolFailure, mediaPolicyToolSuccess, + policyOutputFileName, resolvePolicyToolTimeoutMs, sharpTimeoutSeconds, type MediaPolicyIoParams, @@ -37,8 +38,6 @@ export const DOWNSAMPLE_IMAGE_DEFAULTS = { quality: 75, } as const; -const OUTPUT_FILE_NAME = 'downsampled.jpg'; - export interface DownsampleImageParams extends MediaPolicyIoParams { /** Longest-edge ceiling in pixels; aspect ratio is preserved. */ maxDimension?: number; @@ -142,7 +141,14 @@ class DownsampleImageInvocation extends BaseMediaPolicyToolInvocation { 'video', signal, ); - const outputPath = path.join(outputDir, 'downscaled.mp4'); + const outputPath = path.join(outputDir, 'clip-downscaled.mp4'); expect(mocks.runFfmpeg).toHaveBeenCalledTimes(1); expect(mocks.runFfmpeg).toHaveBeenCalledWith( [ @@ -143,7 +143,7 @@ describe('OmniDownscaleVideoTool', () => { kind: 'video', storage: 'workspace', title: 'Downscaled video', - workspacePath: 'downscaled.mp4', + workspacePath: 'clip-downscaled.mp4', mimeType: 'video/mp4', sizeBytes: OUTPUT_SIZE, metadata: { @@ -176,9 +176,9 @@ describe('OmniDownscaleVideoTool', () => { // Height drops, frame rate does not. probe({ height: 720, frameRate: 8 }); const heightOnly = await run(); - expect( - heightOnly.result.artifacts?.[0]?.metadata?.['omniDisclosure'], - ).toBe('原 720p8/2MB → 480p10/300KB，分辨率下降，细节受损'); + expect(heightOnly.result.artifacts?.[0]?.metadata?.['omniDisclosure']).toBe( + '原 720p8/2MB → 480p10/300KB，分辨率下降，细节受损', + ); // Frame rate drops, height does not. probe({ height: 360, frameRate: 30 }); diff --git a/packages/core/src/omni/policy/tools/downscale-video.ts b/packages/core/src/omni/policy/tools/downscale-video.ts index b6c2f09df11..7443b1fdeb8 100644 --- a/packages/core/src/omni/policy/tools/downscale-video.ts +++ b/packages/core/src/omni/policy/tools/downscale-video.ts @@ -24,6 +24,7 @@ import { mediaPolicyToolError, mediaPolicyToolFailure, mediaPolicyToolSuccess, + policyOutputFileName, resolvePolicyToolTimeoutMs, createPolicyToolTimeoutBudget, type MediaPolicyIoParams, @@ -40,8 +41,6 @@ export const DOWNSCALE_VIDEO_DEFAULTS = { preset: 'veryfast', } as const; -const OUTPUT_FILE_NAME = 'downscaled.mp4'; - /** x264 presets accepted by the `preset` tunable. */ const X264_PRESETS = [ 'ultrafast', @@ -158,7 +157,14 @@ class DownscaleVideoInvocation extends BaseMediaPolicyToolInvocation [ '-y', '-i', @@ -218,14 +224,16 @@ class DownscaleVideoInvocation extends BaseMediaPolicyToolInvocation { 'video', signal, ); - const outputPath = path.join(outputDir, 'extracted.wav'); + const outputPath = path.join(outputDir, 'clip-audio.wav'); expect(mocks.runFfmpeg).toHaveBeenCalledWith( [ '-y', @@ -132,7 +132,7 @@ describe('OmniExtractAudioTool', () => { kind: 'audio', storage: 'workspace', title: 'Extracted audio track', - workspacePath: 'extracted.wav', + workspacePath: 'clip-audio.wav', mimeType: 'audio/wav', sizeBytes: OUTPUT_SIZE, metadata: { @@ -149,7 +149,7 @@ describe('OmniExtractAudioTool', () => { const args = mocks.runFfmpeg.mock.calls[0][0] as string[]; expect(args.join(' ')).toContain('-c:a libmp3lame -b:a 128k'); expect(result.artifacts?.[0]).toMatchObject({ - workspacePath: 'extracted.mp3', + workspacePath: 'clip-audio.mp3', mimeType: 'audio/mpeg', }); expect(result.artifacts?.[0]?.metadata?.['omniDisclosure']).toContain( @@ -163,7 +163,7 @@ describe('OmniExtractAudioTool', () => { const args = mocks.runFfmpeg.mock.calls[0][0] as string[]; expect(args.join(' ')).toContain('-c:a aac -b:a 64k'); expect(result.artifacts?.[0]).toMatchObject({ - workspacePath: 'extracted.m4a', + workspacePath: 'clip-audio.m4a', mimeType: 'audio/mp4', }); }); diff --git a/packages/core/src/omni/policy/tools/extract-audio.ts b/packages/core/src/omni/policy/tools/extract-audio.ts index b60f6a7cf2d..e17884e6524 100644 --- a/packages/core/src/omni/policy/tools/extract-audio.ts +++ b/packages/core/src/omni/policy/tools/extract-audio.ts @@ -25,6 +25,7 @@ import { mediaPolicyToolError, mediaPolicyToolFailure, mediaPolicyToolSuccess, + policyOutputFileName, resolvePolicyToolTimeoutMs, type MediaPolicyIoParams, type MediaPolicyToolConfigView, @@ -42,7 +43,8 @@ export const EXTRACT_AUDIO_DEFAULTS = { } as const; interface OutputFormat { - fileName: string; + /** Output extension, with the leading dot. */ + extension: string; mimeType: string; label: string; /** Codec args; lossy formats consume the bit rate. */ @@ -51,19 +53,19 @@ interface OutputFormat { const OUTPUT_FORMATS: Record = { wav: { - fileName: 'extracted.wav', + extension: '.wav', mimeType: 'audio/wav', label: 'WAV', codecArgs: () => ['-c:a', 'pcm_s16le'], }, mp3: { - fileName: 'extracted.mp3', + extension: '.mp3', mimeType: 'audio/mpeg', label: 'MP3', codecArgs: (kbps) => ['-c:a', 'libmp3lame', '-b:a', `${kbps}k`], }, m4a: { - fileName: 'extracted.m4a', + extension: '.m4a', mimeType: 'audio/mp4', label: 'M4A', codecArgs: (kbps) => ['-c:a', 'aac', '-b:a', `${kbps}k`], @@ -158,7 +160,12 @@ class ExtractAudioInvocation extends BaseMediaPolicyToolInvocation { }); afterEach(async () => { + // Un-freeze Date.now for tests that spied on it — the budget-sharing + // test below depends on real elapsed time. + vi.restoreAllMocks(); await fs.rm(root, { recursive: true, force: true }); }); @@ -103,7 +106,11 @@ describe('OmniExtractKeyframesTool', () => { expect(tool.name).toBe(OMNI_EXTRACT_KEYFRAMES_TOOL_NAME); expect(tool.mediaPolicyDescriptor).toEqual({ kind: 'media_policy', - version: '1', + // Bumped with the `omniRole: 'keyframe'` annotation so pre-role + // cache entries and recorded executions cannot converge onto this + // fingerprint and keep reporting sampled frames as complete visual + // coverage. + version: '2', inputMediaTypes: ['video'], outputs: [ { @@ -125,6 +132,13 @@ describe('OmniExtractKeyframesTool', () => { describe('bucketed extraction (known duration, maxFrames > 1)', () => { it('spreads one frame per equal bucket across the FULL duration', async () => { + // The bucket loop's budget guard calls remainingTimeoutMs() before + // the ffmpeg call reads it again — on a slow runner a millisecond + // elapses in between and the exact-equality assertion below turns + // flaky (observed on CI: 599999 ≠ 600000). Freeze Date so the + // strong assertion stays deterministic; ffmpeg is mocked and + // nothing in this test needs real wall-clock time. + vi.spyOn(Date, 'now').mockReturnValue(1_755_000_000_000); probe({ durationMs: 80_000, width: 1920, height: 1080 }); // Every bucket has a scene change 3.5s into its window. mocks.runFfmpeg.mockImplementation(framesRun(1, [3.5])); @@ -154,7 +168,7 @@ describe('OmniExtractKeyframesTool', () => { '4', '-update', '1', - path.join(outputDir, 'keyframe_0001.jpg'), + path.join(outputDir, 'clip-keyframe-0001.jpg'), ], { signal, timeoutMs: DEFAULT_POLICY_TOOL_TIMEOUT_MS }, ); @@ -173,12 +187,13 @@ describe('OmniExtractKeyframesTool', () => { kind: 'image', storage: 'workspace', title: 'Keyframe 4/4', - workspacePath: 'keyframe_0004.jpg', + workspacePath: 'clip-keyframe-0004.jpg', mimeType: 'image/jpeg', sizeBytes: FRAME_SIZE, metadata: { omniDisclosure: '原视频 80s/1920×1080 → 关键帧 4/4 @ 63.5s，静态抽帧（全片分桶采样），时间连续性丢失', + omniRole: 'keyframe', }, }); for (const artifact of result.artifacts ?? []) { @@ -225,7 +240,7 @@ describe('OmniExtractKeyframesTool', () => { '4', '-update', '1', - path.join(outputDir, 'keyframe_0001.jpg'), + path.join(outputDir, 'clip-keyframe-0001.jpg'), ], { signal, timeoutMs: expect.any(Number) }, ); @@ -309,7 +324,11 @@ describe('OmniExtractKeyframesTool', () => { const secondTimeout = ( mocks.runFfmpeg.mock.calls[1][1] as { timeoutMs: number } ).timeoutMs; - expect(firstTimeout).toBe(DEFAULT_POLICY_TOOL_TIMEOUT_MS); + // The loop's budget guard reads the clock just before the ffmpeg + // call does — this test needs REAL time (the 50ms burn below), so + // tolerate the guard→use drift instead of freezing Date. + expect(firstTimeout).toBeGreaterThan(DEFAULT_POLICY_TOOL_TIMEOUT_MS - 40); + expect(firstTimeout).toBeLessThanOrEqual(DEFAULT_POLICY_TOOL_TIMEOUT_MS); // The second bucket gets only what the first one left, never a // fresh full budget (timers never fire early, so ≥40ms is gone). expect(secondTimeout).toBeLessThanOrEqual( @@ -405,7 +424,7 @@ describe('OmniExtractKeyframesTool', () => { '8', '-q:v', '4', - path.join(outputDir, 'keyframe_%04d.jpg'), + path.join(outputDir, 'clip-keyframe-%04d.jpg'), ], { signal, timeoutMs: DEFAULT_POLICY_TOOL_TIMEOUT_MS }, ); @@ -425,7 +444,7 @@ describe('OmniExtractKeyframesTool', () => { expect(mocks.runFfmpeg).toHaveBeenCalledTimes(1); const args = mocks.runFfmpeg.mock.calls[0][0] as string[]; expect(args[args.length - 1]).toBe( - path.join(outputDir, 'keyframe_%04d.jpg'), + path.join(outputDir, 'clip-keyframe-%04d.jpg'), ); expect(result.artifacts).toHaveLength(1); }); diff --git a/packages/core/src/omni/policy/tools/extract-keyframes.ts b/packages/core/src/omni/policy/tools/extract-keyframes.ts index e3f33dcf293..4b55d8517ef 100644 --- a/packages/core/src/omni/policy/tools/extract-keyframes.ts +++ b/packages/core/src/omni/policy/tools/extract-keyframes.ts @@ -28,6 +28,7 @@ import { MEDIA_POLICY_IO_SCHEMA_PROPERTIES, mediaPolicyToolError, mediaPolicyToolFailure, + policyOutputFileName, resolvePolicyToolTimeoutMs, createPolicyToolTimeoutBudget, type MediaPolicyIoParams, @@ -44,7 +45,13 @@ export const EXTRACT_KEYFRAMES_DEFAULTS = { maxDimension: 768, } as const; -const FRAME_FILE_PATTERN = /^keyframe_(\d{4})\.jpg$/; +/** Matcher for the frames of ONE source, derived from the name template + * so the lister can never pick up a sibling video's frames out of a + * shared outputDir. */ +function frameFileMatcher(nameTemplate: string): RegExp { + const escaped = nameTemplate.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + return new RegExp(`^${escaped.replace('%04d', '(\\d{4})')}$`); +} export interface ExtractKeyframesParams extends MediaPolicyIoParams { /** Maximum number of frames to extract. */ @@ -80,7 +87,13 @@ const TUNABLE_SCHEMA_PROPERTIES = { const DESCRIPTOR: MediaPolicyToolDescriptor = { kind: 'media_policy', - version: '1', + // '2': outputs now carry `metadata.omniRole: 'keyframe'`, which memory + // maps to `sampled` coverage. Pre-'2' cache entries and recorded + // executions hold role-less outputs whose coverage was derived as + // `complete`; sharing a version would let them converge onto the same + // fingerprint and keep reporting sampled frames as complete visual + // coverage — the model would answer about footage it never saw. + version: '2', inputMediaTypes: ['video'], outputs: [ { @@ -125,9 +138,13 @@ export function parseShowinfoTimestamps(stderr: string): number[] { } /** List produced keyframe files in frame order. */ -async function listFrameFiles(outputDir: string): Promise { +async function listFrameFiles( + outputDir: string, + nameTemplate: string, +): Promise { + const matcher = frameFileMatcher(nameTemplate); const entries = await fs.readdir(outputDir); - return entries.filter((name) => FRAME_FILE_PATTERN.test(name)).sort(); + return entries.filter((name) => matcher.test(name)).sort(); } /** @@ -268,6 +285,9 @@ class ExtractKeyframesInvocation extends BaseMediaPolicyToolInvocation { pattern, ); }); + + it('asks for inputPath or resourceId when neither was supplied', () => { + // Schema-level `required` deliberately omits inputPath (the gated + // model surface passes resourceId, resolved before validation) — so + // the neither-provided case must fail HERE with an actionable hint. + expect( + validateMediaPolicyIoParams({ + outputDir: '/b/staging', + } as unknown as Parameters[0]), + ).toMatch(/inputPath.*or resourceId/); + }); +}); + +describe('policyOutputFileName', () => { + it('keeps two spans of one source from colliding', () => { + const a = policyOutputFileName({ + inputPath: '/films/robot-dreams.mkv', + operation: 'clip', + variant: '90s+75s', + extension: '.mp4', + }); + const b = policyOutputFileName({ + inputPath: '/films/robot-dreams.mkv', + operation: 'clip', + variant: '2458s+75s', + extension: '.mp4', + }); + expect(a).toBe('robot-dreams-clip-90s+75s.mp4'); + expect(b).toBe('robot-dreams-clip-2458s+75s.mp4'); + expect(a).not.toBe(b); + }); + + it('keeps two sources from colliding on the same operation', () => { + const opts = { operation: 'audio', extension: '.wav' } as const; + expect( + policyOutputFileName({ inputPath: '/a/movie.mkv', ...opts }), + ).not.toBe(policyOutputFileName({ inputPath: '/b/other.mkv', ...opts })); + }); + + it('re-running one operation resolves to the same name (idempotent)', () => { + const opts = { + inputPath: '/a/movie.mkv', + operation: 'clip', + variant: '0s+30s', + extension: '.mp4', + } as const; + expect(policyOutputFileName(opts)).toBe(policyOutputFileName(opts)); + }); + + it('sanitizes a hostile or non-ASCII stem into a portable component', () => { + const name = policyOutputFileName({ + inputPath: '/tmp/《机器人之梦》 v2; rm -rf.mkv', + operation: 'keyframe', + variant: '0001', + extension: '.jpg', + }); + expect(name).toMatch(/^[A-Za-z0-9._+-]+$/); + expect(name.endsWith('-keyframe-0001.jpg')).toBe(true); + expect(name).not.toContain('/'); + expect(name).not.toContain(';'); + }); + + it('falls back to a placeholder when nothing portable survives', () => { + expect( + policyOutputFileName({ + inputPath: '/tmp/《》.mkv', + operation: 'audio', + extension: '.wav', + }), + ).toBe('media-audio.wav'); + }); }); describe('assertMediaPolicyIo', () => { @@ -146,9 +218,16 @@ describe('assertMediaPolicyIo', () => { it('rejects a missing input file', async () => { const outputDir = path.join(root, 'staging'); await fs.mkdir(outputDir); - await expect( - assertMediaPolicyIo({ inputPath: path.join(root, 'nope'), outputDir }), - ).rejects.toThrow(/input file not found/); + const error = await assertMediaPolicyIo({ + inputPath: path.join(root, 'nope'), + outputDir, + }).catch((err: Error) => err); + expect(error).toBeInstanceOf(Error); + expect((error as Error).message).toMatch(/input file not found/); + // Basename only: this message reaches the model, and a resourceId- + // resolved call must never leak the locator the handle stands in for + // (M §5.2). A full-path message satisfies the matcher above too. + expect((error as Error).message).not.toContain(root); }); it('rejects a symlinked input (never reads through a link)', async () => { diff --git a/packages/core/src/omni/policy/tools/media-policy-tool.ts b/packages/core/src/omni/policy/tools/media-policy-tool.ts index eca35e67002..4f6f6a96743 100644 --- a/packages/core/src/omni/policy/tools/media-policy-tool.ts +++ b/packages/core/src/omni/policy/tools/media-policy-tool.ts @@ -36,7 +36,10 @@ export const DEFAULT_POLICY_TOOL_TIMEOUT_MS = 600_000; /** Parameters every media-policy degradation tool shares: one input file, * one harness-injected output directory (the invocation's staging dir — - * the tool's ONLY permitted output location). */ + * the tool's ONLY permitted output location). `inputPath` is guaranteed + * present by the time an invocation is built: fixed-policy calls always + * carry it, and gated model/client calls that passed `resourceId` instead + * had it resolved by the call gate (model-access.ts) before validation. */ export interface MediaPolicyIoParams { /** Absolute path of the source media file. */ inputPath: string; @@ -44,11 +47,24 @@ export interface MediaPolicyIoParams { outputDir: string; } -/** JSON-schema fragments for the shared io parameters. */ +/** JSON-schema fragments for the shared io parameters. `resourceId` is + * the model-facing alternative to `inputPath` (memory design M §5.2): + * the model references delivered media by its opaque session handle and + * the call gate resolves the handle to the real locator — it never + * appears in the arguments an invocation is built with. */ export const MEDIA_POLICY_IO_SCHEMA_PROPERTIES = { inputPath: { type: 'string', - description: 'Absolute path of the source media file.', + description: + 'Absolute path of the source media file. Provide exactly one of ' + + 'inputPath or resourceId.', + }, + resourceId: { + type: 'string', + description: + 'Opaque session media handle (from a 【媒体资源】 annotation or a ' + + 'recall result) naming the source media. Provide exactly one of ' + + 'inputPath or resourceId.', }, outputDir: { type: 'string', @@ -236,11 +252,77 @@ export abstract class BaseMediaPolicyTool< } } +/** Longest source stem kept in a generated output name: long enough to + * stay recognizable, short enough that a deep outputDir plus a variant + * suffix cannot approach the filesystem's per-component limit. */ +const MAX_OUTPUT_STEM_LENGTH = 48; + +/** + * Build a self-describing output filename for a policy artifact. + * + * These tools used to write fixed names (`clip.mp4`, `downsampled.jpg`, + * `transcript.txt`). Under fixed-policy orchestration that is safe: every + * invocation gets its own staging directory. But `modelAccess` lets a + * caller pick a PERSISTENT `outputDir`, and there two calls collide — the + * second silently destroys the first artifact. Observed in a real + * multi-session run: a clip cut on day one was overwritten by a different + * clip on day three, and the commentary written against the first clip + * silently began describing the wrong footage. + * + * The name carries the two axes that actually distinguish artifacts: the + * SOURCE it came from, and — where the operation has one — a natural + * VARIANT (a clip's time range, a frame's index). Same source and same + * variant deliberately resolve to the same name: re-running one operation + * replaces its own output with identical bytes, which is idempotent + * rather than destructive. + * + * Residual case, documented rather than defended against: one operation + * run twice on one source with DIFFERENT tuning and no natural variant — + * two downscales at different heights — still resolves to one name and + * the later result supersedes. That is an operation superseding itself, + * not one artifact destroying an unrelated one. + */ +export function policyOutputFileName(params: { + /** Source the artifact was derived from. */ + inputPath: string; + /** Operation label, e.g. 'clip', 'audio', 'keyframe'. */ + operation: string; + /** Distinguishing detail within the operation, e.g. '90s+75s', '0007'. */ + variant?: string; + /** Extension WITH the leading dot, e.g. '.mp4'. */ + extension: string; +}): string { + const raw = path.basename(params.inputPath, path.extname(params.inputPath)); + // Collapse anything outside a conservative portable set (`+` is kept: + // it is portable everywhere and appears in this scheme's own variants, + // e.g. a clip's `123s+40s`, so derived artifacts of a clip keep a stem + // that still round-trips). The stem comes + // from user-supplied media names (spaces, quotes, CJK, shell + // metacharacters) and is about to become a real path component. + const stem = + raw + .replace(/[^A-Za-z0-9._+-]+/g, '-') + .replace(/^-+|-+$/g, '') + .slice(0, MAX_OUTPUT_STEM_LENGTH) || 'media'; + const variant = params.variant ? `-${params.variant}` : ''; + return `${stem}-${params.operation}${variant}${params.extension}`; +} + /** Shared structural validation for the io params (schema has already - * checked types/required-ness). Returns an error message or null. */ + * checked types/required-ness). Returns an error message or null. + * `inputPath` is checked for presence here rather than in the schema's + * `required` list: the model-facing alternative is `resourceId`, which + * the call gate resolves into `inputPath` BEFORE validation — so a + * missing inputPath at this point means the caller supplied neither. */ export function validateMediaPolicyIoParams( params: MediaPolicyIoParams, ): string | null { + if ((params.inputPath as string | undefined) === undefined) { + return ( + 'provide exactly one of inputPath (absolute path) or resourceId ' + + '(opaque session media handle)' + ); + } if (!path.isAbsolute(params.inputPath)) { return `inputPath must be an absolute path (got ${JSON.stringify(params.inputPath)})`; } @@ -256,6 +338,12 @@ export function validateMediaPolicyIoParams( * REGULAR file (lstat — a symlink is refused, the tool must never read * through a link planted in its input position) and the output directory * an existing real directory. Returns the input size in bytes. + * + * Input errors name only the file's basename: these messages reach the + * model, and a resourceId-resolved call must not leak the real locator + * the handle stands in for (M §5.2) — the basename matches the + * displayName the model already saw at delivery. `outputDir` errors keep + * the full path (the caller chose it). */ export async function assertMediaPolicyIo( params: MediaPolicyIoParams, @@ -264,10 +352,12 @@ export async function assertMediaPolicyIo( try { inputStat = await fs.lstat(params.inputPath); } catch { - throw new Error(`input file not found: ${params.inputPath}`); + throw new Error(`input file not found: ${path.basename(params.inputPath)}`); } if (!inputStat.isFile()) { - throw new Error(`input is not a regular file: ${params.inputPath}`); + throw new Error( + `input is not a regular file: ${path.basename(params.inputPath)}`, + ); } let outStat; try { diff --git a/packages/core/src/omni/policy/tools/transcribe-audio.test.ts b/packages/core/src/omni/policy/tools/transcribe-audio.test.ts index 18aa9540c3d..9360806c174 100644 --- a/packages/core/src/omni/policy/tools/transcribe-audio.test.ts +++ b/packages/core/src/omni/policy/tools/transcribe-audio.test.ts @@ -261,14 +261,14 @@ describe('OmniTranscribeAudioTool', () => { kind: 'file', storage: 'workspace', title: 'Audio transcript', - workspacePath: 'transcript.txt', + workspacePath: 'speech-transcript.txt', mimeType: 'text/plain', sizeBytes: Buffer.byteLength('你好，世界', 'utf-8'), metadata: { omniDisclosure: disclosure, omniRole: 'transcript' }, }, ]); await expect( - fs.readFile(path.join(outputDir, 'transcript.txt'), 'utf-8'), + fs.readFile(path.join(outputDir, 'speech-transcript.txt'), 'utf-8'), ).resolves.toBe('你好，世界'); }); @@ -390,7 +390,7 @@ describe('OmniTranscribeAudioTool', () => { expect(result.error).toBeUndefined(); await expect( - fs.readFile(path.join(outputDir, 'transcript.txt'), 'utf-8'), + fs.readFile(path.join(outputDir, 'speech-transcript.txt'), 'utf-8'), ).resolves.toBe('你好。再见！'); expect(result.artifacts?.[0]?.metadata?.['omniDisclosure']).toBe( '原 63s 音频 → 转写文本 6 字，检测到重复退化已截断，语气/音色/非语音信息丢失，识别可能有误', @@ -469,7 +469,7 @@ describe('OmniTranscribeAudioTool', () => { expect(result.error).toBeUndefined(); await expect( - fs.readFile(path.join(outputDir, 'transcript.txt'), 'utf-8'), + fs.readFile(path.join(outputDir, 'speech-transcript.txt'), 'utf-8'), ).resolves.toBe( '[00:00-02:13] 片段@0.000\n' + '[02:13-04:27] 片段@133.333\n' + @@ -480,7 +480,9 @@ describe('OmniTranscribeAudioTool', () => { expect(disclosure).not.toContain('段失败'); // Temporary chunk cuts are cleaned up; only the transcript remains. - await expect(fs.readdir(outputDir)).resolves.toEqual(['transcript.txt']); + await expect(fs.readdir(outputDir)).resolves.toEqual([ + 'speech-transcript.txt', + ]); }); it('uses H:MM:SS ranges for audio of an hour or longer', async () => { @@ -490,7 +492,7 @@ describe('OmniTranscribeAudioTool', () => { expect(result.error).toBeUndefined(); const transcript = await fs.readFile( - path.join(outputDir, 'transcript.txt'), + path.join(outputDir, 'speech-transcript.txt'), 'utf-8', ); expect(transcript).toContain('[0:00:00-0:02:54] 对白'); @@ -524,7 +526,7 @@ describe('OmniTranscribeAudioTool', () => { expect(result.error).toBeUndefined(); const transcript = await fs.readFile( - path.join(outputDir, 'transcript.txt'), + path.join(outputDir, 'speech-transcript.txt'), 'utf-8', ); expect(transcript).toContain('[02:13-04:27] （该段转写失败：HTTP 500）'); @@ -552,7 +554,7 @@ describe('OmniTranscribeAudioTool', () => { expect(result.error).toBeUndefined(); const transcript = await fs.readFile( - path.join(outputDir, 'transcript.txt'), + path.join(outputDir, 'speech-transcript.txt'), 'utf-8', ); expect(transcript).toContain('大家好。再见！'); @@ -584,7 +586,7 @@ describe('OmniTranscribeAudioTool', () => { expect(mocks.runFfmpeg).toHaveBeenCalledTimes(3); expect(result.error).toBeUndefined(); const transcript = await fs.readFile( - path.join(outputDir, 'transcript.txt'), + path.join(outputDir, 'speech-transcript.txt'), 'utf-8', ); expect(transcript).toContain( @@ -607,7 +609,7 @@ describe('OmniTranscribeAudioTool', () => { expect(result.error).toBeUndefined(); const transcript = await fs.readFile( - path.join(outputDir, 'transcript.txt'), + path.join(outputDir, 'speech-transcript.txt'), 'utf-8', ); expect(transcript).toContain( diff --git a/packages/core/src/omni/policy/tools/transcribe-audio.ts b/packages/core/src/omni/policy/tools/transcribe-audio.ts index 5cae78d493a..46b1e526af6 100644 --- a/packages/core/src/omni/policy/tools/transcribe-audio.ts +++ b/packages/core/src/omni/policy/tools/transcribe-audio.ts @@ -24,6 +24,7 @@ import { mediaPolicyToolError, mediaPolicyToolFailure, mediaPolicyToolSuccess, + policyOutputFileName, resolvePolicyToolSettings, resolvePolicyToolTimeoutMs, type MediaPolicyIoParams, @@ -49,7 +50,6 @@ export const TRANSCRIBE_AUDIO_DEFAULTS = { } as const; /** Output file the transcript artifact is written to (staging-relative). */ -const OUTPUT_FILE_NAME = 'transcript.txt'; /** How many chunk transcription requests run concurrently. */ const CHUNK_CONCURRENCY = 3; @@ -426,7 +426,14 @@ class TranscribeAudioInvocation extends BaseMediaPolicyToolInvocation vi.fn()); +vi.mock('../core/nonInteractiveToolExecutor.js', () => ({ + executeToolCall: executeToolCallMock, +})); + +// Partial mock: recognizeMediaFile would need ffprobe; hashFileSha256 and +// extensionForMime stay real (the store re-hashes for real). +const recognizeMediaFileMock = vi.hoisted(() => vi.fn()); +vi.mock('./recognition.js', async (importOriginal) => ({ + ...(await importOriginal()), + recognizeMediaFile: recognizeMediaFileMock, +})); + +const uploadFileMock = vi.hoisted(() => vi.fn()); +vi.mock('./upload.js', async (importOriginal) => ({ + ...(await importOriginal()), + DashScopeUploader: class { + uploadFile = uploadFileMock; + }, +})); + afterEach(() => { resetObservedServerInputLimitsForTests(); }); @@ -302,3 +346,250 @@ describe('observed server input limits', () => { expect(getObservedServerInputLimit('m')).toBeUndefined(); }); }); + +describe('degradeOmniMediaAfterServerReject memory wiring (S5)', () => { + const MODEL = 'qwen3-omni-plus'; + const BASE_URL = 'https://dashscope.aliyuncs.com/compatible-mode/v1'; + const API_KEY = 'test-key'; + const SOURCE_BYTES = 'original-video-bytes'; + const DEGRADED_BYTES = 'reactively-degraded-video-bytes'; + const OSS_URL = 'oss://bucket/clip'; + + const SOURCE_RECOGNIZED: RecognizedMedia = { + modality: 'video', + detectedMimeType: 'video/mp4', + sizeBytes: SOURCE_BYTES.length, + metadata: { durationMs: 60_000, width: 1920, height: 1080 }, + }; + const DEGRADED_RECOGNIZED: RecognizedMedia = { + modality: 'video', + detectedMimeType: 'video/mp4', + sizeBytes: DEGRADED_BYTES.length, + metadata: { durationMs: 60_000, width: 640, height: 360 }, + }; + const VIDEO_DESCRIPTOR: MediaPolicyToolDescriptor = { + kind: 'media_policy', + inputMediaTypes: ['video'], + outputs: [ + { kind: 'media', mimeTypes: ['video/mp4'], required: true, lossy: true }, + ], + }; + const LIMITS: NormalizedOmniProcessingLimits = { + maxConcurrentResources: 1, + reservedOutputTokens: 8192, + maxLineageDepth: 8, + maxPolicyRunsPerRoot: 64, + maxArtifactsPerRoot: 256, + maxDerivedBytesPerRoot: 1073741824, + maxTransportPasses: 3, + }; + + let tmpDir: string; + let store: OmniObjectStore; + let sourceSha256: string; + let objectPath: string; + + function sha256Of(text: string): string { + return createHash('sha256').update(text).digest('hex'); + } + + function degradeConfig(): Config { + return { + isOmniEnabled: () => true, + isTrustedFolder: () => true, + getContentGeneratorConfig: () => ({ apiKey: API_KEY, baseUrl: BASE_URL }), + getModel: () => MODEL, + storage: { getQwenDir: () => path.join(tmpDir, '.qwen') }, + getOmniProcessingConfig: () => ({ + transportGuardPolicies: [videoGuardPolicy()], + limits: LIMITS, + }), + getOmniMemoryConfig: () => DEFAULT_OMNI_MEMORY_CONFIG, + getToolRegistry: () => ({ + getTool: (name: string) => + name === ToolNames.OMNI_DOWNSCALE_VIDEO + ? { mediaPolicyDescriptor: VIDEO_DESCRIPTOR } + : undefined, + }), + getOmniPolicyToolsSettings: () => undefined, + } as unknown as Config; + } + + function rejectedContents(): Content[] { + return [ + { + role: 'user', + parts: [ + { text: 'summarize this' }, + { + fileData: { + fileUri: OSS_URL, + mimeType: 'video/mp4', + displayName: 'movie.mkv', + }, + }, + ], + }, + ]; + } + + async function readSnapshot(): Promise { + return JSON.parse( + await fs.readFile( + path.join(store.getOmniRootDir(), MEDIA_MEMORY_FILE_NAME), + 'utf8', + ), + ) as MediaMemorySnapshot; + } + + /** Record a video file into memory the way the delivery pipeline does, + * returning the binding the ladder is supposed to rediscover by hash. */ + async function recordInMemory(fileRef: string, sha256: string) { + const service = new MediaMemoryService(store.getOmniRootDir()); + const binding = await service.recordFileRecognized({ + fileRef, + sha256, + mediaType: 'video', + metadata: SOURCE_RECOGNIZED.metadata, + sizeBytes: SOURCE_RECOGNIZED.sizeBytes, + mimeType: 'video/mp4', + origin: 'user', + source: { protocol: 'local', locator: 'movie.mkv' }, + recognition: { + ingestionConfigHash: '', + detectorVersion: 'omni-sniff-ffprobe/1', + probeStatus: 'complete', + }, + }); + return binding!; + } + + beforeEach(async () => { + vi.clearAllMocks(); + tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'omni-reactive-')); + store = new OmniObjectStore(path.join(tmpDir, '.qwen')); + // The object the rejected oss:// URL maps back to, reachable through + // the upload cache's reverse lookup exactly as in production. + const stagedSource = path.join(tmpDir, 'movie.mkv'); + await fs.writeFile(stagedSource, SOURCE_BYTES); + sourceSha256 = sha256Of(SOURCE_BYTES); + objectPath = (await store.putFile(stagedSource, sourceSha256, '.mp4')) + .objectPath; + const scope = createHash('sha256') + .update(`${BASE_URL}|${API_KEY}`) + .digest('hex') + .slice(0, 16); + await new OmniUploadCache(store.getOmniRootDir(), undefined, scope).put( + sourceSha256, + MODEL, + OSS_URL, + ); + + recognizeMediaFileMock.mockImplementation(async (filePath: string) => + filePath === objectPath ? SOURCE_RECOGNIZED : DEGRADED_RECOGNIZED, + ); + executeToolCallMock.mockImplementation( + async (_config: Config, request: ToolCallRequestInfo) => { + const outputDir = request.args['outputDir'] as string; + await fs.writeFile(path.join(outputDir, 'out.mp4'), DEGRADED_BYTES); + return { + callId: request.callId, + responseParts: [], + resultDisplay: undefined, + error: undefined, + errorType: undefined, + policyArtifacts: { + toolName: request.name, + invocationId: request.callId, + executionOrigin: request.executionOrigin, + artifacts: [ + { + kind: 'video', + storage: 'workspace', + title: 'out.mp4', + workspacePath: 'out.mp4', + mimeType: 'video/mp4', + metadata: { omniDisclosure: 'Downscaled to 360p/0.5fps.' }, + }, + ], + }, + }; + }, + ); + uploadFileMock.mockResolvedValue('oss://bucket/clip-rung-1'); + }); + + afterEach(async () => { + await fs.rm(tmpDir, { recursive: true, force: true }); + }); + + it('commits the re-derivation onto the binding found by content hash', async () => { + // The ladder only ever holds an oss:// URL and the bytes behind it — + // never a memory identity. Losing the sha256 → binding lookup does not + // break the retry, so nothing goes red: the request is degraded, sent, + // and accepted, while the derivative silently becomes a rootless + // orphan. Recall would then report the user's video as never having + // been degraded, and the next rung would re-transcode from scratch. + const rootBinding = await recordInMemory(objectPath, sourceSha256); + const contents = rejectedContents(); + + const outcome = await degradeOmniMediaAfterServerReject( + degradeConfig(), + contents, + 1, + ); + expect(outcome).toEqual({ replacedParts: 1, degradedResources: 1 }); + + const snapshot = await readSnapshot(); + const executions = Object.values(snapshot.executions); + expect(executions).toHaveLength(1); + expect(executions[0]).toMatchObject({ + sourceVersionId: rootBinding.fileVersionId, + rootFileId: rootBinding.rootFileId, + toolName: ToolNames.OMNI_DOWNSCALE_VIDEO, + // The rung's escalated arguments are what actually ran, so they are + // what the record must carry — rung 1 of the video ladder. + finalArguments: { maxHeight: 360, fps: 0.5 }, + executionOrigin: { + kind: 'fixed_policy', + policyId: 'video-downscale.reactive-1', + stage: 'transport_guard', + }, + }); + + const derivedVersion = Object.values(snapshot.versions).find( + (version) => version.sha256 === sha256Of(DEGRADED_BYTES), + ); + expect(derivedVersion).toBeDefined(); + expect(derivedVersion!.parentVersionId).toBe(rootBinding.fileVersionId); + expect(snapshot.files[derivedVersion!.fileId].rootFileId).toBe( + rootBinding.rootFileId, + ); + }); + + it('degrades normally when memory has never seen the bytes', async () => { + // Memory is best-effort here: a session that degraded media before + // memory was switched on (or after the store was wiped) still has to + // retry. And the commit must not fall back to SOME other binding — + // attaching this derivative to an unrelated lineage would be a + // fabricated provenance claim, which is worse than no record. + await recordInMemory(path.join(tmpDir, 'other.mkv'), sha256Of('unrelated')); + const contents = rejectedContents(); + + const outcome = await degradeOmniMediaAfterServerReject( + degradeConfig(), + contents, + 1, + ); + expect(outcome).toEqual({ replacedParts: 1, degradedResources: 1 }); + expect(contents[0].parts![2].fileData?.fileUri).toBe( + 'oss://bucket/clip-rung-1', + ); + + const snapshot = await readSnapshot(); + expect(Object.values(snapshot.executions)).toEqual([]); + expect( + Object.values(snapshot.versions).map((version) => version.sha256), + ).toEqual([sha256Of('unrelated')]); + }); +}); diff --git a/packages/core/src/omni/reactive-degrade.ts b/packages/core/src/omni/reactive-degrade.ts index 8a41dd92254..7a0561ce63e 100644 --- a/packages/core/src/omni/reactive-degrade.ts +++ b/packages/core/src/omni/reactive-degrade.ts @@ -51,6 +51,7 @@ import { } from './recognition.js'; import { runFixedPolicies } from './policy/orchestrator.js'; import type { NormalizedFixedPolicy } from './policy/types.js'; +import { MediaMemoryService } from '../services/media-memory/index.js'; import { formatDisclosureText } from './disclosure.js'; import { isOmniDeliveryActive } from './delivery-gate.js'; @@ -358,6 +359,16 @@ export async function degradeOmniMediaAfterServerReject( config.getOmniUploadUrlTtlHours?.() ?? DEFAULT_UPLOAD_CACHE_TTL_HOURS, cacheScope, ); + // Media-memory collection (S5): the ladder re-derives from a stored + // object whose memory identity — if any — is only reachable through + // its content hash. Best-effort like everything else here: an absent + // config, a missing binding, or a failed lookup simply skips memory. + const memoryConfig = config.getOmniMemoryConfig?.(); + const memoryService = memoryConfig + ? new MediaMemoryService(store.getOmniRootDir(), { + maxInlineTextBytes: memoryConfig.collection.maxInlineTextBytes, + }) + : undefined; // old fileUri → replacement delivery. const replacements = new Map(); @@ -389,6 +400,9 @@ export async function degradeOmniMediaAfterServerReject( recognized.modality, attempt, ); + const sourceBinding = memoryService + ? await memoryService.findBindingBySha256(sha256) + : undefined; const { deliveries } = await runFixedPolicies( config, { @@ -396,12 +410,17 @@ export async function degradeOmniMediaAfterServerReject( recognized, displayName: ref.displayName, origin: 'user', + sha256, }, { store, policies: [policy], signal, limits: processingConfig.limits, + memory: + memoryService && sourceBinding + ? { service: memoryService, sourceBinding } + : undefined, }, ); const delivery = deliveries[0]; diff --git a/packages/core/src/omni/recall-media-memory-tool.test.ts b/packages/core/src/omni/recall-media-memory-tool.test.ts new file mode 100644 index 00000000000..585e5b07992 --- /dev/null +++ b/packages/core/src/omni/recall-media-memory-tool.test.ts @@ -0,0 +1,383 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import type { Config } from '../config/config.js'; +import { ToolNames } from '../tools/tool-names.js'; +import { + DEFAULT_OMNI_MEMORY_CONFIG, + MediaMemoryService, + MediaResourceRegistry, +} from '../services/media-memory/index.js'; +import { + buildMediaMemoryRecallAdvisor, + reanchorRememberedMedia, +} from './memory-recall.js'; +import { OmniRecallMediaMemoryTool } from './recall-media-memory-tool.js'; + +describe('OmniRecallMediaMemoryTool', () => { + let tmpDir: string; + let registry: MediaResourceRegistry; + + beforeEach(async () => { + tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'omni-recall-tool-')); + registry = new MediaResourceRegistry(); + }); + + afterEach(async () => { + await fs.rm(tmpDir, { recursive: true, force: true }); + }); + + function toolConfig(overrides?: Partial>): Config { + return { + getOmniMemoryConfig: () => DEFAULT_OMNI_MEMORY_CONFIG, + getOmniMediaResourceRegistry: () => registry, + storage: { getQwenDir: () => tmpDir }, + getToolRegistry: () => ({ getTool: () => undefined }), + getOmniPolicyToolsSettings: () => undefined, + ...overrides, + } as unknown as Config; + } + + /** Record one image file into the persistent store and bind its session + * handle, mirroring what a delivery does. */ + async function recordAndBind(): Promise { + const memory = new MediaMemoryService(path.join(tmpDir, 'omni')); + const binding = await memory.recordFileRecognized({ + fileRef: path.join(tmpDir, 'pic.png'), + sha256: 'a'.repeat(64), + mediaType: 'image', + metadata: { width: 32, height: 32 }, + sizeBytes: 1234, + mimeType: 'image/png', + origin: 'user', + source: { protocol: 'local', locator: 'pic.png' }, + recognition: { + ingestionConfigHash: '', + detectorVersion: 'omni-sniff-ffprobe/1', + probeStatus: 'complete', + }, + }); + expect(binding).toBeDefined(); + return registry.bind({ + ...binding!, + fileRef: path.join(tmpDir, 'pic.png'), + mediaType: 'image', + }).resourceId; + } + + it('rejects a request naming more handles than maxFilesPerCall', () => { + const tool = new OmniRecallMediaMemoryTool(toolConfig()); + const max = DEFAULT_OMNI_MEMORY_CONFIG.recall.active.maxFilesPerCall; + const resourceIds = Array.from({ length: max + 1 }, (_, i) => `m-${i}`); + expect(() => tool.build({ resourceIds, query: 'q' })).toThrow( + /maxFilesPerCall/, + ); + }); + + it('rejects an empty resourceIds list at the schema layer', () => { + const tool = new OmniRecallMediaMemoryTool(toolConfig()); + expect(() => tool.build({ resourceIds: [], query: 'q' })).toThrow(); + }); + + it('returns invalid_tool_params for a handle this session never issued', async () => { + const tool = new OmniRecallMediaMemoryTool(toolConfig()); + const invocation = tool.build({ + resourceIds: ['media-99-deadbeef'], + query: 'anything', + }); + const result = await invocation.execute(new AbortController().signal); + expect(result.error?.type).toBe('invalid_tool_params'); + expect(result.llmContent).toContain('unknown_resource'); + }); + + it('recalls the recorded metadata entry for a bound handle', async () => { + const resourceId = await recordAndBind(); + const tool = new OmniRecallMediaMemoryTool(toolConfig()); + const invocation = tool.build({ + resourceIds: [resourceId], + query: 'image dimensions', + }); + const result = await invocation.execute(new AbortController().signal); + expect(result.error).toBeUndefined(); + const payload = JSON.parse(result.llmContent as string); + expect(payload.files).toHaveLength(1); + expect(payload.files[0].current).toBe(true); + expect( + payload.entries.some( + (e: { kind: string; content?: string }) => + e.kind === 'metadata' && e.content?.includes('"width":32'), + ), + ).toBe(true); + // No real path anywhere in the model-visible payload (M §5.2). + expect(result.llmContent as string).not.toContain(tmpDir); + }); + + it('degrades to a plain miss when the store has never been written', async () => { + const resourceId = registry.bind({ + fileId: 'f1', + fileVersionId: 'v1', + rootFileId: 'f1', + fileRef: path.join(tmpDir, 'ghost.png'), + mediaType: 'image', + }).resourceId; + const tool = new OmniRecallMediaMemoryTool(toolConfig()); + const invocation = tool.build({ resourceIds: [resourceId], query: 'q' }); + const result = await invocation.execute(new AbortController().signal); + expect(result.error).toBeUndefined(); + expect(JSON.parse(result.llmContent as string).status).toBe('miss'); + }); + + it('reports media memory unavailable on a config without memory', async () => { + const tool = new OmniRecallMediaMemoryTool( + toolConfig({ getOmniMemoryConfig: () => undefined }), + ); + const invocation = tool.build({ resourceIds: ['media-1-ab'], query: 'q' }); + const result = await invocation.execute(new AbortController().signal); + expect(result.error?.type).toBe('execution_failed'); + }); +}); + +describe('reanchorRememberedMedia', () => { + let tmpDir: string; + let registry: MediaResourceRegistry; + + beforeEach(async () => { + tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'omni-reanchor-')); + registry = new MediaResourceRegistry(); + }); + + afterEach(async () => { + await fs.rm(tmpDir, { recursive: true, force: true }); + }); + + const cfg = (overrides?: Record): Config => + ({ + storage: { getQwenDir: () => tmpDir }, + getOmniMemoryConfig: () => DEFAULT_OMNI_MEMORY_CONFIG, + getOmniMediaResourceRegistry: () => registry, + ...overrides, + }) as unknown as Config; + + /** Record a file, then delete its bytes. */ + async function rememberThenDelete(): Promise { + const filePath = path.join(tmpDir, 'gone.mkv'); + await fs.writeFile(filePath, 'bytes'); + const memory = new MediaMemoryService(path.join(tmpDir, 'omni')); + await memory.recordFileRecognized({ + fileRef: filePath, + sha256: 'f'.repeat(64), + mediaType: 'video', + metadata: { durationMs: 1000 }, + sizeBytes: 5, + mimeType: 'video/x-matroska', + origin: 'user', + source: { protocol: 'local', locator: 'gone.mkv' }, + recognition: { + ingestionConfigHash: '', + detectorVersion: 'omni-sniff-ffprobe/1', + probeStatus: 'complete', + }, + }); + await fs.rm(filePath); + return filePath; + } + + it('mints a handle for a remembered file whose bytes are gone', async () => { + const filePath = await rememberThenDelete(); + + const anchored = await reanchorRememberedMedia(cfg(), filePath); + + expect(anchored).toBeDefined(); + // The handle resolves like any delivered one, so recall accepts it — + // which is the whole point: the memory of a deleted file stays + // reachable instead of being stranded forever. + const binding = registry.resolve(anchored!.resourceId); + expect(binding).toMatchObject({ fileRef: filePath, mediaType: 'video' }); + expect(anchored!.annotation).toContain('【媒体资源】gone.mkv：'); + expect(anchored!.annotation).toContain(anchored!.resourceId); + // Never the real path — only the basename the user already typed. + expect(anchored!.annotation).not.toContain(tmpDir); + }); + + it('returns undefined for a path memory has never seen', async () => { + await rememberThenDelete(); + await expect( + reanchorRememberedMedia(cfg(), path.join(tmpDir, 'stranger.mkv')), + ).resolves.toBeUndefined(); + }); + + it('returns undefined when memory is not configured', async () => { + const filePath = await rememberThenDelete(); + await expect( + reanchorRememberedMedia( + cfg({ getOmniMemoryConfig: () => undefined }), + filePath, + ), + ).resolves.toBeUndefined(); + }); +}); + +describe('buildMediaMemoryRecallAdvisor', () => { + function advisorConfig(params: { + registered: string[]; + enabled: string[]; + }): Config { + return { + getToolRegistry: () => ({ + getTool: (name: string) => + params.registered.includes(name) ? { name } : undefined, + }), + getOmniPolicyToolsSettings: () => + Object.fromEntries( + params.enabled.map((name) => [ + name, + { modelAccess: { enabled: true } }, + ]), + ), + } as unknown as Config; + } + + const gap = (channels: string[], reason = 'not_processed') => + ({ scope: {}, channels, reason }) as never; + + it('suggests only registered, model-accessible tools', () => { + const advise = buildMediaMemoryRecallAdvisor( + advisorConfig({ + registered: [ + ToolNames.OMNI_EXTRACT_KEYFRAMES, + ToolNames.OMNI_EXTRACT_AUDIO, + ], + enabled: [ToolNames.OMNI_EXTRACT_KEYFRAMES], + }), + ); + const actions = advise({ + resourceId: 'media-1-ab', + mediaType: 'video', + gap: gap(['visual', 'speech_text']), + }); + // extract-audio is registered but not opened to the model; the + // advisor must not steer the model into a gated call. + expect(actions).toEqual([ + { + toolName: ToolNames.OMNI_EXTRACT_KEYFRAMES, + resourceId: 'media-1-ab', + arguments: {}, + reason: expect.stringContaining('keyframes'), + }, + ]); + }); + + it('never suggests a tool that is not registered in this session', () => { + // modelAccess settings say what the operator ALLOWS; the tool registry + // says what actually exists this turn (omni tools are absent when omni + // is off, when ffmpeg is missing, or under a tool filter). Advising an + // unregistered tool spends the model's next turn on a call that comes + // back "tool not found" — a dead end recall itself invented. + const advise = buildMediaMemoryRecallAdvisor( + advisorConfig({ + registered: [], + enabled: [ToolNames.OMNI_EXTRACT_KEYFRAMES], + }), + ); + expect( + advise({ + resourceId: 'media-6-ab', + mediaType: 'video', + gap: gap(['visual']), + }), + ).toEqual([]); + }); + + it('suggests transcription for an audio speech_text gap', () => { + const advise = buildMediaMemoryRecallAdvisor( + advisorConfig({ + registered: [ToolNames.OMNI_TRANSCRIBE_AUDIO], + enabled: [ToolNames.OMNI_TRANSCRIBE_AUDIO], + }), + ); + const actions = advise({ + resourceId: 'media-2-cd', + mediaType: 'audio', + gap: gap(['speech_text']), + }); + expect(actions.map((a) => a.toolName)).toEqual([ + ToolNames.OMNI_TRANSCRIBE_AUDIO, + ]); + }); + + it('does not re-suggest audio extraction once the track exists', () => { + // The payload that RETURNS the extracted audio still reports the + // video's speech_text channel as open. Matching that channel made the + // advisor suggest extracting the track again in the very same payload; + // the model is supposed to chain to transcription instead. + const advise = buildMediaMemoryRecallAdvisor( + advisorConfig({ + registered: [ + ToolNames.OMNI_EXTRACT_AUDIO, + ToolNames.OMNI_TRANSCRIBE_AUDIO, + ], + enabled: [ + ToolNames.OMNI_EXTRACT_AUDIO, + ToolNames.OMNI_TRANSCRIBE_AUDIO, + ], + }), + ); + expect( + advise({ + resourceId: 'media-4-aa', + mediaType: 'video', + gap: gap(['speech_text']), + }), + ).toEqual([]); + // A wholly unprocessed video still gets the extraction step. + expect( + advise({ + resourceId: 'media-4-aa', + mediaType: 'video', + gap: gap(['acoustic', 'speech_text']), + }).map((a) => a.toolName), + ).toEqual([ToolNames.OMNI_EXTRACT_AUDIO]); + }); + + it('never suggests work that cannot close a sampled-coverage gap', () => { + // Keyframes deliberately never claim complete visual coverage, so + // suggesting keyframe extraction against `partial_coverage` would + // advise the same step forever. + const advise = buildMediaMemoryRecallAdvisor( + advisorConfig({ + registered: [ToolNames.OMNI_EXTRACT_KEYFRAMES], + enabled: [ToolNames.OMNI_EXTRACT_KEYFRAMES], + }), + ); + expect( + advise({ + resourceId: 'media-5-bb', + mediaType: 'video', + gap: gap(['visual'], 'partial_coverage'), + }), + ).toEqual([]); + }); + + it('never suggests anything for an unavailable artifact', () => { + const advise = buildMediaMemoryRecallAdvisor( + advisorConfig({ + registered: [ToolNames.OMNI_EXTRACT_KEYFRAMES], + enabled: [ToolNames.OMNI_EXTRACT_KEYFRAMES], + }), + ); + expect( + advise({ + resourceId: 'media-3-ef', + mediaType: 'video', + gap: gap(['visual'], 'artifact_unavailable'), + }), + ).toEqual([]); + }); +}); diff --git a/packages/core/src/omni/recall-media-memory-tool.ts b/packages/core/src/omni/recall-media-memory-tool.ts new file mode 100644 index 00000000000..51a8e6fc984 --- /dev/null +++ b/packages/core/src/omni/recall-media-memory-tool.ts @@ -0,0 +1,227 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import type { Config } from '../config/config.js'; +import { + BaseDeclarativeTool, + BaseToolInvocation, + Kind, +} from '../tools/tools.js'; +import type { ToolInvocation, ToolResult } from '../tools/tools.js'; +import { ToolErrorType } from '../tools/tool-error.js'; +import { ToolDisplayNames, ToolNames } from '../tools/tool-names.js'; +import { + MediaMemoryRecallRejection, + OMNI_MEMORY_RECALL_KINDS, + type OmniMemoryRecallKind, +} from '../services/media-memory/index.js'; +import { createMediaMemoryRecallService } from './memory-recall.js'; + +/** + * Active-mode recall surface (M §9, D10): registered ONLY when + * `omni.memory.recall.mode === 'active'` — the sideQuery selector never + * runs in that mode, and vice versa. Read-only by constitution (D11): + * the tool consults persistent media memory and binds session handles; + * it never writes a record. + */ + +export interface OmniRecallMediaMemoryParams { + /** Session resource handles (from 【媒体资源】 annotations). */ + resourceIds: string[]; + query: string; + kinds?: OmniMemoryRecallKind[]; + roles?: string[]; + includeHistoricalVersions?: boolean; + limit?: number; +} + +class OmniRecallMediaMemoryInvocation extends BaseToolInvocation< + OmniRecallMediaMemoryParams, + ToolResult +> { + constructor( + private readonly config: Config, + params: OmniRecallMediaMemoryParams, + ) { + super(params); + } + + getDescription(): string { + const handles = this.params.resourceIds.join(', '); + return `Recall media memory for ${handles}`; + } + + async execute(_signal: AbortSignal): Promise { + const service = createMediaMemoryRecallService(this.config); + if (!service) { + // Defensive: registration is gated on the normalized memory config, + // so a missing config here means a stub/uninitialized embedding. + return { + llmContent: + 'Media memory is not available in this session (omni memory is ' + + 'not configured).', + returnDisplay: 'Media memory unavailable', + error: { + message: 'omni memory is not configured', + type: ToolErrorType.EXECUTION_FAILED, + }, + }; + } + try { + const result = await service.recall({ + resourceIds: this.params.resourceIds, + query: this.params.query, + ...(this.params.kinds !== undefined + ? { kinds: this.params.kinds } + : {}), + ...(this.params.roles !== undefined + ? { roles: this.params.roles } + : {}), + ...(this.params.includeHistoricalVersions !== undefined + ? { + includeHistoricalVersions: this.params.includeHistoricalVersions, + } + : {}), + ...(this.params.limit !== undefined + ? { limit: this.params.limit } + : {}), + }); + return { + llmContent: JSON.stringify(result, null, 1), + returnDisplay: + `Recall ${result.status}: ${result.entries.length} entr${ + result.entries.length === 1 ? 'y' : 'ies' + }, ${result.gaps.length} gap${result.gaps.length === 1 ? '' : 's'}` + + (result.nextPolicyActions?.length + ? `, ${result.nextPolicyActions.length} suggested action${ + result.nextPolicyActions.length === 1 ? '' : 's' + }` + : ''), + }; + } catch (err) { + if (err instanceof MediaMemoryRecallRejection) { + // Whole-request rejection (M §9.2): the request itself is invalid + // — a parameter-level error the model can correct and retry. + return { + llmContent: `Recall request rejected (${err.reason}): ${err.message}`, + returnDisplay: `Recall rejected: ${err.reason}`, + error: { + message: err.message, + type: ToolErrorType.INVALID_TOOL_PARAMS, + }, + }; + } + throw err; + } + } +} + +export class OmniRecallMediaMemoryTool extends BaseDeclarativeTool< + OmniRecallMediaMemoryParams, + ToolResult +> { + static readonly Name = ToolNames.OMNI_RECALL_MEDIA_MEMORY; + + constructor(private readonly config: Config) { + super( + OmniRecallMediaMemoryTool.Name, + ToolDisplayNames.OMNI_RECALL_MEDIA_MEMORY, + 'Recalls what is already known about media resources delivered in ' + + 'this session: prior transcripts, extracted keyframes, technical ' + + 'metadata, and processing history persisted by earlier sessions. ' + + 'Pass the opaque resourceId handles announced in 【媒体资源】 ' + + 'annotations next to delivered media (handles from recall results ' + + 'work too). Returns matching entries plus honest gaps — channels ' + + 'never processed or artifacts no longer available — and may ' + + 'suggest follow-up tool calls to gather missing evidence. Use ' + + 'this BEFORE reprocessing media: a transcript or keyframe set ' + + 'that already exists is returned instantly.', + Kind.Read, + { + type: 'object', + properties: { + resourceIds: { + type: 'array', + items: { type: 'string', minLength: 1, maxLength: 256 }, + minItems: 1, + description: + 'Session resource handles to consult (from 【媒体资源】 ' + + 'annotations or prior recall results). Unknown handles ' + + 'reject the whole request.', + }, + query: { + type: 'string', + minLength: 1, + maxLength: 2048, + description: + 'Free-text information need; orders results by relevance.', + }, + kinds: { + type: 'array', + items: { + type: 'string', + enum: [...OMNI_MEMORY_RECALL_KINDS], + }, + // An empty list would read as "restrict to nothing" and return a + // silent miss; omit the key instead to mean "all kinds". + minItems: 1, + uniqueItems: true, + description: + 'Restrict to entry kinds (default: all configured kinds).', + }, + roles: { + type: 'array', + items: { type: 'string', minLength: 1, maxLength: 128 }, + minItems: 1, + uniqueItems: true, + description: + 'Restrict to artifact roles (e.g. "transcript", "keyframe").', + }, + includeHistoricalVersions: { + type: 'boolean', + description: + 'Also consult superseded file versions (default: only the ' + + 'current version).', + }, + limit: { + type: 'integer', + minimum: 1, + description: + 'Maximum entries to return (capped by session config).', + }, + }, + required: ['resourceIds', 'query'], + additionalProperties: false, + }, + false, // isOutputMarkdown — structured JSON payload + false, // canUpdateOutput + true, // shouldDefer — recall is an occasional lookup (matches web_fetch) + false, // alwaysLoad + 'recall media memory transcript keyframe history resource', + ); + } + + protected override validateToolParamValues( + params: OmniRecallMediaMemoryParams, + ): string | null { + const maxFiles = + this.config.getOmniMemoryConfig()?.recall.active.maxFilesPerCall; + if (maxFiles !== undefined && params.resourceIds.length > maxFiles) { + return ( + `resourceIds lists ${params.resourceIds.length} handles; at most ` + + `${maxFiles} may be consulted per call ` + + `(omni.memory.recall.active.maxFilesPerCall). Split the request.` + ); + } + return null; + } + + protected createInvocation( + params: OmniRecallMediaMemoryParams, + ): ToolInvocation { + return new OmniRecallMediaMemoryInvocation(this.config, params); + } +} diff --git a/packages/core/src/omni/tool-result-media.test.ts b/packages/core/src/omni/tool-result-media.test.ts index 58e36c779b1..36d38aa4073 100644 --- a/packages/core/src/omni/tool-result-media.test.ts +++ b/packages/core/src/omni/tool-result-media.test.ts @@ -47,6 +47,18 @@ function inlinePart(mimeType: string, bytes: Buffer): Part { return { inlineData: { mimeType, data: bytes.toString('base64') } }; } +/** Role each replacement Part plays in the group, so one assertion can pin + * the whole group's order. */ +function tagPart(part: Part): string { + if (part.fileData) return 'media'; + const text = part.text ?? ''; + if (text.startsWith('【媒体资源】')) return 'handle'; + if (text.startsWith('【媒体省略】')) return 'omission'; + if (text.startsWith('【媒体降质】')) return 'disclosure'; + if (text.startsWith('【媒体转写】')) return 'transcript'; + return 'text'; +} + // Per-run isolated qwen dir: a shared hardcoded path would leak staging // files across runs and collide between concurrent test invocations. let testQwenDir: string; @@ -199,6 +211,29 @@ describe('processToolResultOmniMedia', () => { expect(result[1]!.fileData?.fileUri).toBe('oss://bucket/key3'); }); + it('keeps the recall handle on a guard-rejected part', async () => { + // The bind happens before the guard rules, so the session already has a + // record of this media. Withholding the handle too would make the + // rejection the one path that strands a recorded resource: the model + // cannot see the bytes AND cannot ask memory about them — while the + // omission branch, whose verdict is identical, does hand the handle over. + const { OmniTransportGuardError } = await import('./guard.js'); + const rejection = new OmniTransportGuardError( + 'x.png exceeds the omni upload limit', + ); + rejection.sessionResourceId = 'media-6-ab12'; + deliverMock.mockRejectedValueOnce(rejection); + + const result = await processToolResultOmniMedia( + [inlinePart('image/png', PNG_BYTES)], + cfg({ image: true }), + signal, + ); + + expect(result[0]!.text).toContain('media-6-ab12'); + expect(result[1]!.text).toMatch(/withheld by the omni transport guard/); + }); + it('withholds the part when guard-stage PROCESSING fails (never inline the rejected bytes)', async () => { // A guard-policy execution failure arrives as OmniTransportGuardError // with the underlying error as `cause` (see processMediaForOmniDelivery's @@ -251,6 +286,75 @@ describe('processToolResultOmniMedia', () => { }); }); + // Session resource handle (M §5.2): the only identity the model ever gets + // for tool-produced media. A branch that drops it hands the model media it + // can never name again — no recall, no follow-up omni tool call, and no + // path to fall back on. The handle must LEAD the group so the disclosure + // keeps its D8 adjacency to the media part. + const RESOURCE_ID = 'media-3-c0ffee01'; + const HANDLE_TEXT = `【媒体资源】tool-media.image：${RESOURCE_ID}`; + const HANDLE_DELIVERY = { + mimeType: 'image/png', + sha256: 'a'.repeat(64), + recognized: { modality: 'image' }, + tokenEstimate: { + estimatedTokenCount: 1, + method: 'raw-resource-v1', + status: 'ok', + }, + deduped: false, + resourceId: RESOURCE_ID, + }; + + it.each([ + { + branch: 'plain upload', + delivery: { fileUri: 'oss://bucket/key' }, + tags: ['handle', 'media'], + }, + { + branch: 'degraded upload', + delivery: { + fileUri: 'oss://bucket/degraded', + disclosure: 'downsampled to 1568px', + degraded: true, + }, + tags: ['handle', 'disclosure', 'media'], + }, + { + branch: 'omitted media', + delivery: { fileUri: '', omission: { reason: 'still over the limit' } }, + tags: ['handle', 'omission'], + }, + { + branch: 'pure transcript', + delivery: { fileUri: '', transcripts: [{ text: '你好，世界' }] }, + tags: ['handle', 'transcript'], + }, + { + branch: 'degraded pure transcript', + delivery: { + fileUri: '', + transcripts: [{ text: '你好，世界' }], + disclosure: 'transcribed after downsampling', + degraded: true, + }, + tags: ['handle', 'disclosure', 'transcript'], + }, + ])( + 'leads the $branch replacement group with the resource handle', + async ({ delivery, tags }) => { + deliverMock.mockResolvedValueOnce({ ...HANDLE_DELIVERY, ...delivery }); + const result = await processToolResultOmniMedia( + [inlinePart('image/png', PNG_BYTES)], + cfg({ image: true }), + signal, + ); + expect(result.map(tagPart)).toEqual(tags); + expect(result[0]!.text).toBe(HANDLE_TEXT); + }, + ); + it('charges uploaded additionalMedia extras against the upload-count budget', async () => { // One part whose delivery carries 7 uploaded extras uses 1 + 7 = 8 // upload slots — a multi-output policy must not let a tool result fan diff --git a/packages/core/src/omni/tool-result-media.ts b/packages/core/src/omni/tool-result-media.ts index 1dbc7bee67b..c7e9adae947 100644 --- a/packages/core/src/omni/tool-result-media.ts +++ b/packages/core/src/omni/tool-result-media.ts @@ -16,7 +16,11 @@ import { isOmniDeliveryActive, processMediaForOmniDelivery, } from './index.js'; -import { formatDisclosureText, formatOmissionText } from './disclosure.js'; +import { + formatDisclosureText, + formatOmissionText, + formatResourceHandleText, +} from './disclosure.js'; import { OmniTransportGuardError } from './guard.js'; import { OmniObjectStore, prepareOmniDownloadsDir } from './storage.js'; import { sniffMediaType } from './recognition.js'; @@ -100,6 +104,9 @@ export async function processToolResultOmniMedia( // which would report a tool that succeeded as failed. const store = new OmniObjectStore(config.storage.getQwenDir()); let tempPath: string | undefined; + // Hoisted out of the try: the guard-rejection path below names the part + // in the handle annotation it emits. + const displayName = inline.displayName ?? `tool-media.${top}`; try { // Symlink-guarded (fail closed → this part stays inline): a link // planted at downloads/ would redirect the write outside the store. @@ -111,7 +118,6 @@ export async function processToolResultOmniMedia( `${randomBytes(8).toString('hex')}.part`, ); await fs.writeFile(tempPath, bytes, { mode: 0o600 }); - const displayName = inline.displayName ?? `tool-media.${top}`; const delivery = await processMediaForOmniDelivery(tempPath, config, { expectedModality: sniffed.modality, signal, @@ -135,6 +141,16 @@ export async function processToolResultOmniMedia( ); uploadsRemaining -= delivery.additionalMedia?.filter((e) => !e.omission).length ?? 0; + // Session resource handle (M §5.2): leads the replacement group in + // every branch, keeping the disclosure's D8 adjacency to the media + // part intact. + const handleParts: Part[] = delivery.resourceId + ? [ + { + text: formatResourceHandleText(displayName, delivery.resourceId), + }, + ] + : []; if (delivery.omission) { // Explicit omission (policy design §10.2): the transport guard // could not bring the part within limits even after the guard @@ -143,6 +159,7 @@ export async function processToolResultOmniMedia( // were already charged above). changed = true; return [ + ...handleParts, { text: formatOmissionText(displayName, delivery.omission.reason) }, ...additionalParts, ...transcriptParts, @@ -157,11 +174,12 @@ export async function processToolResultOmniMedia( changed = true; return delivery.disclosure ? [ + ...handleParts, { text: formatDisclosureText(displayName, delivery.disclosure) }, ...additionalParts, ...transcriptParts, ] - : [...additionalParts, ...transcriptParts]; + : [...handleParts, ...additionalParts, ...transcriptParts]; } changed = true; uploadsRemaining--; @@ -175,12 +193,18 @@ export async function processToolResultOmniMedia( }; return delivery.disclosure ? [ + ...handleParts, { text: formatDisclosureText(displayName, delivery.disclosure) }, fileDataPart, ...additionalParts, ...transcriptParts, ] - : [fileDataPart, ...additionalParts, ...transcriptParts]; + : [ + ...handleParts, + fileDataPart, + ...additionalParts, + ...transcriptParts, + ]; } catch (err) { if (signal.aborted) throw err; if (err instanceof OmniTransportGuardError) { @@ -191,7 +215,21 @@ export async function processToolResultOmniMedia( // rationale ("produced locally, already in memory") covers only // failures of the *transfer*. changed = true; + // The source was bound before the guard ruled, and the omission + // branch with the identical "over-limit, withheld" verdict does + // disclose its handle — so withholding it here would be the one + // path that strands a resource the session already recorded. return [ + ...(err.sessionResourceId + ? [ + { + text: formatResourceHandleText( + displayName, + err.sessionResourceId, + ), + }, + ] + : []), { text: `[Tool media part withheld by the omni transport guard: ${err.message}]`, }, diff --git a/packages/core/src/services/media-memory/config.test.ts b/packages/core/src/services/media-memory/config.test.ts new file mode 100644 index 00000000000..d72259d4db4 --- /dev/null +++ b/packages/core/src/services/media-memory/config.test.ts @@ -0,0 +1,231 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { describe, expect, it } from 'vitest'; +import { + DEFAULT_OMNI_MEMORY_CONFIG, + OMNI_MEMORY_RECALL_KINDS, + OmniMemoryConfigError, + normalizeOmniMemoryConfig, +} from './config.js'; + +describe('normalizeOmniMemoryConfig', () => { + it('returns the full defaults for undefined / empty input', () => { + expect(normalizeOmniMemoryConfig(undefined)).toEqual( + DEFAULT_OMNI_MEMORY_CONFIG, + ); + expect(normalizeOmniMemoryConfig({})).toEqual(DEFAULT_OMNI_MEMORY_CONFIG); + }); + + it('never returns the shared default object references', () => { + const normalized = normalizeOmniMemoryConfig(undefined); + expect(normalized.recall.kinds).not.toBe( + DEFAULT_OMNI_MEMORY_CONFIG.recall.kinds, + ); + expect(normalized.recall.sideQuery).not.toBe( + DEFAULT_OMNI_MEMORY_CONFIG.recall.sideQuery, + ); + expect(normalized.recall.active).not.toBe( + DEFAULT_OMNI_MEMORY_CONFIG.recall.active, + ); + + // The defaults are a module-global that every later normalization in + // the process reads: an aliased nested object turns one session's + // override into the baseline for the next config load (and for the + // Config accessors already holding the object), which is how a + // per-session budget becomes a permanent one nobody configured. + normalized.recall.active.maxFilesPerCall = 99; + normalized.recall.sideQuery.maxAttempts = 99; + normalized.recall.kinds.length = 0; + expect(DEFAULT_OMNI_MEMORY_CONFIG.recall.active.maxFilesPerCall).toBe(8); + expect(DEFAULT_OMNI_MEMORY_CONFIG.recall.sideQuery.maxAttempts).toBe(1); + expect(DEFAULT_OMNI_MEMORY_CONFIG.recall.kinds).toEqual([ + ...OMNI_MEMORY_RECALL_KINDS, + ]); + }); + + it('merges per-key overrides over the defaults', () => { + const normalized = normalizeOmniMemoryConfig({ + collection: { maxInlineTextBytes: 1024 }, + recall: { + mode: 'sideQuery', + maxEntries: 6, + kinds: ['derived_media'], + includeHistoricalVersions: true, + active: { maxFilesPerCall: 2 }, + sideQuery: { model: 'qwen3.5-omni-plus', maxSelectedEntries: 3 }, + }, + }); + expect(normalized.collection.maxInlineTextBytes).toBe(1024); + expect(normalized.recall).toMatchObject({ + mode: 'sideQuery', + maxEntries: 6, + kinds: ['derived_media'], + includeHistoricalVersions: true, + active: { maxFilesPerCall: 2 }, + }); + expect(normalized.recall.sideQuery).toMatchObject({ + model: 'qwen3.5-omni-plus', + maxSelectedEntries: 3, + // Untouched keys keep their defaults. + timeoutMs: 30000, + maxAttempts: 1, + }); + // Unset scalar keeps its default. + expect(normalized.recall.maxTextChars).toBe(24000); + }); + + it('rejects unknown keys at every level', () => { + for (const raw of [ + // Root level too: a typo'd section would otherwise silently discard + // the WHOLE configuration and run defaults (active mode) while the + // operator believes sideQuery is configured. + { recalll: { mode: 'sideQuery' } }, + { collection: {}, memory: {} }, + { collection: { maxInlineBytes: 1 } }, + { recall: { maxEntry: 1 } }, + { recall: { active: { maxFiles: 1 } } }, + { recall: { sideQuery: { timeout: 1 } } }, + ]) { + expect(() => normalizeOmniMemoryConfig(raw)).toThrow( + OmniMemoryConfigError, + ); + } + }); + + it('rejects a non-object root', () => { + expect(() => normalizeOmniMemoryConfig(5 as never)).toThrow( + OmniMemoryConfigError, + ); + }); + + it('rejects non-object sections', () => { + expect(() => normalizeOmniMemoryConfig({ collection: 5 })).toThrow( + OmniMemoryConfigError, + ); + expect(() => normalizeOmniMemoryConfig({ recall: [] })).toThrow( + OmniMemoryConfigError, + ); + }); + + it('rejects invalid scalar values', () => { + for (const raw of [ + { collection: { maxInlineTextBytes: 0 } }, + { collection: { maxInlineTextBytes: 1.5 } }, + { recall: { maxEntries: -1 } }, + { recall: { maxTextChars: '24000' } }, + { recall: { mode: 'passive' } }, + { recall: { includeHistoricalVersions: 'yes' } }, + { recall: { active: { maxFilesPerCall: 0 } } }, + { recall: { sideQuery: { maxAttempts: 0 } } }, + { recall: { sideQuery: { model: '' } } }, + { recall: { sideQuery: { model: 42 } } }, + ]) { + expect(() => normalizeOmniMemoryConfig(raw)).toThrow( + OmniMemoryConfigError, + ); + } + }); + + it('accepts model: null as "use the session model"', () => { + const normalized = normalizeOmniMemoryConfig({ + recall: { sideQuery: { model: null } }, + }); + expect(normalized.recall.sideQuery.model).toBeNull(); + }); + + it('accepts an explicit mode: "active" override', () => { + // Every accepted value of a startup-fatal enum needs its own witness: + // writing the default out explicitly is the most common thing an + // operator does when documenting a settings file, and rejecting it + // would abort the session over configuration that changes nothing. + const normalized = normalizeOmniMemoryConfig({ + recall: { mode: 'active' }, + }); + expect(normalized.recall.mode).toBe('active'); + }); + + it('validates the kinds array: non-empty, known, no duplicates, wholesale replace', () => { + expect(() => normalizeOmniMemoryConfig({ recall: { kinds: [] } })).toThrow( + OmniMemoryConfigError, + ); + expect(() => + normalizeOmniMemoryConfig({ recall: { kinds: ['nonsense'] } }), + ).toThrow(OmniMemoryConfigError); + expect(() => + normalizeOmniMemoryConfig({ + recall: { kinds: ['metadata', 'metadata'] }, + }), + ).toThrow(OmniMemoryConfigError); + const normalized = normalizeOmniMemoryConfig({ + recall: { kinds: ['execution'] }, + }); + expect(normalized.recall.kinds).toEqual(['execution']); + expect(OMNI_MEMORY_RECALL_KINDS).toContain('execution'); + }); + + it('enforces cross-field budget ordering', () => { + // maxSelectedEntries must not exceed maxEntries. + expect(() => + normalizeOmniMemoryConfig({ + recall: { maxEntries: 4, sideQuery: { maxSelectedEntries: 5 } }, + }), + ).toThrow(OmniMemoryConfigError); + // maxEntries must not exceed maxCandidateEntries. + expect(() => + normalizeOmniMemoryConfig({ + recall: { maxEntries: 200 }, + }), + ).toThrow(OmniMemoryConfigError); + // A consistent triple passes. + const normalized = normalizeOmniMemoryConfig({ + recall: { + maxEntries: 20, + sideQuery: { maxCandidateEntries: 40, maxSelectedEntries: 20 }, + }, + }); + expect(normalized.recall.maxEntries).toBe(20); + }); + + it('validates the budget ordering against the DEFAULTED siblings', () => { + // The ordering holds over the EFFECTIVE configuration, not over the + // keys the operator happened to write: lowering maxEntries alone + // leaves the default selector budget (12) able to pick more entries + // than recall will ever return, so the selector's picks would be + // silently dropped mid-request. Fail loud at startup instead. + expect(() => + normalizeOmniMemoryConfig({ recall: { maxEntries: 6 } }), + ).toThrow(OmniMemoryConfigError); + // Same in the other direction: shrinking the manifest below the + // default maxEntries would have recall return entries the selector was + // never shown. + expect(() => + normalizeOmniMemoryConfig({ + recall: { sideQuery: { maxCandidateEntries: 8 } }, + }), + ).toThrow(OmniMemoryConfigError); + }); + + it('allows the budgets to be exactly equal at both boundaries', () => { + // The bounds are "must not exceed", not "must be smaller": a session + // that selects, returns, and shows the same number of entries is the + // tightest legal configuration, and rejecting it would make the + // strictest sensible setting unusable. + const normalized = normalizeOmniMemoryConfig({ + recall: { + maxEntries: 12, + sideQuery: { maxCandidateEntries: 12, maxSelectedEntries: 12 }, + }, + }); + expect(normalized.recall).toMatchObject({ + maxEntries: 12, + sideQuery: expect.objectContaining({ + maxCandidateEntries: 12, + maxSelectedEntries: 12, + }), + }); + }); +}); diff --git a/packages/core/src/services/media-memory/config.ts b/packages/core/src/services/media-memory/config.ts new file mode 100644 index 00000000000..10a21f39835 --- /dev/null +++ b/packages/core/src/services/media-memory/config.ts @@ -0,0 +1,326 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { isPlainRecord } from '../../omni/policy/types.js'; + +/** + * Startup normalization of `omni.memory` (memory design M §9). Raw + * settings enter, a fully defaulted and validated + * {@link NormalizedOmniMemoryConfig} leaves; any violation throws + * {@link OmniMemoryConfigError} and MUST abort startup — mirroring the + * `omni.processing` stance (S4): a mis-configured budget must fail loud, + * not silently fall back. + */ + +/** A configuration error in `omni.memory.*`. Startup-fatal. */ +export class OmniMemoryConfigError extends Error { + constructor(message: string) { + super(message); + this.name = 'OmniMemoryConfigError'; + } +} + +/** Entry kinds recall can surface (M §8). */ +export const OMNI_MEMORY_RECALL_KINDS = [ + 'metadata', + 'derived_media', + 'policy_result', + 'execution', +] as const; +export type OmniMemoryRecallKind = (typeof OMNI_MEMORY_RECALL_KINDS)[number]; + +export interface NormalizedOmniMemoryCollection { + /** Upper bound for `inlineText` persisted on an entry; longer text is + * truncated (the promoted artifact keeps the full content). */ + maxInlineTextBytes: number; +} + +export interface NormalizedOmniMemorySideQuery { + /** Selector model; null = whatever the side-query runner picks by + * default (the configured FAST model, falling back to the session + * model) — NOT necessarily the session's active model. */ + model: string | null; + timeoutMs: number; + maxCandidateEntries: number; + maxSelectedEntries: number; + maxAttempts: number; +} + +export interface NormalizedOmniMemoryRecall { + /** Mutually exclusive exposure (M §8.1): 'active' registers the recall + * tool; 'sideQuery' runs the passive selector before the main request. */ + mode: 'active' | 'sideQuery'; + maxEntries: number; + maxTextChars: number; + kinds: OmniMemoryRecallKind[]; + includeHistoricalVersions: boolean; + active: { maxFilesPerCall: number }; + sideQuery: NormalizedOmniMemorySideQuery; +} + +export interface NormalizedOmniMemoryConfig { + collection: NormalizedOmniMemoryCollection; + recall: NormalizedOmniMemoryRecall; +} + +/** Structural view of the Config accessor (same pattern as + * `OmniProcessingConfigView`): a config without the accessor — stub + * configs, embedders skipping initialize — reads as "memory off". */ +export interface OmniMemoryConfigView { + getOmniMemoryConfig?: () => NormalizedOmniMemoryConfig | undefined; +} + +/** Raw inputs to normalization, as threaded from settings + * (`omni.memory.collection` / `omni.memory.recall`). */ +export interface RawOmniMemorySettings { + collection?: unknown; + recall?: unknown; +} + +/** M §9 defaults. */ +export const DEFAULT_OMNI_MEMORY_CONFIG: NormalizedOmniMemoryConfig = { + collection: { maxInlineTextBytes: 65536 }, + recall: { + mode: 'active', + maxEntries: 12, + maxTextChars: 24000, + kinds: [...OMNI_MEMORY_RECALL_KINDS], + includeHistoricalVersions: false, + active: { maxFilesPerCall: 8 }, + sideQuery: { + model: null, + timeoutMs: 30000, + maxCandidateEntries: 100, + maxSelectedEntries: 12, + maxAttempts: 1, + }, + }, +}; + +const ROOT_KEYS = new Set(['collection', 'recall']); +const COLLECTION_KEYS = new Set(['maxInlineTextBytes']); +const RECALL_KEYS = new Set([ + 'mode', + 'maxEntries', + 'maxTextChars', + 'kinds', + 'includeHistoricalVersions', + 'active', + 'sideQuery', +]); +const ACTIVE_KEYS = new Set(['maxFilesPerCall']); +const SIDE_QUERY_KEYS = new Set([ + 'model', + 'timeoutMs', + 'maxCandidateEntries', + 'maxSelectedEntries', + 'maxAttempts', +]); + +function fail(message: string): never { + throw new OmniMemoryConfigError(message); +} + +function requireRecord( + value: unknown, + where: string, + allowedKeys: Set, +): Record { + if (!isPlainRecord(value)) { + fail(`${where}: must be an object (got ${JSON.stringify(value)})`); + } + for (const key of Object.keys(value)) { + if (!allowedKeys.has(key)) { + fail( + `${where}: unknown key "${key}" (allowed: ${[...allowedKeys].join(', ')})`, + ); + } + } + return value; +} + +function positiveInteger(value: unknown, where: string): number { + if (typeof value !== 'number' || !Number.isInteger(value) || value <= 0) { + fail(`${where}: must be a positive integer (got ${JSON.stringify(value)})`); + } + return value; +} + +function requireBoolean(value: unknown, where: string): boolean { + if (typeof value !== 'boolean') { + fail(`${where}: must be a boolean (got ${JSON.stringify(value)})`); + } + return value; +} + +function normalizeKinds(value: unknown, where: string): OmniMemoryRecallKind[] { + if (!Array.isArray(value) || value.length === 0) { + fail(`${where}: must be a non-empty array of entry kinds`); + } + const known = new Set(OMNI_MEMORY_RECALL_KINDS); + const seen = new Set(); + for (const kind of value) { + if (typeof kind !== 'string' || !known.has(kind)) { + fail( + `${where}: unknown entry kind ${JSON.stringify(kind)} ` + + `(allowed: ${OMNI_MEMORY_RECALL_KINDS.join(', ')})`, + ); + } + if (seen.has(kind as OmniMemoryRecallKind)) { + fail(`${where}: duplicate entry kind "${kind}"`); + } + seen.add(kind as OmniMemoryRecallKind); + } + return [...seen]; +} + +/** + * Normalize `omni.memory`. Scalars default per-key; the `kinds` array + * replaces wholesale when present (never element-merged — M §9). Invalid + * configuration is startup-fatal. + */ +export function normalizeOmniMemoryConfig( + raw: RawOmniMemorySettings | undefined, +): NormalizedOmniMemoryConfig { + const defaults = DEFAULT_OMNI_MEMORY_CONFIG; + + // Reject unknown ROOT keys too (same stance as every nested level, and as + // the sibling `omni.processing` normalizer): a typo'd or renamed section + // would otherwise silently discard the whole configuration — the session + // would run default `active` mode while the user believes sideQuery is + // configured, which is exactly the silent fallback this module forbids. + if (raw !== undefined) { + requireRecord(raw, 'omni.memory', ROOT_KEYS); + } + + let maxInlineTextBytes = defaults.collection.maxInlineTextBytes; + if (raw?.collection !== undefined) { + const collection = requireRecord( + raw.collection, + 'omni.memory.collection', + COLLECTION_KEYS, + ); + if (collection['maxInlineTextBytes'] !== undefined) { + maxInlineTextBytes = positiveInteger( + collection['maxInlineTextBytes'], + 'omni.memory.collection.maxInlineTextBytes', + ); + } + } + + const recall = { ...defaults.recall }; + recall.kinds = [...defaults.recall.kinds]; + recall.active = { ...defaults.recall.active }; + recall.sideQuery = { ...defaults.recall.sideQuery }; + + if (raw?.recall !== undefined) { + const rawRecall = requireRecord( + raw.recall, + 'omni.memory.recall', + RECALL_KEYS, + ); + + if (rawRecall['mode'] !== undefined) { + const mode = rawRecall['mode']; + if (mode !== 'active' && mode !== 'sideQuery') { + fail( + `omni.memory.recall.mode: must be "active" or "sideQuery" ` + + `(got ${JSON.stringify(mode)})`, + ); + } + recall.mode = mode; + } + if (rawRecall['maxEntries'] !== undefined) { + recall.maxEntries = positiveInteger( + rawRecall['maxEntries'], + 'omni.memory.recall.maxEntries', + ); + } + if (rawRecall['maxTextChars'] !== undefined) { + recall.maxTextChars = positiveInteger( + rawRecall['maxTextChars'], + 'omni.memory.recall.maxTextChars', + ); + } + if (rawRecall['kinds'] !== undefined) { + recall.kinds = normalizeKinds( + rawRecall['kinds'], + 'omni.memory.recall.kinds', + ); + } + if (rawRecall['includeHistoricalVersions'] !== undefined) { + recall.includeHistoricalVersions = requireBoolean( + rawRecall['includeHistoricalVersions'], + 'omni.memory.recall.includeHistoricalVersions', + ); + } + + if (rawRecall['active'] !== undefined) { + const active = requireRecord( + rawRecall['active'], + 'omni.memory.recall.active', + ACTIVE_KEYS, + ); + if (active['maxFilesPerCall'] !== undefined) { + recall.active.maxFilesPerCall = positiveInteger( + active['maxFilesPerCall'], + 'omni.memory.recall.active.maxFilesPerCall', + ); + } + } + + if (rawRecall['sideQuery'] !== undefined) { + const sq = requireRecord( + rawRecall['sideQuery'], + 'omni.memory.recall.sideQuery', + SIDE_QUERY_KEYS, + ); + if (sq['model'] !== undefined) { + const model = sq['model']; + if (model !== null && (typeof model !== 'string' || model === '')) { + fail( + `omni.memory.recall.sideQuery.model: must be null or a ` + + `non-empty string (got ${JSON.stringify(model)})`, + ); + } + recall.sideQuery.model = model; + } + for (const key of [ + 'timeoutMs', + 'maxCandidateEntries', + 'maxSelectedEntries', + 'maxAttempts', + ] as const) { + if (sq[key] !== undefined) { + recall.sideQuery[key] = positiveInteger( + sq[key], + `omni.memory.recall.sideQuery.${key}`, + ); + } + } + } + } + + // Cross-field budget ordering (M §9): a selector may never pick more + // than recall returns, and recall may never return more than the + // selector was shown. + if (recall.sideQuery.maxSelectedEntries > recall.maxEntries) { + fail( + `omni.memory.recall.sideQuery.maxSelectedEntries ` + + `(${recall.sideQuery.maxSelectedEntries}) must not exceed ` + + `omni.memory.recall.maxEntries (${recall.maxEntries})`, + ); + } + if (recall.maxEntries > recall.sideQuery.maxCandidateEntries) { + fail( + `omni.memory.recall.maxEntries (${recall.maxEntries}) must not ` + + `exceed omni.memory.recall.sideQuery.maxCandidateEntries ` + + `(${recall.sideQuery.maxCandidateEntries})`, + ); + } + + return { collection: { maxInlineTextBytes }, recall }; +} diff --git a/packages/core/src/services/media-memory/index.ts b/packages/core/src/services/media-memory/index.ts new file mode 100644 index 00000000000..150467ef1b0 --- /dev/null +++ b/packages/core/src/services/media-memory/index.ts @@ -0,0 +1,80 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Multimodal media memory (upstream design M). The service below is the + * ONLY entry point: collection writes happen exclusively from the omni + * harness (M §14 — no daemon, no MCP surface), and Stage B recall reads + * go through the same facade. Store/type modules are internal except for + * the record types recall consumers need. + */ + +export { + MediaMemoryService, + DEFAULT_MAX_INLINE_TEXT_BYTES, + MEDIA_DETECTOR_VERSION, + type MediaMemoryBinding, + type PolicyOutputInput, + type PolicyMediaOutputInput, + type PolicyTextOutputInput, + type PolicySucceededInput, + type PolicySucceededCommit, + type ReusableExecutionOutputs, + type ReusableOutputRecord, +} from './service.js'; +export { + MediaResourceRegistry, + type MediaResourceBinding, + type OmniMediaRegistryView, +} from './registry.js'; +export { + MediaMemoryRecallService, + MediaMemoryRecallRejection, + type MediaMemoryCandidateSummary, + type MediaMemoryRecallRequest, + type MediaMemoryRecallResult, + type MediaMemoryRecallFile, + type MediaMemoryRecallEntry, + type MediaMemoryRecallGap, + type MediaMemoryRecallProvenance, + type MediaMemoryRecallAdvisor, + type MediaMemoryNextPolicyAction, +} from './recall.js'; +export { + OmniMemoryConfigError, + DEFAULT_OMNI_MEMORY_CONFIG, + OMNI_MEMORY_RECALL_KINDS, + normalizeOmniMemoryConfig, + type NormalizedOmniMemoryConfig, + type NormalizedOmniMemoryCollection, + type NormalizedOmniMemoryRecall, + type NormalizedOmniMemorySideQuery, + type OmniMemoryConfigView, + type OmniMemoryRecallKind, + type RawOmniMemorySettings, +} from './config.js'; +export type { + FileRecognizedCommit, + FileRecognizedEvent, + KnownMediaMemoryRole, + MediaArtifactRef, + MediaChannel, + MediaCoverage, + MediaExecutionOrigin, + MediaFileId, + MediaFileOrigin, + MediaFileRecord, + MediaFileVersionId, + MediaFileVersionRecord, + MediaMemoryEntryId, + MediaMemorySnapshot, + MediaPolicyExecutionRecord, + MediaScope, + MediaVersionRecognition, + MediaVersionSource, + NormalizedPolicyOutput, + PolicyExecutionId, +} from './types.js'; diff --git a/packages/core/src/services/media-memory/recall.test.ts b/packages/core/src/services/media-memory/recall.test.ts new file mode 100644 index 00000000000..f648b340357 --- /dev/null +++ b/packages/core/src/services/media-memory/recall.test.ts @@ -0,0 +1,981 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { + DEFAULT_OMNI_MEMORY_CONFIG, + MediaMemoryRecallRejection, + MediaMemoryRecallService, + MediaMemoryService, + MediaResourceRegistry, + type MediaMemoryBinding, +} from './index.js'; +import type { NormalizedOmniMemoryRecall } from './config.js'; + +let root: string; +let moviePath: string; +let clock: number; +let service: TestMediaMemoryService; +let registry: MediaResourceRegistry; + +class TestMediaMemoryService extends MediaMemoryService { + protected override now(): string { + return new Date(clock++).toISOString(); + } +} + +const SHA_MOVIE = 'a'.repeat(64); +const SHA_MOVIE_V2 = 'b'.repeat(64); +const SHA_DEGRADED = 'c'.repeat(64); +const SHA_TRANSCRIPT = 'd'.repeat(64); +const SHA_AUDIO = 'e'.repeat(64); + +beforeEach(async () => { + root = await fs.mkdtemp(path.join(os.tmpdir(), 'omni-memory-recall-')); + clock = 1_754_870_400_000; + service = new TestMediaMemoryService(root); + registry = new MediaResourceRegistry(); + moviePath = path.join(root, 'src', 'breaking-surface.mkv'); + await fs.mkdir(path.dirname(moviePath), { recursive: true }); + await fs.writeFile(moviePath, 'movie-bytes'); +}); + +afterEach(async () => { + await fs.rm(root, { recursive: true, force: true }); +}); + +function recallConfig( + overrides?: Partial, +): NormalizedOmniMemoryRecall { + return { + ...DEFAULT_OMNI_MEMORY_CONFIG.recall, + kinds: [...DEFAULT_OMNI_MEMORY_CONFIG.recall.kinds], + ...overrides, + }; +} + +function recallService( + config = recallConfig(), + options?: ConstructorParameters[3], +): MediaMemoryRecallService { + return new MediaMemoryRecallService(root, config, registry, options); +} + +async function recognizeMovie(sha256 = SHA_MOVIE): Promise { + const commit = await service.recordFileRecognized({ + fileRef: moviePath, + sha256, + mediaType: 'video', + metadata: { durationMs: 4_860_000, width: 1920, height: 1080 }, + sizeBytes: 123_456_789, + mimeType: 'video/x-matroska', + origin: 'user', + source: { protocol: 'local', locator: 'breaking-surface.mkv' }, + recognition: { + ingestionConfigHash: 'ingest-hash', + detectorVersion: 'omni-sniff-ffprobe/1', + probeStatus: 'complete', + }, + }); + expect(commit).toBeDefined(); + return commit!; +} + +/** Write a fake managed object and return its absolute path. */ +async function writeObject(name: string): Promise { + const objectPath = path.join(root, 'objects', name); + await fs.mkdir(path.dirname(objectPath), { recursive: true }); + await fs.writeFile(objectPath, `bytes-of-${name}`); + return objectPath; +} + +async function commitDegrade(source: MediaMemoryBinding): Promise<{ + degradedPath: string; + binding: MediaMemoryBinding; +}> { + const degradedPath = await writeObject('degraded.mp4'); + const commit = await service.commitPolicySucceeded({ + invocationId: 'inv-degrade', + source, + executionOrigin: { + kind: 'fixed_policy', + policyId: 'video-default', + stage: 'preprocessing', + }, + toolName: 'omni_degrade_video', + toolVersion: '1', + finalArguments: { height: 480 }, + omniConfigHash: 'hash-degrade', + startedAt: '2026-08-11T00:00:00.000Z', + completedAt: '2026-08-11T00:00:05.000Z', + outputs: [ + { + kind: 'media', + objectPath: degradedPath, + sha256: SHA_DEGRADED, + mediaType: 'video', + metadata: { durationMs: 4_860_000, width: 854, height: 480 }, + sizeBytes: 1_000_000, + mimeType: 'video/mp4', + role: 'degraded', + disclosure: 'downscaled to 480p', + }, + ], + }); + expect(commit).toBeDefined(); + return { degradedPath, binding: commit!.mediaBindings.get(SHA_DEGRADED)! }; +} + +async function commitTranscript( + source: MediaMemoryBinding, + text = 'Two divers surface at dawn near the wreck.', +): Promise { + const objectPath = await writeObject('transcript.txt'); + const commit = await service.commitPolicySucceeded({ + invocationId: 'inv-transcribe', + source, + executionOrigin: { kind: 'model' }, + toolName: 'omni_transcribe', + finalArguments: { language: 'auto' }, + omniConfigHash: 'hash-transcribe', + startedAt: '2026-08-11T00:01:00.000Z', + completedAt: '2026-08-11T00:01:30.000Z', + outputs: [ + { + kind: 'text', + objectPath, + sha256: SHA_TRANSCRIPT, + mimeType: 'text/plain', + text, + sizeBytes: text.length, + role: 'transcript', + }, + ], + }); + expect(commit).toBeDefined(); +} + +function bindSource(source: MediaMemoryBinding): string { + return registry.bind({ + ...source, + fileRef: moviePath, + mediaType: 'video', + }).resourceId; +} + +describe('MediaMemoryRecallService — request validation', () => { + it('rejects an empty request outright', async () => { + await expect( + recallService().recall({ resourceIds: [], query: 'anything' }), + ).rejects.toThrow(MediaMemoryRecallRejection); + }); + + it('rejects the whole request on a handle this session never issued', async () => { + const source = await recognizeMovie(); + const resourceId = bindSource(source); + await expect( + recallService().recall({ + resourceIds: [resourceId, 'media-9-deadbeef'], + query: 'anything', + }), + ).rejects.toThrow(/not issued in this session/); + }); +}); + +describe('MediaMemoryRecallService — full graph recall', () => { + it('returns metadata, outputs, and executions of the current version graph', async () => { + const source = await recognizeMovie(); + await commitDegrade(source); + await commitTranscript(source); + const resourceId = bindSource(source); + + const result = await recallService().recall({ + resourceIds: [resourceId], + query: 'what happens in the movie', + }); + + expect(result.status).toBe('hit'); + expect(result.gaps).toEqual([]); + expect(result.files).toEqual([ + { + fileId: source.fileId, + fileVersionId: source.fileVersionId, + current: true, + mediaType: 'video', + }, + ]); + + const byKind = new Map(result.entries.map((e) => [e.kind, e])); + expect([...byKind.keys()].sort()).toEqual([ + 'derived_media', + 'execution', + 'metadata', + 'policy_result', + ]); + expect(result.entries).toHaveLength(5); // 2 executions + + const metadata = byKind.get('metadata')!; + expect(metadata.content).toContain('"width":1920'); + expect(metadata.channels).toEqual(['technical_metadata']); + expect(metadata.provenance.omniConfigHash).toBe('ingest-hash'); + + const transcript = byKind.get('policy_result')!; + expect(transcript.role).toBe('transcript'); + expect(transcript.content).toContain('surface at dawn'); + expect(transcript.channels).toEqual(['speech_text']); + expect(transcript.evidenceRefs[0].fileVersionId).toBe(source.fileVersionId); + expect(transcript.evidenceRefs[0].executionId).toBeDefined(); + expect(transcript.provenance.toolName).toBe('omni_transcribe'); + + const derived = byKind.get('derived_media')!; + expect(derived.disclosure).toBe('downscaled to 480p'); + expect(derived.provenance.policyId).toBe('video-default'); + expect(derived.provenance.stage).toBe('preprocessing'); + // A fresh session handle was bound for the derived artifact and never + // leaks a path. + expect(derived.resourceId).toBeDefined(); + const bound = registry.resolve(derived.resourceId!); + expect(bound?.fileVersionId).toBeDefined(); + expect(JSON.stringify(result)).not.toContain(root); + }); + + it('reaches entries parented on intermediate derivatives (audio → transcript chain)', async () => { + const source = await recognizeMovie(); + const audioPath = await writeObject('audio.m4a'); + const audioCommit = await service.commitPolicySucceeded({ + invocationId: 'inv-extract', + source, + executionOrigin: { + kind: 'fixed_policy', + policyId: 'video-default', + stage: 'preprocessing', + }, + toolName: 'omni_extract_audio', + finalArguments: {}, + omniConfigHash: 'hash-extract', + startedAt: '2026-08-11T00:00:00.000Z', + completedAt: '2026-08-11T00:00:02.000Z', + outputs: [ + { + kind: 'media', + objectPath: audioPath, + sha256: SHA_AUDIO, + mediaType: 'audio', + metadata: { durationMs: 4_860_000 }, + sizeBytes: 5_000_000, + mimeType: 'audio/mp4', + role: 'extracted_audio', + }, + ], + }); + await commitTranscript(audioCommit!.mediaBindings.get(SHA_AUDIO)!); + const resourceId = bindSource(source); + + const result = await recallService().recall({ + resourceIds: [resourceId], + query: 'transcript', + kinds: ['policy_result'], + }); + + expect(result.entries).toHaveLength(1); + expect(result.entries[0].role).toBe('transcript'); + // Evidence points at the audio derivative it was transcribed from. + expect(result.entries[0].evidenceRefs[0].fileVersionId).not.toBe( + source.fileVersionId, + ); + }); +}); + +describe('MediaMemoryRecallService — filters, limit, ranking', () => { + it('narrows by kinds and roles', async () => { + const source = await recognizeMovie(); + await commitDegrade(source); + await commitTranscript(source); + const resourceId = bindSource(source); + const svc = recallService(); + + const kindsOnly = await svc.recall({ + resourceIds: [resourceId], + query: 'q', + kinds: ['derived_media'], + }); + expect(kindsOnly.entries.map((e) => e.kind)).toEqual(['derived_media']); + + const rolesOnly = await svc.recall({ + resourceIds: [resourceId], + query: 'q', + roles: ['transcript'], + }); + expect(rolesOnly.entries.map((e) => e.role)).toEqual(['transcript']); + }); + + it('ranks query-relevant entries first and applies the entry budget', async () => { + const source = await recognizeMovie(); + await commitDegrade(source); + await commitTranscript(source, 'The captain reads the tide tables.'); + const resourceId = bindSource(source); + + const result = await recallService().recall({ + resourceIds: [resourceId], + query: 'tide tables transcript', + limit: 2, + }); + + expect(result.entries).toHaveLength(2); + expect(result.entries[0].role).toBe('transcript'); + }); + + it('ranks a Chinese query by partial phrase overlap', async () => { + // Chinese writes without separators, so splitting on them yielded ONE + // token per phrase, scored by whole-substring containment: unless an + // entry repeated the caller's exact phrasing, every candidate scored + // zero and ordering silently collapsed to newest-first. Here the + // transcript is the OLDEST entry, so newest-first would rank it last. + const source = await recognizeMovie(); + await commitTranscript(source, '机器人独自走在海滩上，梦见了狗。'); + await commitDegrade(source); + const resourceId = bindSource(source); + + const result = await recallService().recall({ + resourceIds: [resourceId], + // Not a substring of the transcript — overlapping in phrasing only. + query: '机器人在海滩梦见什么', + }); + + expect(result.entries[0].role).toBe('transcript'); + }); + + it('caps request limit at the configured maxEntries', async () => { + const source = await recognizeMovie(); + await commitDegrade(source); + await commitTranscript(source); + const resourceId = bindSource(source); + + const result = await recallService(recallConfig({ maxEntries: 3 })).recall({ + resourceIds: [resourceId], + query: 'q', + limit: 100, + }); + expect(result.entries).toHaveLength(3); + }); + + it('bounds entry content at maxTextChars', async () => { + const source = await recognizeMovie(); + await commitTranscript(source, 'x'.repeat(500)); + const resourceId = bindSource(source); + + const result = await recallService( + recallConfig({ maxTextChars: 40 }), + ).recall({ resourceIds: [resourceId], query: 'q', roles: ['transcript'] }); + expect(result.entries[0].content).toHaveLength(40); + }); + + it('never cuts a surrogate pair in half at the maxTextChars boundary', async () => { + const source = await recognizeMovie(); + // Emoji and CJK extension characters are two UTF-16 units each, so an + // odd budget lands mid-pair — the common case for a transcript of a + // chat recording, not an exotic one. + await commitTranscript(source, '🎬'.repeat(30)); + const resourceId = bindSource(source); + + const result = await recallService( + recallConfig({ maxTextChars: 41 }), + ).recall({ resourceIds: [resourceId], query: 'q', roles: ['transcript'] }); + + // A trailing lone surrogate is not text: it cannot round-trip through + // UTF-8, so the recall payload handed to the model carries a replacement + // character or an encoder error where a character used to be. + expect(result.entries[0].content).toBe('🎬'.repeat(20)); + expect(result.entries[0].contentTruncated).toBe(true); + }); +}); + +describe('MediaMemoryRecallService — current-version-first (§9.5)', () => { + it('consults only the current version and hints at bound history', async () => { + const source = await recognizeMovie(); + await commitDegrade(source); + await commitTranscript(source); + const staleResourceId = bindSource(source); + const fresh = await recognizeMovie(SHA_MOVIE_V2); // content changed on disk + + const result = await recallService().recall({ + resourceIds: [staleResourceId], + query: 'q', + }); + + // Nothing processed for the new content yet → only its metadata comes + // back, with an explicit not_processed gap and the stale version + // listed as history. + expect(result.status).toBe('partial'); + expect(result.entries.map((e) => e.kind)).toEqual(['metadata']); + expect(result.gaps).toEqual([ + { + scope: {}, + channels: ['visual', 'acoustic', 'speech_text'], + reason: 'not_processed', + }, + ]); + expect(result.files).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + fileVersionId: fresh.fileVersionId, + current: true, + }), + expect.objectContaining({ + fileVersionId: source.fileVersionId, + current: false, + }), + ]), + ); + }); + + it('returns historical entries when explicitly requested', async () => { + const source = await recognizeMovie(); + await commitTranscript(source); + const staleResourceId = bindSource(source); + await recognizeMovie(SHA_MOVIE_V2); + + const result = await recallService().recall({ + resourceIds: [staleResourceId], + query: 'q', + includeHistoricalVersions: true, + roles: ['transcript'], + }); + + expect(result.entries.map((e) => e.role)).toEqual(['transcript']); + }); + + it('keeps gaps speaking for the current version when history is included', async () => { + // The first content state was never processed; the file then changed on + // disk and the NEW content got the full treatment. + const stale = await recognizeMovie(); + const staleResourceId = bindSource(stale); + const fresh = await recognizeMovie(SHA_MOVIE_V2); + await commitDegrade(fresh); // visual + acoustic + await commitTranscript(fresh); // speech_text + + const result = await recallService().recall({ + resourceIds: [staleResourceId], + query: 'q', + includeHistoricalVersions: true, + }); + + // History widens what memory OFFERS — both content states are listed. + expect(result.files.map((f) => f.fileVersionId).sort()).toEqual( + [stale.fileVersionId, fresh.fileVersionId].sort(), + ); + expect(result.entries.filter((e) => e.kind === 'metadata')).toHaveLength(2); + // ...and never what memory OWES. Deriving gaps from the consulted set + // would report a superseded version's untouched channels as holes in the + // content that exists NOW, and the advisor would then hand the model + // follow-up calls to re-derive work that is already complete. + expect(result.gaps).toEqual([]); + expect(result.status).toBe('hit'); + }); +}); + +describe('MediaMemoryRecallService — gaps and availability', () => { + it('never lets a request filter manufacture a gap', async () => { + const source = await recognizeMovie(); + await commitDegrade(source); // visual + acoustic + await commitTranscript(source); // speech_text + const resourceId = bindSource(source); + const svc = recallService(); + + const unfiltered = await svc.recall({ + resourceIds: [resourceId], + query: 'q', + }); + expect(unfiltered.gaps).toEqual([]); + + // Gap truth is a property of the graph, not of the question. Deriving + // it from the filtered entry set would report the transcript's channel + // as never processed the moment a caller asked only for metadata — and + // the advisor would then suggest re-transcribing a film that already + // has a transcript, at full cost, on every narrowed recall. + const kindsOnly = await svc.recall({ + resourceIds: [resourceId], + query: 'q', + kinds: ['metadata'], + }); + expect(kindsOnly.entries.map((e) => e.kind)).toEqual(['metadata']); + expect(kindsOnly.gaps).toEqual([]); + expect(kindsOnly.status).toBe('hit'); + + // Even a filter that matches nothing at all leaves the gaps empty: + // seeing nothing is not the same as nothing being there. + const rolesOnly = await svc.recall({ + resourceIds: [resourceId], + query: 'q', + roles: ['keyframe'], + }); + expect(rolesOnly.entries).toEqual([]); + expect(rolesOnly.gaps).toEqual([]); + }); + + it('reports unprocessed channels as gaps (partial status)', async () => { + const source = await recognizeMovie(); + await commitDegrade(source); // visual+acoustic covered, no transcript + const resourceId = bindSource(source); + + const result = await recallService().recall({ + resourceIds: [resourceId], + query: 'q', + }); + + expect(result.status).toBe('partial'); + expect(result.gaps).toEqual([ + { scope: {}, channels: ['speech_text'], reason: 'not_processed' }, + ]); + }); + + it('reports sampled-only coverage as partial_coverage', async () => { + const source = await recognizeMovie(); + const keyframePath = await writeObject('keyframe.jpg'); + await service.commitPolicySucceeded({ + invocationId: 'inv-keyframes', + source, + executionOrigin: { kind: 'model' }, + toolName: 'omni_extract_keyframes', + finalArguments: {}, + omniConfigHash: 'hash-keyframes', + startedAt: '2026-08-11T00:00:00.000Z', + completedAt: '2026-08-11T00:00:01.000Z', + outputs: [ + { + kind: 'media', + objectPath: keyframePath, + sha256: SHA_DEGRADED, + mediaType: 'image', + metadata: { width: 1920, height: 1080 }, + sizeBytes: 100_000, + mimeType: 'image/jpeg', + role: 'keyframe', + }, + ], + }); + const resourceId = bindSource(source); + + const result = await recallService().recall({ + resourceIds: [resourceId], + query: 'q', + }); + + expect(result.gaps).toEqual( + expect.arrayContaining([ + { scope: {}, channels: ['visual'], reason: 'partial_coverage' }, + { + scope: {}, + channels: ['acoustic', 'speech_text'], + reason: 'not_processed', + }, + ]), + ); + }); + + it('flags a deleted source file as artifact_unavailable (D5)', async () => { + const source = await recognizeMovie(); + await commitDegrade(source); + await commitTranscript(source); + const resourceId = bindSource(source); + await fs.rm(moviePath); + + const result = await recallService().recall({ + resourceIds: [resourceId], + query: 'q', + }); + + expect(result.status).toBe('partial'); + expect(result.gaps).toEqual([ + { + scope: {}, + channels: ['visual', 'acoustic', 'speech_text'], + reason: 'artifact_unavailable', + }, + ]); + // The memory itself is intact — entries still come back. + expect(result.entries.length).toBeGreaterThan(0); + }); + + it('never emits sibling gaps or advice for a deleted source (C7)', async () => { + // Only the audio track was ever extracted, so `visual` has no + // evidence: the pre-fix code emitted `artifact_unavailable` AND a + // `not_processed` sibling, and the advisor — which only filters + // `artifact_unavailable` — then suggested keyframe extraction on a + // handle whose file is gone (a guaranteed-to-fail turn). + const source = await recognizeMovie(); + await commitTranscript(source); + const resourceId = bindSource(source); + await fs.rm(moviePath); + + const result = await recallService(recallConfig(), { + advise: ({ resourceId: rid, gap }) => + gap.reason === 'artifact_unavailable' + ? [] + : [ + { + toolName: 'omni_extract_keyframes', + resourceId: rid, + arguments: {}, + reason: 'no visual evidence', + }, + ], + }).recall({ resourceIds: [resourceId], query: 'q' }); + + expect(result.gaps).toEqual([ + { + scope: {}, + channels: ['visual', 'acoustic', 'speech_text'], + reason: 'artifact_unavailable', + }, + ]); + // Nothing can be gathered from a deleted file — no advice at all. + expect(result.nextPolicyActions).toBeUndefined(); + }); + + it('withholds the handle of a deleted derived artifact', async () => { + const source = await recognizeMovie(); + const { degradedPath } = await commitDegrade(source); + await commitTranscript(source); + const resourceId = bindSource(source); + await fs.rm(degradedPath); + + const result = await recallService().recall({ + resourceIds: [resourceId], + query: 'q', + }); + + const derived = result.entries.find((e) => e.kind === 'derived_media')!; + expect(derived.resourceId).toBeUndefined(); + expect(result.gaps).toEqual( + expect.arrayContaining([ + expect.objectContaining({ reason: 'artifact_unavailable' }), + ]), + ); + }); + + it('degrades to a miss with an unavailability gap when the graph lost the binding', async () => { + // A handle minted in-session, but the store never saw the version + // (e.g. memory.json was corrupt-rebuilt after binding). + const resourceId = registry.bind({ + fileId: 'f-ghost', + fileVersionId: 'v-ghost', + rootFileId: 'f-ghost', + fileRef: moviePath, + mediaType: 'video', + }).resourceId; + + const result = await recallService().recall({ + resourceIds: [resourceId], + query: 'q', + }); + + expect(result.status).toBe('miss'); + expect(result.entries).toEqual([]); + expect(result.gaps).toEqual([ + { + scope: {}, + channels: ['visual', 'acoustic', 'speech_text'], + reason: 'artifact_unavailable', + }, + ]); + }); +}); + +describe('MediaMemoryRecallService — nextPolicyActions', () => { + it('omits the field without an advisor and emits its suggestions with one', async () => { + const source = await recognizeMovie(); + await commitDegrade(source); // speech_text gap + const resourceId = bindSource(source); + + const plain = await recallService().recall({ + resourceIds: [resourceId], + query: 'q', + }); + expect(plain.nextPolicyActions).toBeUndefined(); + + const advised = await recallService(recallConfig(), { + advise: ({ resourceId: rid, gap }) => + gap.reason === 'not_processed' && gap.channels.includes('speech_text') + ? [ + { + toolName: 'omni_transcribe', + resourceId: rid, + arguments: {}, + reason: 'speech has not been transcribed', + }, + ] + : [], + }).recall({ resourceIds: [resourceId], query: 'q' }); + + expect(advised.nextPolicyActions).toEqual([ + { + toolName: 'omni_transcribe', + resourceId, + arguments: {}, + reason: 'speech has not been transcribed', + }, + ]); + }); +}); + +describe('MediaMemoryRecallService — truncation visibility', () => { + it('says how many matched when the budget cut the list short', async () => { + const source = await recognizeMovie(); + await commitDegrade(source); + await commitTranscript(source); + const resourceId = bindSource(source); + + const full = await recallService().recall({ + resourceIds: [resourceId], + query: 'q', + }); + // An exhaustive page carries no counter, so a reader seeing the field + // absent knows it is looking at everything. + expect(full.matchedEntries).toBeUndefined(); + expect(full.entries.length).toBeGreaterThan(1); + + const page = await recallService().recall({ + resourceIds: [resourceId], + query: 'q', + limit: 1, + }); + expect(page.entries).toHaveLength(1); + // A real audit concluded "no keyframes were ever extracted" from a + // truncated page; the counter is what makes that mistake impossible. + expect(page.matchedEntries).toBe(full.entries.length); + }); +}); + +describe('MediaMemoryRecallService — sideQuery manifest and selection (§9.3)', () => { + it('summarizes candidates without full text or local paths', async () => { + const source = await recognizeMovie(); + await commitDegrade(source); + await commitTranscript(source); + const resourceId = bindSource(source); + // Bind the derived artifact's handle too, so a leak has something to + // leak: an active recall over the same graph mints one. + const active = await recallService().recall({ + resourceIds: [resourceId], + query: 'q', + }); + const derivedHandle = active.entries.find( + (e) => e.kind === 'derived_media', + )!.resourceId!; + + const manifest = await recallService().candidateSummaries([resourceId]); + + expect(manifest.length).toBeGreaterThanOrEqual(3); + expect(manifest.some((c) => c.role === 'transcript')).toBe(true); + for (const candidate of manifest) { + expect((candidate.description ?? '').length).toBeLessThanOrEqual(200); + } + // Never a local path or a session handle in selector-visible data. + // Manifest rows exist to be judged for relevance and answered with + // entryIds; a handle there is an executable capability that skipped the + // availability pass `finishResult` runs, so the selector prompt would + // carry a resolvable reference to bytes nobody checked still exist. + const serialized = JSON.stringify(manifest); + expect(serialized).not.toContain(root); + expect(serialized).not.toContain(resourceId); + expect(serialized).not.toContain(derivedHandle); + expect(serialized).not.toMatch(/media-\d+-[0-9a-f]{8}/); + }); + + it('cuts a manifest description at 200 chars, keeping it a text prefix', async () => { + const source = await recognizeMovie(); + // A real transcript is thousands of characters; the manifest holds up + // to maxCandidateEntries (100) rows and is prompt for the selector + // model, so an uncapped preview would put whole transcripts in the very + // request whose job is to decide which transcripts to load. + const text = 'Two divers surface at dawn near the wreck. '.repeat(20); + await commitTranscript(source, text); + const resourceId = bindSource(source); + + const manifest = await recallService().candidateSummaries([resourceId]); + const transcript = manifest.find((c) => c.role === 'transcript')!; + + expect(text.length).toBeGreaterThan(200); + expect(transcript.description).toHaveLength(200); + // A preview is only honest if it is a prefix — a reshaped or elided + // description would have the selector judge relevance on text memory + // does not hold. + expect(text.startsWith(transcript.description!)).toBe(true); + }); + + it('orders the manifest newest first, so the cap drops the oldest rows', async () => { + const source = await recognizeMovie(); + await commitDegrade(source); + await commitTranscript(source); + const resourceId = bindSource(source); + + const manifest = await recallService().candidateSummaries([resourceId]); + + // Newest first: the two policy executions (recorded with 2026 + // completion times), then the transcript, the degraded derivative, and + // finally the metadata synthesized at recognition. + expect( + manifest.map((c) => `${c.kind}:${c.role ?? c.producer ?? ''}`), + ).toEqual([ + 'execution:omni_transcribe', + 'execution:omni_degrade_video', + 'policy_result:transcript', + 'derived_media:degraded', + 'metadata:', + ]); + + // Ordering is what decides which rows the cap silently withholds: + // insertion order would hide the most recent processing — exactly the + // knowledge most likely to answer the request — behind rows the + // selector has no way to know were dropped. + const capped = await recallService( + recallConfig({ + sideQuery: { + ...DEFAULT_OMNI_MEMORY_CONFIG.recall.sideQuery, + maxCandidateEntries: 2, + }, + }), + ).candidateSummaries([resourceId]); + expect(capped).toEqual(manifest.slice(0, 2)); + }); + + it('caps the manifest at maxCandidateEntries deterministically', async () => { + const source = await recognizeMovie(); + await commitDegrade(source); + await commitTranscript(source); + const resourceId = bindSource(source); + const config = recallConfig({ + sideQuery: { + ...DEFAULT_OMNI_MEMORY_CONFIG.recall.sideQuery, + maxCandidateEntries: 2, + }, + }); + + const first = await recallService(config).candidateSummaries([resourceId]); + const second = await recallService(config).candidateSummaries([resourceId]); + + expect(first).toHaveLength(2); + expect(second).toEqual(first); + }); + + it('validates handles like recall(): an unknown handle rejects', async () => { + await expect( + recallService().candidateSummaries(['media-9-ffff']), + ).rejects.toMatchObject({ reason: 'unknown_resource' }); + }); + + it('materializes exactly the selected entries via the unified protocol', async () => { + const source = await recognizeMovie(); + const { degradedPath } = await commitDegrade(source); + await commitTranscript(source); + const resourceId = bindSource(source); + const svc = recallService(); + const manifest = await svc.candidateSummaries([resourceId]); + const transcript = manifest.find((c) => c.role === 'transcript')!; + const degraded = manifest.find((c) => c.role === 'degraded')!; + + const result = await svc.recallSelection( + [resourceId], + [transcript.entryId, degraded.entryId], + ); + + expect(result.entries.map((e) => e.entryId).sort()).toEqual( + [transcript.entryId, degraded.entryId].sort(), + ); + expect(result.status).not.toBe('miss'); + // The selected derived artifact is session-bound like an active + // recall would bind it. + const degradedEntry = result.entries.find( + (e) => e.entryId === degraded.entryId, + )!; + expect(degradedEntry.resourceId).toBeDefined(); + expect(registry.resolve(degradedEntry.resourceId!)?.fileRef).toBe( + degradedPath, + ); + }); + + it('materializes a repeated pick once', async () => { + const source = await recognizeMovie(); + await commitTranscript(source); + const resourceId = bindSource(source); + const svc = recallService(); + const manifest = await svc.candidateSummaries([resourceId]); + const entryId = manifest[0]!.entryId; + + const result = await svc.recallSelection([resourceId], [entryId, entryId]); + + // One pick, not two copies spending the budget on the same content. + expect(result.entries.map((e) => e.entryId)).toEqual([entryId]); + }); + + it('rejects an entryId outside the manifest wholesale', async () => { + const source = await recognizeMovie(); + await commitTranscript(source); + const resourceId = bindSource(source); + const svc = recallService(); + const manifest = await svc.candidateSummaries([resourceId]); + + await expect( + svc.recallSelection( + [resourceId], + [manifest[0]!.entryId, 'entry-not-in-manifest'], + ), + ).rejects.toMatchObject({ reason: 'invalid_selection' }); + }); + + it('rejects a real entryId that sits beyond the manifest cap', async () => { + const source = await recognizeMovie(); + await commitDegrade(source); + await commitTranscript(source); + const resourceId = bindSource(source); + const config = recallConfig({ + sideQuery: { + ...DEFAULT_OMNI_MEMORY_CONFIG.recall.sideQuery, + maxCandidateEntries: 2, + }, + }); + const svc = recallService(config); + const shown = await svc.candidateSummaries([resourceId]); + const hidden = (await recallService().candidateSummaries([resourceId])).at( + -1, + )!.entryId; + expect(shown.map((c) => c.entryId)).not.toContain(hidden); + + // The manifest IS the budget: validating against the uncapped candidate + // set would let a selector materialize entries this turn never showed + // it — a manifest cached from an earlier turn, or a guessed id that + // happens to exist — quietly spending context on rows the operator's + // cap had ruled out. + await expect( + svc.recallSelection([resourceId], [hidden]), + ).rejects.toMatchObject({ reason: 'invalid_selection' }); + }); + + it('rejects a selection over maxSelectedEntries wholesale', async () => { + const source = await recognizeMovie(); + await commitDegrade(source); + await commitTranscript(source); + const resourceId = bindSource(source); + const config = recallConfig({ + sideQuery: { + ...DEFAULT_OMNI_MEMORY_CONFIG.recall.sideQuery, + maxSelectedEntries: 1, + }, + }); + const svc = recallService(config); + const manifest = await svc.candidateSummaries([resourceId]); + + await expect( + svc.recallSelection( + [resourceId], + manifest.slice(0, 2).map((c) => c.entryId), + ), + ).rejects.toMatchObject({ reason: 'invalid_selection' }); + }); +}); diff --git a/packages/core/src/services/media-memory/recall.ts b/packages/core/src/services/media-memory/recall.ts new file mode 100644 index 00000000000..35a5985ee4b --- /dev/null +++ b/packages/core/src/services/media-memory/recall.ts @@ -0,0 +1,1075 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import fs from 'node:fs/promises'; +import { createDebugLogger } from '../../utils/debugLogger.js'; +import type { OmniModality } from '../../omni/recognition.js'; +import type { + NormalizedOmniMemoryRecall, + OmniMemoryRecallKind, +} from './config.js'; +import type { MediaResourceRegistry } from './registry.js'; +import { MediaMemoryStore } from './store.js'; +import type { + MediaChannel, + MediaCoverage, + MediaFileId, + MediaFileVersionId, + MediaFileVersionRecord, + MediaMemoryEntryId, + MediaMemorySnapshot, + MediaPolicyExecutionRecord, + MediaScope, + NormalizedPolicyOutput, + PolicyExecutionId, +} from './types.js'; + +const debugLogger = createDebugLogger('omni:memory'); + +// ─── Recall protocol (M §9.2 request / §9.4 minimal return) ────────────── + +/** One recall request, active tool and sideQuery alike (M §9.2). Every + * resourceId must have been issued by THIS session's registry — an + * unknown handle rejects the whole request (never partial fulfilment of + * a request that references resources the session was not given). */ +export interface MediaMemoryRecallRequest { + resourceIds: string[]; + /** Free-text information need. v1 has no semantic index: the query + * only orders structurally-matched entries (term overlap), it never + * widens or narrows the match set. */ + query: string; + kinds?: OmniMemoryRecallKind[]; + roles?: string[]; + scope?: MediaScope; + includeHistoricalVersions?: boolean; + limit?: number; +} + +export interface MediaMemoryRecallFile { + fileId: MediaFileId; + fileVersionId: MediaFileVersionId; + /** Whether this is the file's CURRENT_VERSION. A stale bound version + * listed with `current: false` is the explicit history hint of §9.5. */ + current: boolean; + mediaType: OmniModality; +} + +export interface MediaMemoryRecallProvenance { + toolName?: string; + toolVersion?: string; + policyId?: string; + stage?: 'preprocessing' | 'transport_guard'; + omniConfigHash: string; +} + +export interface MediaMemoryRecallEntry { + entryId: MediaMemoryEntryId; + kind: OmniMemoryRecallKind; + role?: string; + /** Bounded text payload (`recall.maxTextChars`): inline text for + * policy outputs, a compact technical summary for synthesized + * metadata/execution entries. */ + content?: string; + /** Fidelity disclosure recorded at collection time — travels with the + * content so degraded derivatives are never mistaken for originals. */ + disclosure?: string; + /** + * True when `content` is a PREFIX of what memory holds, cut by + * `recall.maxTextChars`. Coverage speaks for what was PROCESSED, so a + * transcript entry legitimately reports `complete` — without this flag + * the model reads a truncated transcript, sees complete coverage and no + * gap, and answers about late audio it never saw. Defaults collide here + * by design: maxTextChars (24000 chars) is smaller than the collection + * bound (65536 bytes), so any long transcript is cut at READ time. + */ + contentTruncated?: boolean; + /** Session handle for a derived media artifact, freshly bound through + * the registry (M §5.2). Absent when the artifact is gone. */ + resourceId?: string; + scope: MediaScope; + channels: MediaChannel[]; + coverage: MediaCoverage; + evidenceRefs: Array<{ + fileVersionId: MediaFileVersionId; + executionId?: PolicyExecutionId; + }>; + provenance: MediaMemoryRecallProvenance; +} + +export interface MediaMemoryRecallGap { + scope: MediaScope; + channels: MediaChannel[]; + reason: 'not_processed' | 'partial_coverage' | 'artifact_unavailable'; +} + +export interface MediaMemoryNextPolicyAction { + toolName: string; + resourceId: string; + arguments: Record; + reason: string; +} + +export interface MediaMemoryRecallResult { + status: 'hit' | 'partial' | 'miss'; + files: MediaMemoryRecallFile[]; + entries: MediaMemoryRecallEntry[]; + gaps: MediaMemoryRecallGap[]; + nextPolicyActions?: MediaMemoryNextPolicyAction[]; + /** + * Total entries that matched, present ONLY when the entry budget cut the + * list short. Without it a truncated page is indistinguishable from an + * exhaustive one: a real audit read 6 clips under `limit: 12` and + * concluded "no keyframes were ever extracted" while the store held 72 + * of them. The reader was being honest about what it saw — it simply had + * no way to know it was looking at a page. + */ + matchedEntries?: number; +} + +/** One candidate-manifest row for the sideQuery selector (M §9.3): + * enough structure to judge relevance — never the raw media, the full + * text, a local path, or a secret. */ +export interface MediaMemoryCandidateSummary { + entryId: MediaMemoryEntryId; + kind: OmniMemoryRecallKind; + role?: string; + scope: MediaScope; + channels: MediaChannel[]; + coverage: MediaCoverage; + /** Bounded content preview. */ + description?: string; + /** Producing tool, when known. */ + producer?: string; +} + +/** Character budget for one manifest row's content preview. */ +const CANDIDATE_DESCRIPTION_MAX_CHARS = 200; + +/** Whole-request rejection (M §9.2): the request itself is invalid — + * distinct from a valid request that finds nothing (`status: 'miss'`). + * `invalid_selection` is the sideQuery variant (M §9.3): the selector + * returned an entryId outside the candidate manifest (unknown, cross-root + * — the manifest is root-bounded by construction) or over budget. */ +export class MediaMemoryRecallRejection extends Error { + constructor( + readonly reason: 'empty_request' | 'unknown_resource' | 'invalid_selection', + message: string, + ) { + super(message); + this.name = 'MediaMemoryRecallRejection'; + } +} + +/** Hook for suggesting `nextPolicyActions` from a gap. Recall itself + * cannot name tools — which media-policy tools exist is session + * configuration the caller owns — so suggestions only appear when the + * wiring layer supplies an advisor built from the live tool registry. */ +export type MediaMemoryRecallAdvisor = (input: { + resourceId: string; + mediaType: OmniModality; + gap: MediaMemoryRecallGap; +}) => MediaMemoryNextPolicyAction[]; + +// ─── Internals ──────────────────────────────────────────────────────────── + +/** Truncate to a character budget without splitting a surrogate pair. + * (`maxTextChars` is a UTF-16 length bound, unlike the byte-bounded + * collection-side truncateUtf8.) */ +function truncateChars(text: string, maxChars: number): string { + if (text.length <= maxChars) return text; + let cut = text.slice(0, maxChars); + const last = cut.charCodeAt(cut.length - 1); + if (last >= 0xd800 && last <= 0xdbff) cut = cut.slice(0, -1); + return cut; +} + +/** Channels a fully-processed version of this modality would cover. + * Conservative v1 baseline: onscreen_text (OCR) is not expected by + * default — its absence is not reported as a gap. */ +function expectedChannels(mediaType: OmniModality): MediaChannel[] { + switch (mediaType) { + case 'image': + return ['visual']; + case 'audio': + return ['acoustic', 'speech_text']; + case 'video': + return ['visual', 'acoustic', 'speech_text']; + default: + return []; + } +} + +/** Character classes that write without spaces between words. A run of + * these cannot be split on separators, so it is indexed as overlapping + * character bigrams instead. */ +const UNSEGMENTED = + /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/u; + +/** Split a separator-delimited run into maximal same-class segments, so a + * mixed run like `480p字幕` yields `480p` and `字幕` rather than bigrams + * that straddle the boundary. */ +const SEGMENT_RUN = + /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]+|[^\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]+/gu; + +/** + * Query terms for relevance ordering. + * + * Splitting on separators alone works for languages that write with them + * and fails completely for those that do not: `机器人的梦想 剧情` became the + * two whole phrases `机器人的梦想` and `剧情`, each scored by substring + * containment — so unless an entry repeated the caller's exact phrasing, + * every candidate scored zero and ordering silently collapsed to + * newest-first. Runs in unsegmented scripts therefore become overlapping + * character bigrams, which match partial phrasings the way word tokens do. + */ +function tokenize(query: string): string[] { + const tokens = new Set(); + for (const run of query.toLowerCase().split(/[^\p{L}\p{N}]+/u)) { + for (const segment of run.match(SEGMENT_RUN) ?? []) { + if (!UNSEGMENTED.test(segment)) { + if (segment.length > 1) tokens.add(segment); + continue; + } + const chars = [...segment]; + if (chars.length === 1) { + tokens.add(segment); + continue; + } + for (let i = 0; i + 1 < chars.length; i++) { + tokens.add(chars[i] + chars[i + 1]); + } + } + } + return [...tokens]; +} + +/** Temporal-overlap scope filter. Entries without a temporal scope speak + * for the whole version and always pass. Non-temporal scope dimensions + * are not filtered in v1. */ +function scopeMatches( + requested: MediaScope | undefined, + entryScope: MediaScope, +): boolean { + const want = requested?.temporal; + const have = entryScope.temporal; + if (!want || !have) return true; + return have.startMs < want.endMs && have.endMs > want.startMs; +} + +interface CandidateEntry { + entry: MediaMemoryRecallEntry; + createdAt: string; + /** Resource the entry was recalled for (gap/advisor attribution). */ + forResourceId: string; + /** Set when a derived artifact must exist on disk to be bindable. */ + derived?: { + versionId: MediaFileVersionId; + fileId: MediaFileId; + fileRef: string; + mediaType: OmniModality; + }; +} + +/** The shared output of one snapshot walk (collectFromSnapshot). */ +interface CollectedRecall { + files: Map; + candidates: Map; + gaps: Array< + MediaMemoryRecallGap & { forResourceId: string; mediaType: OmniModality } + >; +} + +/** Deterministic manifest order (M §9.3): newest first, entryId + * tiebreak — the cap must cut the same rows on every run. */ +function orderCandidates(candidates: CandidateEntry[]): CandidateEntry[] { + return [...candidates].sort( + (a, b) => + b.createdAt.localeCompare(a.createdAt) || + a.entry.entryId.localeCompare(b.entry.entryId), + ); +} + +interface SnapshotIndexes { + childrenByParent: Map; + entriesByParent: Map; + executionsBySource: Map; + versionsByFile: Map; +} + +function indexSnapshot(snapshot: MediaMemorySnapshot): SnapshotIndexes { + const childrenByParent = new Map< + MediaFileVersionId, + MediaFileVersionRecord[] + >(); + const versionsByFile = new Map(); + for (const version of Object.values(snapshot.versions)) { + if (version.parentVersionId) { + const list = childrenByParent.get(version.parentVersionId) ?? []; + list.push(version); + childrenByParent.set(version.parentVersionId, list); + } + const byFile = versionsByFile.get(version.fileId) ?? []; + byFile.push(version); + versionsByFile.set(version.fileId, byFile); + } + const entriesByParent = new Map< + MediaFileVersionId, + NormalizedPolicyOutput[] + >(); + for (const entry of Object.values(snapshot.entries)) { + const list = entriesByParent.get(entry.parentVersionId) ?? []; + list.push(entry); + entriesByParent.set(entry.parentVersionId, list); + } + const executionsBySource = new Map< + MediaFileVersionId, + MediaPolicyExecutionRecord[] + >(); + for (const execution of Object.values(snapshot.executions)) { + const list = executionsBySource.get(execution.sourceVersionId) ?? []; + list.push(execution); + executionsBySource.set(execution.sourceVersionId, list); + } + return { + childrenByParent, + entriesByParent, + executionsBySource, + versionsByFile, + }; +} + +/** All versions reachable from `start` over DERIVED_FROM child edges, + * bounded by the binding's root (M §8: every traversal stays inside one + * root graph — a version filed under another root is never followed). */ +function derivedSubgraph( + snapshot: MediaMemorySnapshot, + indexes: SnapshotIndexes, + start: MediaFileVersionRecord, + rootFileId: MediaFileId, +): MediaFileVersionRecord[] { + const result: MediaFileVersionRecord[] = []; + const visited = new Set(); + const queue: MediaFileVersionRecord[] = [start]; + while (queue.length > 0) { + const version = queue.shift() as MediaFileVersionRecord; + if (visited.has(version.fileVersionId)) continue; + visited.add(version.fileVersionId); + const file = snapshot.files[version.fileId]; + if (!file || file.rootFileId !== rootFileId) continue; + result.push(version); + for (const child of indexes.childrenByParent.get(version.fileVersionId) ?? + []) { + queue.push(child); + } + } + return result; +} + +function provenanceOf( + execution: MediaPolicyExecutionRecord | undefined, +): MediaMemoryRecallProvenance { + if (!execution) return { omniConfigHash: '' }; + const origin = execution.executionOrigin; + return { + toolName: execution.toolName, + ...(execution.toolVersion !== undefined + ? { toolVersion: execution.toolVersion } + : {}), + ...(origin.kind === 'fixed_policy' + ? { policyId: origin.policyId, stage: origin.stage } + : {}), + omniConfigHash: execution.omniConfigHash, + }; +} + +// ─── Recall service ─────────────────────────────────────────────────────── + +/** + * The read side of multimodal media memory (M §9): given session resource + * handles, return what memory already knows about the underlying media — + * current-version-first (§9.5), bounded to each resource's root graph + * (§8), with honest gaps for what was never processed or is no longer on + * disk. Both recall surfaces (the active `omni_recall_media_memory` tool + * and the passive sideQuery selector) sit on this one service. + * + * Read-only by constitution (M §14 / D11): the service holds a store but + * only ever calls `read`. The only state it mutates is the session + * registry, binding derived artifacts so the model can reference them. + */ +export class MediaMemoryRecallService { + private readonly store: MediaMemoryStore; + + constructor( + omniRootDir: string, + private readonly config: NormalizedOmniMemoryRecall, + private readonly registry: MediaResourceRegistry, + private readonly options?: { advise?: MediaMemoryRecallAdvisor }, + ) { + this.store = new MediaMemoryStore(omniRootDir); + } + + /** + * Execute one recall request. Throws {@link MediaMemoryRecallRejection} + * when the request itself is invalid (empty, or referencing a handle + * this session never issued); an unreadable store degrades to a plain + * miss — recall is an enhancement and must never break the caller. + */ + async recall( + request: MediaMemoryRecallRequest, + ): Promise { + const bindings = this.resolveBindings(request.resourceIds); + const miss: MediaMemoryRecallResult = { + status: 'miss', + files: [], + entries: [], + gaps: [], + }; + try { + return await this.store.read(miss, (snapshot) => + this.recallFromSnapshot(snapshot, request, bindings), + ); + } catch (err) { + debugLogger.debug( + `recall failed: ${err instanceof Error ? err.message : err}`, + ); + return miss; + } + } + + /** + * Bounded candidate manifest for the sideQuery selector (M §9.3): every + * entry reachable from the named resources — same root-bounded walk as + * {@link recall}, configured kinds, current-version-first — summarized + * without full text/paths and capped at `sideQuery.maxCandidateEntries` + * (newest first; deterministic tiebreak). Rejections propagate like + * {@link recall}; an unreadable store degrades to an empty manifest. + */ + async candidateSummaries( + resourceIds: string[], + ): Promise { + const bindings = this.resolveBindings(resourceIds); + const request: MediaMemoryRecallRequest = { resourceIds, query: '' }; + // No catch-all here: an unreadable/absent store already resolves to the + // empty manifest inside `read`, so anything reaching this frame is a + // defect in the walk. The caller records it as `manifest_failed` and + // sends the main request anyway — swallowing it here would only cost us + // the stack trace. + return await this.store.read( + [] as MediaMemoryCandidateSummary[], + async (snapshot) => { + const { candidates } = await this.collectFromSnapshot( + snapshot, + request, + bindings, + ); + return orderCandidates([...candidates.values()]) + .slice(0, this.config.sideQuery.maxCandidateEntries) + .map((candidate) => ({ + entryId: candidate.entry.entryId, + kind: candidate.entry.kind, + ...(candidate.entry.role !== undefined + ? { role: candidate.entry.role } + : {}), + scope: candidate.entry.scope, + channels: candidate.entry.channels, + coverage: candidate.entry.coverage, + ...(candidate.entry.content !== undefined + ? { + description: truncateChars( + candidate.entry.content, + CANDIDATE_DESCRIPTION_MAX_CHARS, + ), + } + : {}), + ...(candidate.entry.provenance.toolName !== undefined + ? { producer: candidate.entry.provenance.toolName } + : {}), + })); + }, + ); + } + + /** + * Materialize a selector's picks by the unified protocol (M §9.3): the + * selection must be a subset of the manifest {@link candidateSummaries} + * would produce for the same resources and within + * `sideQuery.maxSelectedEntries` — anything else (unknown id, cross-root + * id, over budget) rejects the WHOLE selection with `invalid_selection`, + * never a partial fulfilment. + */ + async recallSelection( + resourceIds: string[], + entryIds: string[], + ): Promise { + const bindings = this.resolveBindings(resourceIds); + // A repeated pick is one pick: a selector naming the same entry twice + // asked for the same content, and materializing it twice would spend + // the budget on a duplicate. + const selectedIds = [...new Set(entryIds)]; + if (selectedIds.length > this.config.sideQuery.maxSelectedEntries) { + throw new MediaMemoryRecallRejection( + 'invalid_selection', + `selector returned ${selectedIds.length} entryIds; at most ` + + `${this.config.sideQuery.maxSelectedEntries} may be selected`, + ); + } + const request: MediaMemoryRecallRequest = { resourceIds, query: '' }; + return await this.store.read( + { + status: 'miss', + files: [], + entries: [], + gaps: [], + } satisfies MediaMemoryRecallResult, + async (snapshot) => { + const collected = await this.collectFromSnapshot( + snapshot, + request, + bindings, + ); + // The manifest the selector saw is the ordered, capped view — + // validate against exactly that, so an id beyond the cap (which + // the selector was never shown) rejects like any unknown id. + const manifest = new Map( + orderCandidates([...collected.candidates.values()]) + .slice(0, this.config.sideQuery.maxCandidateEntries) + .map((candidate) => [candidate.entry.entryId, candidate]), + ); + const selected: CandidateEntry[] = []; + for (const entryId of selectedIds) { + const candidate = manifest.get(entryId); + if (!candidate) { + throw new MediaMemoryRecallRejection( + 'invalid_selection', + `selector returned entryId ${entryId} that is not in the ` + + `candidate manifest`, + ); + } + selected.push(candidate); + } + return this.finishResult(snapshot, selected, collected); + }, + ); + } + + /** Shared request validation (M §9.2): every handle must have been + * issued by THIS session's registry; an empty list or an unknown handle + * rejects the whole request. */ + private resolveBindings( + resourceIds: string[], + ): Array>> { + if (resourceIds.length === 0) { + throw new MediaMemoryRecallRejection( + 'empty_request', + 'recall request must name at least one resourceId', + ); + } + // Deduplicated: gaps are pushed per binding and the advisor runs per + // gap, so a handle repeated in one request would otherwise yield + // identical duplicate gaps and duplicate suggested actions — telling + // the model to run the same follow-up call twice on the same resource. + return [...new Set(resourceIds)].map((resourceId) => { + const binding = this.registry.resolve(resourceId); + if (!binding) { + throw new MediaMemoryRecallRejection( + 'unknown_resource', + `resourceId ${resourceId} was not issued in this session`, + ); + } + return binding; + }); + } + + private async recallFromSnapshot( + snapshot: MediaMemorySnapshot, + request: MediaMemoryRecallRequest, + bindings: ReadonlyArray>, + ): Promise { + const collected = await this.collectFromSnapshot( + snapshot, + request, + bindings, + ); + const limit = Math.min( + request.limit ?? this.config.maxEntries, + this.config.maxEntries, + ); + const entries = rankAndSlice( + [...collected.candidates.values()], + request.query, + limit, + ); + return this.finishResult(snapshot, entries, collected); + } + + /** The shared walk both recall shapes sit on: resolve each binding to + * its file, list consulted versions (current-first §9.5), collect + * candidate entries across the root-bounded derivation subgraph, and + * derive the CURRENT version's honest gaps. */ + private async collectFromSnapshot( + snapshot: MediaMemorySnapshot, + request: MediaMemoryRecallRequest, + bindings: ReadonlyArray>, + ): Promise { + const indexes = indexSnapshot(snapshot); + const kinds = this.effectiveKinds(request); + const includeHistorical = + request.includeHistoricalVersions ?? + this.config.includeHistoricalVersions; + + const files = new Map(); + const candidates = new Map(); + const gaps: CollectedRecall['gaps'] = []; + + for (const binding of bindings) { + if (!binding) continue; // narrowed by the caller; keeps types honest + const boundVersion = snapshot.versions[binding.fileVersionId]; + const file = boundVersion + ? snapshot.files[boundVersion.fileId] + : undefined; + if (!boundVersion || !file) { + // The persistent graph no longer holds this binding (store was + // corrupt-rebuilt). The session resource still exists — report + // the memory about it as unavailable rather than erroring. + gaps.push({ + scope: {}, + channels: expectedChannels(binding.mediaType), + reason: 'artifact_unavailable', + forResourceId: binding.resourceId, + mediaType: binding.mediaType, + }); + continue; + } + + // Current-version-first (§9.5): consult the CURRENT version's graph + // by default; historical versions only on explicit request. + const fileVersions = indexes.versionsByFile.get(file.fileId) ?? []; + const currentVersion = snapshot.versions[file.currentVersionId]; + const consulted = includeHistorical + ? [...fileVersions].sort((a, b) => + b.createdAt.localeCompare(a.createdAt), + ) + : currentVersion + ? [currentVersion] + : []; + + for (const version of consulted) { + files.set(version.fileVersionId, { + fileId: file.fileId, + fileVersionId: version.fileVersionId, + current: version.fileVersionId === file.currentVersionId, + mediaType: version.mediaType, + }); + } + // Explicit history hint (§9.5): the handle points at content that + // is no longer current — list it as such even when not consulted. + if (!files.has(binding.fileVersionId)) { + files.set(binding.fileVersionId, { + fileId: file.fileId, + fileVersionId: binding.fileVersionId, + current: false, + mediaType: boundVersion.mediaType, + }); + } + + for (const version of consulted) { + await this.collectVersion( + snapshot, + indexes, + binding.resourceId, + binding.rootFileId, + version, + kinds, + request, + candidates, + ); + } + + // Gaps speak for the CURRENT version's processing state, always — + // history inclusion widens the entries, not the obligation. + if (currentVersion) { + gaps.push( + ...(await this.gapsForVersion( + snapshot, + indexes, + binding.resourceId, + binding.rootFileId, + file.fileRef, + currentVersion, + )), + ); + } + } + + return { files, candidates, gaps }; + } + + /** Availability pass + assembly shared by both recall shapes: bind a + * session handle for each returned derived artifact still on disk, + * degrade the lost ones to gaps, derive advisor suggestions and the + * hit/partial/miss verdict. */ + private async finishResult( + snapshot: MediaMemorySnapshot, + entries: CandidateEntry[], + collected: CollectedRecall, + ): Promise { + const { files, gaps } = collected; + // Artifact-availability pass for the derived media actually returned: + // bind a session handle when the object is still on disk; otherwise + // surface the loss as a gap instead of handing out a dead handle. + for (const candidate of entries) { + const derived = candidate.derived; + if (!derived) continue; + if (await pathExists(derived.fileRef)) { + candidate.entry.resourceId = this.registry.bind({ + fileId: derived.fileId, + fileVersionId: derived.versionId, + rootFileId: + snapshot.files[derived.fileId]?.rootFileId ?? derived.fileId, + fileRef: derived.fileRef, + mediaType: derived.mediaType, + }).resourceId; + } else { + gaps.push({ + scope: candidate.entry.scope, + channels: candidate.entry.channels, + reason: 'artifact_unavailable', + forResourceId: candidate.forResourceId, + mediaType: derived.mediaType, + }); + } + } + + const resultEntries = entries.map((c) => c.entry); + const resultGaps: MediaMemoryRecallGap[] = gaps.map( + ({ scope, channels, reason }) => ({ scope, channels, reason }), + ); + const nextPolicyActions = this.options?.advise + ? gaps.flatMap((gap) => + this.options!.advise!({ + resourceId: gap.forResourceId, + mediaType: gap.mediaType, + gap: { + scope: gap.scope, + channels: gap.channels, + reason: gap.reason, + }, + }), + ) + : []; + + const status: MediaMemoryRecallResult['status'] = + resultEntries.length === 0 + ? 'miss' + : resultGaps.length > 0 + ? 'partial' + : 'hit'; + + // Only when the budget actually cut something: an exhaustive page stays + // as small as it was before this field existed (§9.4 minimal return). + const matched = collected.candidates.size; + return { + status, + files: [...files.values()], + entries: resultEntries, + gaps: resultGaps, + ...(nextPolicyActions.length > 0 ? { nextPolicyActions } : {}), + ...(matched > resultEntries.length ? { matchedEntries: matched } : {}), + }; + } + + /** Request kinds narrowed to what configuration allows; an absent + * request filter means "everything configured". */ + private effectiveKinds( + request: MediaMemoryRecallRequest, + ): Set { + const allowed = new Set(this.config.kinds); + if (!request.kinds) return allowed; + return new Set(request.kinds.filter((k) => allowed.has(k))); + } + + /** Collect every matching entry reachable from one consulted version: + * synthesized metadata for the version itself, then policy outputs and + * executions across its whole derivation subgraph (the transcript of an + * extracted audio track parents on the AUDIO version — only the + * subgraph walk surfaces it for the movie's handle). */ + private async collectVersion( + snapshot: MediaMemorySnapshot, + indexes: SnapshotIndexes, + forResourceId: string, + rootFileId: MediaFileId, + version: MediaFileVersionRecord, + kinds: Set, + request: MediaMemoryRecallRequest, + out: Map, + ): Promise { + const rolesFilter = request.roles; + if (kinds.has('metadata') && !rolesFilter) { + const entryId = `metadata:${version.fileVersionId}`; + out.set(entryId, { + entry: { + entryId, + kind: 'metadata', + content: truncateChars( + JSON.stringify({ + mediaType: version.mediaType, + mimeType: version.mimeType, + sizeBytes: version.sizeBytes, + source: version.source, + ...version.metadata, + }), + this.config.maxTextChars, + ), + scope: {}, + channels: ['technical_metadata'], + coverage: { mode: 'complete', scope: {} }, + evidenceRefs: [{ fileVersionId: version.fileVersionId }], + provenance: { + omniConfigHash: version.recognition.ingestionConfigHash, + }, + }, + createdAt: version.createdAt, + forResourceId, + }); + } + + for (const node of derivedSubgraph( + snapshot, + indexes, + version, + rootFileId, + )) { + if (kinds.has('derived_media') || kinds.has('policy_result')) { + for (const entry of indexes.entriesByParent.get(node.fileVersionId) ?? + []) { + if (!kinds.has(entry.kind)) continue; + if (rolesFilter && (!entry.role || !rolesFilter.includes(entry.role))) + continue; + if (!scopeMatches(request.scope, entry.scope)) continue; + const execution = snapshot.executions[entry.producedByExecutionId]; + const derivedVersion = entry.derivedVersionId + ? snapshot.versions[entry.derivedVersionId] + : undefined; + out.set(entry.outputId, { + entry: { + entryId: entry.outputId, + kind: entry.kind, + ...(entry.role !== undefined ? { role: entry.role } : {}), + ...(entry.inlineText !== undefined + ? { + content: truncateChars( + entry.inlineText, + this.config.maxTextChars, + ), + // Say so when the payload is a prefix: coverage speaks + // for what was processed, so it stays `complete` and + // no gap is raised — the flag is the only signal that + // the model is not holding the whole text. + ...(entry.inlineText.length > this.config.maxTextChars + ? { contentTruncated: true } + : {}), + } + : {}), + ...(entry.disclosure !== undefined + ? { disclosure: entry.disclosure } + : {}), + scope: entry.scope, + channels: entry.channels, + coverage: entry.coverage, + evidenceRefs: [ + { + fileVersionId: entry.parentVersionId, + executionId: entry.producedByExecutionId, + }, + ], + provenance: provenanceOf(execution), + }, + createdAt: entry.createdAt, + forResourceId, + ...(derivedVersion + ? { + derived: { + versionId: derivedVersion.fileVersionId, + fileId: derivedVersion.fileId, + fileRef: + snapshot.files[derivedVersion.fileId]?.fileRef ?? '', + mediaType: derivedVersion.mediaType, + }, + } + : {}), + }); + } + } + + if (kinds.has('execution') && !rolesFilter) { + for (const execution of indexes.executionsBySource.get( + node.fileVersionId, + ) ?? []) { + if (!scopeMatches(request.scope, execution.inputScope)) continue; + const entryId = `execution:${execution.executionId}`; + out.set(entryId, { + entry: { + entryId, + kind: 'execution', + content: truncateChars( + JSON.stringify({ + toolName: execution.toolName, + ...(execution.toolVersion !== undefined + ? { toolVersion: execution.toolVersion } + : {}), + finalArguments: execution.finalArguments, + completedAt: execution.completedAt, + outputCount: execution.outputRefs.length, + }), + this.config.maxTextChars, + ), + scope: execution.inputScope, + channels: [], + coverage: { mode: 'complete', scope: execution.inputScope }, + evidenceRefs: [ + { + fileVersionId: execution.sourceVersionId, + executionId: execution.executionId, + }, + ], + provenance: provenanceOf(execution), + }, + createdAt: execution.completedAt, + forResourceId, + }); + } + } + } + } + + /** Honest-gaps derivation (§9.4) for one CURRENT version: which of the + * modality's expected channels have no evidence (not_processed), only + * sampled/partial evidence (partial_coverage), and whether the source + * bytes themselves are still on disk (artifact_unavailable, D5). Gap + * truth is computed from the FULL subgraph, unfiltered — what the + * request chose to see never changes what was processed. */ + private async gapsForVersion( + snapshot: MediaMemorySnapshot, + indexes: SnapshotIndexes, + forResourceId: string, + rootFileId: MediaFileId, + fileRef: string, + version: MediaFileVersionRecord, + ): Promise< + Array< + MediaMemoryRecallGap & { forResourceId: string; mediaType: OmniModality } + > + > { + const gaps: Array< + MediaMemoryRecallGap & { forResourceId: string; mediaType: OmniModality } + > = []; + if (!(await pathExists(fileRef))) { + // The source bytes are gone (D5): report the loss and STOP. Continuing + // the channel scan would emit sibling `not_processed` gaps, and the + // advisor — which only filters `artifact_unavailable` — would then + // suggest evidence-gathering calls carrying a handle whose file no + // longer exists: the gate resolves it and `assertMediaPolicyIo` fails, + // a guaranteed-to-fail turn. Nothing can be gathered from a deleted + // file, so "unavailable" is the complete and only honest gap. + return [ + { + scope: {}, + channels: expectedChannels(version.mediaType), + reason: 'artifact_unavailable', + forResourceId, + mediaType: version.mediaType, + }, + ]; + } + + const complete = new Set(); + const partial = new Set(); + for (const node of derivedSubgraph( + snapshot, + indexes, + version, + rootFileId, + )) { + for (const entry of indexes.entriesByParent.get(node.fileVersionId) ?? + []) { + const full = + entry.coverage.mode === 'complete' || + entry.coverage.mode === 'continuous'; + for (const channel of entry.channels) { + (full ? complete : partial).add(channel); + } + } + } + + const notProcessed: MediaChannel[] = []; + const partialOnly: MediaChannel[] = []; + for (const channel of expectedChannels(version.mediaType)) { + if (complete.has(channel)) continue; + (partial.has(channel) ? partialOnly : notProcessed).push(channel); + } + if (notProcessed.length > 0) { + gaps.push({ + scope: {}, + channels: notProcessed, + reason: 'not_processed', + forResourceId, + mediaType: version.mediaType, + }); + } + if (partialOnly.length > 0) { + gaps.push({ + scope: {}, + channels: partialOnly, + reason: 'partial_coverage', + forResourceId, + mediaType: version.mediaType, + }); + } + return gaps; + } +} + +async function pathExists(fileRef: string): Promise { + if (!fileRef) return false; + try { + await fs.access(fileRef); + return true; + } catch { + return false; + } +} + +/** Order candidates by naive query-term overlap, then recency, then id + * (full determinism), and apply the entry budget. Ranking never filters: + * a zero-score entry still returns when the budget allows. */ +function rankAndSlice( + candidates: CandidateEntry[], + query: string, + limit: number, +): CandidateEntry[] { + const tokens = tokenize(query); + const scored = candidates.map((candidate) => { + const haystack = [ + candidate.entry.role ?? '', + candidate.entry.content ?? '', + candidate.entry.provenance.toolName ?? '', + ] + .join(' ') + .toLowerCase(); + let score = 0; + for (const token of tokens) { + if (haystack.includes(token)) score += 1; + } + return { candidate, score }; + }); + scored.sort( + (a, b) => + b.score - a.score || + b.candidate.createdAt.localeCompare(a.candidate.createdAt) || + a.candidate.entry.entryId.localeCompare(b.candidate.entry.entryId), + ); + return scored.slice(0, Math.max(0, limit)).map((s) => s.candidate); +} diff --git a/packages/core/src/services/media-memory/registry.test.ts b/packages/core/src/services/media-memory/registry.test.ts new file mode 100644 index 00000000000..e933395756e --- /dev/null +++ b/packages/core/src/services/media-memory/registry.test.ts @@ -0,0 +1,77 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { describe, expect, it } from 'vitest'; +import { MediaResourceRegistry } from './registry.js'; + +const BINDING = { + fileId: 'f-movie', + fileVersionId: 'v-movie-1', + rootFileId: 'f-movie', + fileRef: '/movies/breaking-surface.mkv', + mediaType: 'video' as const, +}; + +describe('MediaResourceRegistry', () => { + it('mints an opaque handle that resolves back to the binding', () => { + const registry = new MediaResourceRegistry(); + const bound = registry.bind(BINDING); + expect(bound.resourceId).toMatch(/^media-1-[0-9a-f]{8}$/); + expect(registry.resolve(bound.resourceId)).toEqual(bound); + expect(registry.resolveVersion('v-movie-1')).toEqual(bound); + }); + + it('is idempotent per fileVersionId', () => { + const registry = new MediaResourceRegistry(); + const first = registry.bind(BINDING); + const second = registry.bind(BINDING); + expect(second.resourceId).toBe(first.resourceId); + }); + + it('keeps the first binding when a version is rebound with different fields', () => { + // fileVersionId IS the identity here, so the version wins over the + // details: the binding a handle was minted with is the one the harness + // will resolve for the rest of the session. Overwriting `fileRef` in + // place would silently repoint a handle the model already holds at + // other bytes; minting a second handle for one version would break the + // correlation across recall calls that the idempotency exists for. + const registry = new MediaResourceRegistry(); + const first = registry.bind(BINDING); + const second = registry.bind({ + ...BINDING, + fileRef: '/objects/sha256/promoted.mp4', + mediaType: 'audio', + }); + + expect(second).toBe(first); + expect(second).toEqual({ ...BINDING, resourceId: first.resourceId }); + expect(registry.resolve(first.resourceId)).toMatchObject({ + fileRef: BINDING.fileRef, + mediaType: 'video', + }); + // The rejected locator was never indexed either, so a path-based + // recovery cannot resolve it back to this handle. + expect( + registry.resolveByFileRef('/objects/sha256/promoted.mp4'), + ).toBeUndefined(); + expect(registry.resolveByFileRef(BINDING.fileRef)).toBe(first); + }); + + it('issues distinct handles for distinct versions', () => { + const registry = new MediaResourceRegistry(); + const first = registry.bind(BINDING); + const second = registry.bind({ ...BINDING, fileVersionId: 'v-movie-2' }); + expect(second.resourceId).not.toBe(first.resourceId); + }); + + it('never resolves a handle it did not issue', () => { + const registry = new MediaResourceRegistry(); + registry.bind(BINDING); + expect(registry.resolve('media-1-deadbeef')).toBeUndefined(); + expect(registry.resolve('/movies/breaking-surface.mkv')).toBeUndefined(); + expect(registry.resolveVersion('v-unknown')).toBeUndefined(); + }); +}); diff --git a/packages/core/src/services/media-memory/registry.ts b/packages/core/src/services/media-memory/registry.ts new file mode 100644 index 00000000000..3e388b6ac5f --- /dev/null +++ b/packages/core/src/services/media-memory/registry.ts @@ -0,0 +1,92 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { randomBytes } from 'node:crypto'; +import type { OmniModality } from '../../omni/recognition.js'; +import type { MediaFileId, MediaFileVersionId } from './types.js'; + +/** + * One session-scoped binding between a persistent memory identity and the + * opaque handle the model is allowed to see (M §5.2): recall rebinds a + * persistent `fileVersionId` into the session registry and returns the + * `resourceId`; the model passes that handle to media-policy tools, and + * the harness resolves it back to a real locator at execution time. + */ +export interface MediaResourceBinding { + /** Opaque session handle — the ONLY identity the model sees. */ + resourceId: string; + fileId: MediaFileId; + fileVersionId: MediaFileVersionId; + rootFileId: MediaFileId; + /** Harness-side locator (absolute local path or promoted object path). + * Consumed when a tool call resolves resourceId → inputPath; NEVER + * included in a model-visible payload (M §5.2/§15). */ + fileRef: string; + mediaType: OmniModality; +} + +/** + * Session-lifetime bidirectional binder: fileVersionId ↔ resourceId. + * One instance per CLI session (hung off Config); bindings never persist + * — a resourceId is only meaningful inside the session that minted it, + * which is what keeps recall payloads free of stable path-like handles. + * + * Binding is idempotent per version: re-recalling the same derivative in + * one session returns the handle already issued, so the model can + * correlate results across recall calls. + */ +export class MediaResourceRegistry { + private readonly byResourceId = new Map(); + private readonly byVersionId = new Map< + MediaFileVersionId, + MediaResourceBinding + >(); + private counter = 0; + + /** Bind (or return the existing binding of) one file version. */ + bind(input: Omit): MediaResourceBinding { + const existing = this.byVersionId.get(input.fileVersionId); + if (existing) return existing; + // Sequential prefix keeps handles readable in transcripts; the random + // suffix stops the model from guessing handles it was never given. + const resourceId = `media-${++this.counter}-${randomBytes(4).toString('hex')}`; + const binding: MediaResourceBinding = { resourceId, ...input }; + this.byResourceId.set(resourceId, binding); + this.byVersionId.set(input.fileVersionId, binding); + return binding; + } + + /** Resolve a model-supplied handle. Undefined = never issued in this + * session (an unauthorized or fabricated id — callers must reject). */ + resolve(resourceId: string): MediaResourceBinding | undefined { + return this.byResourceId.get(resourceId); + } + + /** Look up the binding already issued for a version, if any. */ + resolveVersion( + fileVersionId: MediaFileVersionId, + ): MediaResourceBinding | undefined { + return this.byVersionId.get(fileVersionId); + } + + /** Look up a binding by its harness-side locator. Lets collection paths + * that only hold a resolved `inputPath` (a gated model call, where the + * gate already turned the handle back into a path) recover the memory + * identity without re-hashing the file. */ + resolveByFileRef(fileRef: string): MediaResourceBinding | undefined { + for (const binding of this.byVersionId.values()) { + if (binding.fileRef === fileRef) return binding; + } + return undefined; + } +} + +/** Structural view of the Config accessor (same pattern as + * `OmniMemoryConfigView`): a config without the accessor — stub configs, + * embedders skipping initialize — reads as "no session registry". */ +export interface OmniMediaRegistryView { + getOmniMediaResourceRegistry?: () => MediaResourceRegistry; +} diff --git a/packages/core/src/services/media-memory/service.test.ts b/packages/core/src/services/media-memory/service.test.ts new file mode 100644 index 00000000000..ced24b3c3be --- /dev/null +++ b/packages/core/src/services/media-memory/service.test.ts @@ -0,0 +1,544 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { + MediaMemoryService, + type FileRecognizedEvent, + type PolicySucceededInput, +} from './index.js'; +import { truncateUtf8 } from './service.js'; +import { MEDIA_MEMORY_FILE_NAME } from './store.js'; +import type { MediaMemorySnapshot } from './types.js'; + +let root: string; +let clock: number; + +/** Deterministic, strictly increasing timestamps per commit. */ +class TestMediaMemoryService extends MediaMemoryService { + protected override now(): string { + return new Date(clock++).toISOString(); + } +} + +let service: TestMediaMemoryService; + +beforeEach(async () => { + root = await fs.mkdtemp(path.join(os.tmpdir(), 'omni-memory-svc-')); + clock = 1_754_870_400_000; // 2025-08-11T00:00:00Z, arbitrary fixed base + service = new TestMediaMemoryService(root); +}); + +afterEach(async () => { + await fs + .chmod(path.join(root, MEDIA_MEMORY_FILE_NAME), 0o600) + .catch(() => {}); + await fs.rm(root, { recursive: true, force: true }); +}); + +const canDropPermissions = + process.platform !== 'win32' && + (typeof process.getuid !== 'function' || process.getuid() !== 0); + +async function readSnapshot(): Promise { + const raw = await fs.readFile( + path.join(root, MEDIA_MEMORY_FILE_NAME), + 'utf8', + ); + return JSON.parse(raw) as MediaMemorySnapshot; +} + +const SHA_A = 'a'.repeat(64); +const SHA_B = 'b'.repeat(64); +const SHA_OUT = 'c'.repeat(64); +const SHA_TEXT = 'd'.repeat(64); + +function recognizedEvent( + overrides?: Partial, +): FileRecognizedEvent { + return { + fileRef: '/movies/breaking-surface.mkv', + sha256: SHA_A, + mediaType: 'video', + metadata: { durationMs: 4_860_000, width: 1920, height: 1080 }, + sizeBytes: 123_456_789, + mimeType: 'video/x-matroska', + origin: 'user', + source: { protocol: 'local', locator: 'breaking-surface.mkv' }, + recognition: { + ingestionConfigHash: '', + detectorVersion: 'omni-sniff-ffprobe/1', + probeStatus: 'complete', + }, + ...overrides, + }; +} + +function succeededInput( + source: { fileId: string; fileVersionId: string; rootFileId: string }, + overrides?: Partial, +): PolicySucceededInput { + return { + invocationId: 'deadbeef01234567', + source, + executionOrigin: { + kind: 'fixed_policy', + policyId: 'downscale-video', + stage: 'preprocessing', + }, + toolName: 'omni_downscale_video', + toolVersion: '1', + finalArguments: { maxHeight: 480, fps: 1 }, + omniConfigHash: 'fp-' + '0'.repeat(61), + startedAt: '2026-08-11T00:00:00.000Z', + completedAt: '2026-08-11T00:00:05.000Z', + outputs: [ + { + kind: 'media', + objectPath: `/store/objects/${SHA_OUT}.mp4`, + sha256: SHA_OUT, + mediaType: 'video', + metadata: { durationMs: 4_860_000, width: 854, height: 480 }, + sizeBytes: 10_000_000, + mimeType: 'video/mp4', + disclosure: 'downscaled to 480p/1fps', + }, + ], + ...overrides, + }; +} + +describe('MediaMemoryService.recordFileRecognized', () => { + it('creates file + version and is idempotent for identical content', async () => { + const first = await service.recordFileRecognized(recognizedEvent()); + expect(first).toBeDefined(); + expect(first!.created).toBe(true); + expect(first!.rootFileId).toBe(first!.fileId); + + const second = await service.recordFileRecognized(recognizedEvent()); + expect(second).toMatchObject({ + fileId: first!.fileId, + fileVersionId: first!.fileVersionId, + created: false, + }); + + const snapshot = await readSnapshot(); + expect(Object.keys(snapshot.files)).toHaveLength(1); + expect(Object.keys(snapshot.versions)).toHaveLength(1); + }); + + it('creates a new version on content change and moves CURRENT_VERSION both ways', async () => { + const v1 = await service.recordFileRecognized(recognizedEvent()); + const v2 = await service.recordFileRecognized( + recognizedEvent({ sha256: SHA_B }), + ); + expect(v2!.fileId).toBe(v1!.fileId); + expect(v2!.fileVersionId).not.toBe(v1!.fileVersionId); + expect(v2!.created).toBe(true); + + let snapshot = await readSnapshot(); + expect(snapshot.files[v1!.fileId].currentVersionId).toBe(v2!.fileVersionId); + + // Revert on disk: the pointer moves back, no third version appears. + const reverted = await service.recordFileRecognized(recognizedEvent()); + expect(reverted).toMatchObject({ + fileVersionId: v1!.fileVersionId, + created: false, + }); + snapshot = await readSnapshot(); + expect(snapshot.files[v1!.fileId].currentVersionId).toBe(v1!.fileVersionId); + expect(Object.keys(snapshot.versions)).toHaveLength(2); + }); + + it('keeps two files with identical bytes as two distinct records (M §11)', async () => { + const a = await service.recordFileRecognized(recognizedEvent()); + const b = await service.recordFileRecognized( + recognizedEvent({ fileRef: '/movies/copy.mkv' }), + ); + expect(b!.fileId).not.toBe(a!.fileId); + expect(b!.fileVersionId).not.toBe(a!.fileVersionId); + const snapshot = await readSnapshot(); + expect(Object.keys(snapshot.files)).toHaveLength(2); + }); + + it.runIf(canDropPermissions)( + 'returns undefined instead of throwing when persistence fails', + async () => { + await service.recordFileRecognized(recognizedEvent()); + await fs.chmod(path.join(root, MEDIA_MEMORY_FILE_NAME), 0o000); + const result = await service.recordFileRecognized( + recognizedEvent({ sha256: SHA_B }), + ); + expect(result).toBeUndefined(); + }, + ); +}); + +describe('MediaMemoryService.commitPolicySucceeded', () => { + it("gives a byte-identical sibling file its own execution, not the first file's", async () => { + // Two Files with identical bytes and the same policy configuration. + // The execution key used to be content-only, so B adopted A's + // execution node: B got ZERO records of its own while A's derivative + // versions were handed back stamped with B's root (mixed lineage). + // M §11.2/§11.3: each File writes its own PolicyExecution and + // provenance; only the underlying computation and bytes are reused. + const a = (await service.recordFileRecognized(recognizedEvent()))!; + const b = (await service.recordFileRecognized( + recognizedEvent({ fileRef: '/movies/copy.mkv' }), + ))!; + + const commitA = (await service.commitPolicySucceeded(succeededInput(a)))!; + const commitB = (await service.commitPolicySucceeded(succeededInput(b)))!; + + // Separate execution nodes, and B's is recorded as a reuse of A's. + expect(commitB.executionId).not.toBe(commitA.executionId); + expect(commitB.created).toBe(true); + const snapshot = await readSnapshot(); + expect(snapshot.executions[commitB.executionId]).toMatchObject({ + sourceVersionId: b.fileVersionId, + rootFileId: b.rootFileId, + reusedExecutionId: commitA.executionId, + }); + // A's execution is the original — it points at nothing. + expect( + snapshot.executions[commitA.executionId]!.reusedExecutionId, + ).toBeUndefined(); + + // Each side's derivative is rooted in its OWN tree (no borrowed + // lineage), while both name the same content-addressed object. + const bindingA = commitA.mediaBindings.get(SHA_OUT)!; + const bindingB = commitB.mediaBindings.get(SHA_OUT)!; + expect(bindingA.rootFileId).toBe(a.rootFileId); + expect(bindingB.rootFileId).toBe(b.rootFileId); + expect(bindingB.fileId).not.toBe(bindingA.fileId); + expect(snapshot.files[bindingA.fileId]!.fileRef).toBe( + snapshot.files[bindingB.fileId]!.fileRef, + ); + }); + + it('stays idempotent when the SAME file replays the same execution', async () => { + const a = (await service.recordFileRecognized(recognizedEvent()))!; + const first = (await service.commitPolicySucceeded(succeededInput(a)))!; + const replay = (await service.commitPolicySucceeded(succeededInput(a)))!; + + expect(replay.executionId).toBe(first.executionId); + expect(replay.created).toBe(false); + expect(replay.mediaBindings.get(SHA_OUT)).toEqual( + first.mediaBindings.get(SHA_OUT), + ); + const snapshot = await readSnapshot(); + expect(Object.keys(snapshot.executions)).toHaveLength(1); + expect( + snapshot.executions[first.executionId]!.reusedExecutionId, + ).toBeUndefined(); + }); + + it('leaves the derived version pointing at the execution that made it', async () => { + // A second execution over the same source under a different omni + // configuration can land on byte-identical output — same derived File, + // same version. Rewriting the version's producer would make it name an + // execution whose outputs it is not. + const source = (await service.recordFileRecognized(recognizedEvent()))!; + const first = (await service.commitPolicySucceeded( + succeededInput(source), + ))!; + const second = (await service.commitPolicySucceeded( + succeededInput(source, { omniConfigHash: 'fp-' + '1'.repeat(61) }), + ))!; + + expect(second.executionId).not.toBe(first.executionId); + const derivedVersionId = first.mediaBindings.get(SHA_OUT)!.fileVersionId; + expect(second.mediaBindings.get(SHA_OUT)!.fileVersionId).toBe( + derivedVersionId, + ); + const snapshot = await readSnapshot(); + expect(snapshot.versions[derivedVersionId]!.producedByExecutionId).toBe( + first.executionId, + ); + }); + + it('commits execution + derived version + entry atomically with lineage edges', async () => { + const source = (await service.recordFileRecognized(recognizedEvent()))!; + const commit = await service.commitPolicySucceeded(succeededInput(source)); + expect(commit).toBeDefined(); + expect(commit!.created).toBe(true); + + const binding = commit!.mediaBindings.get(SHA_OUT); + expect(binding).toBeDefined(); + expect(binding!.rootFileId).toBe(source.rootFileId); + + const snapshot = await readSnapshot(); + const execution = snapshot.executions[commit!.executionId]; + expect(execution).toMatchObject({ + invocationId: 'deadbeef01234567', + sourceVersionId: source.fileVersionId, + rootFileId: source.rootFileId, + toolName: 'omni_downscale_video', + }); + expect(execution.outputRefs).toHaveLength(1); + + const derivedVersion = snapshot.versions[binding!.fileVersionId]; + expect(derivedVersion).toMatchObject({ + sha256: SHA_OUT, + parentVersionId: source.fileVersionId, + producedByExecutionId: commit!.executionId, + }); + expect(snapshot.files[binding!.fileId]).toMatchObject({ + origin: 'policy', + rootFileId: source.rootFileId, + }); + + const entry = snapshot.entries[execution.outputRefs[0]]; + expect(entry).toMatchObject({ + kind: 'derived_media', + derivedVersionId: binding!.fileVersionId, + parentVersionId: source.fileVersionId, + producedByExecutionId: commit!.executionId, + channels: ['visual', 'acoustic'], + coverage: { mode: 'complete', scope: {} }, + }); + expect(entry.artifactRef).toMatchObject({ + storage: 'managed', + managedId: `sha256/${SHA_OUT}`, + }); + }); + + it('converges replays on the same execution node (content-identity key)', async () => { + const source = (await service.recordFileRecognized(recognizedEvent()))!; + const first = await service.commitPolicySucceeded(succeededInput(source)); + // Different invocation (degradation-cache hit), same content identity. + const replay = await service.commitPolicySucceeded( + succeededInput(source, { invocationId: 'cache-hit' }), + ); + expect(replay!.executionId).toBe(first!.executionId); + expect(replay!.created).toBe(false); + expect(replay!.mediaBindings.get(SHA_OUT)).toEqual( + first!.mediaBindings.get(SHA_OUT), + ); + const snapshot = await readSnapshot(); + expect(Object.keys(snapshot.executions)).toHaveLength(1); + expect(Object.keys(snapshot.entries)).toHaveLength(1); + // The replay's invocationId is NOT rewritten onto the record. + expect(snapshot.executions[first!.executionId].invocationId).toBe( + 'deadbeef01234567', + ); + }); + + it('creates distinct executions for distinct tool configurations', async () => { + const source = (await service.recordFileRecognized(recognizedEvent()))!; + const first = await service.commitPolicySucceeded(succeededInput(source)); + const other = await service.commitPolicySucceeded( + succeededInput(source, { omniConfigHash: 'fp-other' }), + ); + expect(other!.executionId).not.toBe(first!.executionId); + expect(other!.created).toBe(true); + }); + + it('persists text outputs as policy_result entries with bounded inlineText', async () => { + const bounded = new TestMediaMemoryService(root, { + maxInlineTextBytes: 10, + }); + const source = (await bounded.recordFileRecognized(recognizedEvent()))!; + const commit = await bounded.commitPolicySucceeded( + succeededInput(source, { + outputs: [ + { + kind: 'text', + objectPath: `/store/objects/${SHA_OUT}.txt`, + sha256: SHA_OUT, + mimeType: 'text/plain', + text: '你好世界这段文本很长', // 3 bytes per CJK code point + sizeBytes: 30, + role: 'transcript', + }, + ], + }), + ); + expect(commit!.mediaBindings.size).toBe(0); + const snapshot = await readSnapshot(); + const entry = + snapshot.entries[snapshot.executions[commit!.executionId].outputRefs[0]]; + expect(entry.kind).toBe('policy_result'); + expect(entry.derivedVersionId).toBeUndefined(); + // 10-byte budget over 3-byte code points → 3 characters, never split. + expect(entry.inlineText).toBe('你好世'); + expect(entry.channels).toEqual(['speech_text']); + // Full content stays reachable through the managed artifact. + expect(entry.artifactRef?.managedId).toBe(`sha256/${SHA_OUT}`); + }); + + it('records every output of a multi-output execution under one execution', async () => { + // Real policy runs deliver more than one artifact (extract the audio + // track AND its transcript; downscale AND a poster frame). Recording + // only the first would have memory report the run as done while half + // its products are invisible to recall and to reuse — the next + // identical run then re-derives what memory silently dropped. + const source = (await service.recordFileRecognized(recognizedEvent()))!; + const input = succeededInput(source, { + toolName: 'omni_extract_audio', + outputs: [ + { + kind: 'media', + objectPath: `/store/objects/${SHA_OUT}.m4a`, + sha256: SHA_OUT, + mediaType: 'audio', + metadata: { durationMs: 4_860_000 }, + sizeBytes: 5_000_000, + mimeType: 'audio/mp4', + role: 'extracted_audio', + }, + { + kind: 'text', + objectPath: `/store/objects/${SHA_TEXT}.txt`, + sha256: SHA_TEXT, + mimeType: 'text/plain', + text: 'Two divers surface at dawn.', + sizeBytes: 27, + role: 'transcript', + }, + ], + }); + const commit = (await service.commitPolicySucceeded(input))!; + + // Only the media output becomes a version the orchestrator can thread. + expect([...commit.mediaBindings.keys()]).toEqual([SHA_OUT]); + + const snapshot = await readSnapshot(); + const execution = snapshot.executions[commit.executionId]; + expect(execution.outputRefs).toHaveLength(2); + expect(Object.keys(snapshot.executions)).toHaveLength(1); + const [audio, transcript] = execution.outputRefs.map( + (id) => snapshot.entries[id], + ); + expect(audio).toMatchObject({ + kind: 'derived_media', + role: 'extracted_audio', + derivedVersionId: commit.mediaBindings.get(SHA_OUT)!.fileVersionId, + parentVersionId: source.fileVersionId, + producedByExecutionId: commit.executionId, + channels: ['acoustic'], + }); + expect(transcript).toMatchObject({ + kind: 'policy_result', + role: 'transcript', + inlineText: 'Two divers surface at dawn.', + parentVersionId: source.fileVersionId, + producedByExecutionId: commit.executionId, + channels: ['speech_text'], + }); + expect(transcript.derivedVersionId).toBeUndefined(); + expect(transcript.outputId).not.toBe(audio.outputId); + + // Reuse must offer the whole set: a caller that re-runs this exact + // computation skips it entirely, so a partially recorded set would hand + // back an audio track with no transcript and call it a complete reuse. + const reusable = await service.findReusableOutputs( + SHA_A, + input.omniConfigHash, + ); + expect(reusable!.executionId).toBe(commit.executionId); + expect(reusable!.outputs.map((o) => `${o.kind}:${o.sha256}`)).toEqual([ + `media:${SHA_OUT}`, + `text:${SHA_TEXT}`, + ]); + }); + + it('derives sampled coverage for keyframe outputs', async () => { + const source = (await service.recordFileRecognized(recognizedEvent()))!; + const commit = await service.commitPolicySucceeded( + succeededInput(source, { + outputs: [ + { + kind: 'media', + objectPath: `/store/objects/${SHA_OUT}.jpg`, + sha256: SHA_OUT, + mediaType: 'image', + metadata: { width: 854, height: 480 }, + sizeBytes: 50_000, + mimeType: 'image/jpeg', + role: 'keyframe', + }, + ], + }), + ); + const snapshot = await readSnapshot(); + const entry = + snapshot.entries[snapshot.executions[commit!.executionId].outputRefs[0]]; + expect(entry.coverage).toEqual({ mode: 'sampled', scope: {} }); + expect(entry.channels).toEqual(['visual']); + }); + + it('derives partial coverage for clip outputs', async () => { + const source = (await service.recordFileRecognized(recognizedEvent()))!; + const commit = await service.commitPolicySucceeded( + succeededInput(source, { + toolName: 'omni_extract_clip', + outputs: [ + { + kind: 'media', + objectPath: `/store/objects/${SHA_OUT}.mp4`, + sha256: SHA_OUT, + mediaType: 'video', + metadata: { durationMs: 30_000, width: 1920, height: 1080 }, + sizeBytes: 2_000_000, + mimeType: 'video/mp4', + role: 'clip', + }, + ], + }), + ); + const snapshot = await readSnapshot(); + const entry = + snapshot.entries[snapshot.executions[commit!.executionId].outputRefs[0]]; + // A clip is one slice of a 81-minute film. Recording it as `complete` + // would let the gap derivation count the version's visual and acoustic + // channels as fully covered, so recall reports nothing missing and the + // model answers about the whole film from a 30-second excerpt — the + // overclaim coverage exists to prevent. + expect(entry.coverage).toEqual({ mode: 'partial', scope: {} }); + expect(entry.channels).toEqual(['visual', 'acoustic']); + }); +}); + +describe('MediaMemoryService.findBindingBySha256', () => { + it('returns the binding for known content and undefined for unknown', async () => { + const source = (await service.recordFileRecognized(recognizedEvent()))!; + await expect(service.findBindingBySha256(SHA_A)).resolves.toEqual({ + fileId: source.fileId, + fileVersionId: source.fileVersionId, + rootFileId: source.rootFileId, + }); + await expect(service.findBindingBySha256(SHA_B)).resolves.toBeUndefined(); + }); + + it('prefers the newest version when several match', async () => { + await service.recordFileRecognized(recognizedEvent()); + const newer = (await service.recordFileRecognized( + recognizedEvent({ fileRef: '/movies/copy.mkv' }), + ))!; + const found = await service.findBindingBySha256(SHA_A); + expect(found).toEqual({ + fileId: newer.fileId, + fileVersionId: newer.fileVersionId, + rootFileId: newer.rootFileId, + }); + }); +}); + +describe('truncateUtf8', () => { + it('returns short text unchanged', () => { + expect(truncateUtf8('hello', 10)).toBe('hello'); + }); + + it('never splits a code point', () => { + expect(truncateUtf8('a你b', 3)).toBe('a'); // '你' needs 3 bytes, only 2 left + expect(truncateUtf8('a你b', 4)).toBe('a你'); + expect(truncateUtf8('🎬🎬', 5)).toBe('🎬'); // 4-byte emoji + }); +}); diff --git a/packages/core/src/services/media-memory/service.ts b/packages/core/src/services/media-memory/service.ts new file mode 100644 index 00000000000..d09462296f6 --- /dev/null +++ b/packages/core/src/services/media-memory/service.ts @@ -0,0 +1,723 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { createHash } from 'node:crypto'; +import { createDebugLogger } from '../../utils/debugLogger.js'; +import type { MediaProbeResult } from '../../omni/ffmpeg.js'; +import type { OmniModality } from '../../omni/recognition.js'; +import { MediaMemoryStore } from './store.js'; +import type { + FileRecognizedCommit, + FileRecognizedEvent, + MediaChannel, + MediaCoverage, + MediaExecutionOrigin, + MediaFileId, + MediaFileRecord, + MediaFileVersionId, + MediaFileVersionRecord, + MediaMemoryEntryId, + MediaMemorySnapshot, + MediaPolicyExecutionRecord, + MediaVersionRecognition, + NormalizedPolicyOutput, + PolicyExecutionId, +} from './types.js'; + +const debugLogger = createDebugLogger('omni:memory'); + +/** Default bound for inline text persisted on an entry (M §9, + * `omni.memory.collection.maxInlineTextBytes`). Oversized text is + * truncated on the entry — the full content stays reachable through the + * artifactRef (transcripts are promoted objects). */ +export const DEFAULT_MAX_INLINE_TEXT_BYTES = 65536; + +/** Recognition provenance constants for the S4 detector (content sniff + + * ffprobe). Bump when the recognition pipeline changes meaningfully. */ +export const MEDIA_DETECTOR_VERSION = 'omni-sniff-ffprobe/1'; + +function hashId(prefix: string, material: string): string { + return ( + prefix + createHash('sha256').update(material).digest('hex').slice(0, 24) + ); +} + +/** fileId is deterministic in the fileRef: the same logical file always + * resolves to the same identity, which makes FileRecognized an upsert. */ +function fileIdFor(fileRef: string): MediaFileId { + return hashId('f', fileRef); +} + +/** + * fileId of a POLICY DERIVATIVE, keyed by (root, object path). + * + * Derivatives live in the content-addressed object store, so two roots + * that derive byte-identical output land on the same object path. Keying + * the graph node on the path alone would make them ONE File node whose + * `rootFileId` belongs to whichever root created it first — leaking one + * file's lineage into another's (M §11.2: 不能把一个文件的权限或 + * provenance 泄漏给另一个文件) and putting the node outside the second + * root's bounded traversal (M §8). + * + * Folding the root in gives each root its own cheap metadata rows while + * the BYTES stay deduplicated by the object store — the reuse M §11.2 + * actually sanctions ("受管文件的物理存储块"). + */ +function derivedFileIdFor( + objectPath: string, + rootFileId: MediaFileId, +): MediaFileId { + return hashId('f', `${rootFileId}|${objectPath}`); +} + +/** versionId is deterministic in (fileId, sha256): re-recognizing the + * same content is a no-op, and two files sharing bytes keep two distinct + * version records (M §11 — same hash never merges Files). */ +function versionIdFor(fileId: MediaFileId, sha256: string): MediaFileVersionId { + return hashId('v', `${fileId}|${sha256}`); +} + +/** One media deliverable of a successful policy execution, as known at + * the orchestrator's success point (validated + promoted to objects/). */ +export interface PolicyMediaOutputInput { + kind: 'media'; + /** Promoted object path (fileRef of the derivative). */ + objectPath: string; + sha256: string; + mediaType: OmniModality; + metadata: MediaProbeResult; + sizeBytes: number; + mimeType: string; + role?: string; + disclosure?: string; +} + +/** One non-media text artifact (transcript protocol) of a successful + * policy execution. `text` was validated as bounded UTF-8 by the + * orchestrator; the promoted object retains the full content. */ +export interface PolicyTextOutputInput { + kind: 'text'; + objectPath: string; + sha256: string; + mimeType: string; + text: string; + sizeBytes: number; + role?: string; + disclosure?: string; +} + +export type PolicyOutputInput = PolicyMediaOutputInput | PolicyTextOutputInput; + +/** Memory-side identity of a resource flowing through the policy + * pipeline; returned by every commit and threaded on work items. */ +export interface MediaMemoryBinding { + fileId: MediaFileId; + fileVersionId: MediaFileVersionId; + rootFileId: MediaFileId; +} + +/** Complete payload of one OmniPolicySucceeded commit (M §6.4): the + * execution plus every validated output, committed atomically. */ +export interface PolicySucceededInput { + invocationId: string; + source: MediaMemoryBinding; + executionOrigin: MediaExecutionOrigin; + toolName: string; + toolVersion?: string; + /** Effective arguments the tool ran with, reserved runtime keys + * (inputPath/outputDir/resourceId) excluded by the caller. */ + finalArguments: Record; + /** Content-identity hash of the resolved policy/tool configuration + * (the degradation-cache fingerprint at the S4 boundary). */ + omniConfigHash: string; + startedAt: string; + completedAt: string; + outputs: PolicyOutputInput[]; +} + +/** One recorded output of a reusable execution: where its bytes live and + * the provenance the reusing caller must reproduce. */ +export interface ReusableOutputRecord { + kind: 'media' | 'text'; + sha256: string; + /** Object-store path recorded for the derivative. Absent for text + * outputs (no version node); reconstruct it from `sha256`. */ + objectPath?: string; + mimeType: string; + sizeBytes: number; + role?: string; + disclosure?: string; +} + +/** Outputs of a prior execution eligible for reuse (M §11.3). */ +export interface ReusableExecutionOutputs { + /** The original execution — recorded as `reusedExecutionId` when the + * reusing file commits its own execution. */ + executionId: PolicyExecutionId; + outputs: ReusableOutputRecord[]; +} + +export interface PolicySucceededCommit { + executionId: PolicyExecutionId; + /** Bindings for derived media outputs, keyed by output sha256, so the + * orchestrator can thread memory identity onto derived work items. */ + mediaBindings: Map; + /** False when the execution was already recorded (content-identity + * replay: degradation cache hit, same-invocation retry). */ + created: boolean; +} + +/** Conservative v1 channel derivation by modality/role. */ +function channelsFor( + mediaType: OmniModality | undefined, + role: string | undefined, +): MediaChannel[] { + if (role === 'transcript') return ['speech_text']; + if (role === 'ocr') return ['onscreen_text']; + switch (mediaType) { + case 'image': + return ['visual']; + case 'audio': + return ['acoustic']; + case 'video': + return ['visual', 'acoustic']; + default: + return []; + } +} + +/** Conservative v1 coverage: sampled for keyframes, partial for clips, + * complete (whole-version, degraded fidelity disclosed separately) for + * everything else. Honesty over precision — never overclaim. */ +function coverageFor(role: string | undefined): MediaCoverage { + if (role === 'keyframe') return { mode: 'sampled', scope: {} }; + if (role === 'clip') return { mode: 'partial', scope: {} }; + return { mode: 'complete', scope: {} }; +} + +/** Truncate UTF-8 text to a byte budget without splitting a code point. */ +export function truncateUtf8(text: string, maxBytes: number): string { + const encoder = new TextEncoder(); + if (encoder.encode(text).byteLength <= maxBytes) return text; + let result = ''; + let bytes = 0; + for (const ch of text) { + const chBytes = encoder.encode(ch).byteLength; + if (bytes + chBytes > maxBytes) break; + result += ch; + bytes += chBytes; + } + return result; +} + +/** + * The single write facade of multimodal media memory (M §14): the omni + * harness calls the two collection triggers below; nothing else in the + * system writes memory. The Agent side is read-only (recall — Stage B). + * + * Failure stance: memory is an enhancement, not the delivery path. Every + * public method catches its own persistence errors, logs, and reports + * `undefined` — a failed commit must never break a delivery. Within one + * commit, the store's transact gives all-or-nothing semantics. + */ +export class MediaMemoryService { + private readonly store: MediaMemoryStore; + private readonly maxInlineTextBytes: number; + + constructor(omniRootDir: string, options?: { maxInlineTextBytes?: number }) { + this.store = new MediaMemoryStore(omniRootDir); + this.maxInlineTextBytes = + options?.maxInlineTextBytes ?? DEFAULT_MAX_INLINE_TEXT_BYTES; + } + + /** + * Collection trigger 1 — FileRecognized (M §6.1). Preconditions are the + * caller's contract: mediaType decided, FULL sha256 computed, metadata + * and a definite probeStatus in hand. Idempotent upsert: same fileRef + + * same content is a no-op; new content at a known fileRef creates a new + * immutable version and moves CURRENT_VERSION. + * + * Returns undefined when persistence failed (logged, never thrown). + */ + async recordFileRecognized( + event: FileRecognizedEvent, + ): Promise { + try { + return await this.store.transact(undefined, (snapshot) => { + const commit = upsertRecognizedFile(snapshot, event, this.now()); + return { result: commit, changed: commit.changed }; + }); + } catch (err) { + debugLogger.debug( + `recordFileRecognized failed for ${event.source.locator}: ` + + `${err instanceof Error ? err.message : err}`, + ); + return undefined; + } + } + + /** + * Collection trigger 2 — OmniPolicySucceeded (M §6.4). One atomic + * transaction commits the execution record, a file+version per derived + * media output (DERIVED_FROM / PRODUCED_BY edges), and one entry per + * output (HAS_OUTPUT edges). The executionId is deterministic in the + * content-identity reuse key (source sha256 ⊕ omniConfigHash, M §11), + * so degradation-cache hits and invocation replays converge on the + * same execution node instead of duplicating it. + * + * Returns undefined when persistence failed (logged, never thrown) — + * the delivery proceeds regardless. + */ + async commitPolicySucceeded( + input: PolicySucceededInput, + ): Promise { + try { + return await this.store.transact(undefined, (snapshot) => { + const result = commitExecution( + snapshot, + input, + this.maxInlineTextBytes, + this.now(), + ); + return { result: result.commit, changed: result.changed }; + }); + } catch (err) { + debugLogger.debug( + `commitPolicySucceeded failed for ${input.toolName} ` + + `(invocation ${input.invocationId}): ` + + `${err instanceof Error ? err.message : err}`, + ); + return undefined; + } + } + + /** + * Recorded outputs of a prior execution that already performed THIS + * computation on THESE bytes (content-identity reuse key, M §11.3) — + * the read side of «同文件同 settings 二次触发同一 policy：直接复用， + * 无重复执行». Covers every output shape, so multi-output tools and text + * products (transcripts) are reusable too, unlike the S4 degradation + * cache which maps one input to a single media derivative. + * + * Returns locators and recorded provenance only: the caller must verify + * the bytes still exist and still hash to `sha256` before reusing them + * (memory.json is project-local and hand-editable — the same stance the + * degradation-cache hit path takes). Undefined when nothing matches or + * the store is unreadable (logged, never thrown). + */ + async findReusableOutputs( + sourceSha256: string, + omniConfigHash: string, + ): Promise { + try { + return await this.store.read(undefined, (snapshot) => { + // No self to exclude: this is a pre-execution lookup. + const match = findReusableExecution( + snapshot, + `${sourceSha256}|${omniConfigHash}`, + '', + ); + if (!match) return undefined; + const outputs: ReusableOutputRecord[] = []; + for (const entryId of match.outputRefs) { + const entry = snapshot.entries[entryId]; + if (!entry?.artifactRef) return undefined; // incomplete — no reuse + const managedId = entry.artifactRef.managedId; + const sha256 = managedId?.startsWith('sha256/') + ? managedId.slice('sha256/'.length) + : undefined; + if (sha256 === undefined) return undefined; + const version = entry.derivedVersionId + ? snapshot.versions[entry.derivedVersionId] + : undefined; + // Media outputs resolve their object through the derived + // version's file record; a text output has no version node, so + // the caller re-derives its path from the content hash. + const objectPath = version + ? snapshot.files[version.fileId]?.fileRef + : undefined; + outputs.push({ + kind: entry.kind === 'derived_media' ? 'media' : 'text', + sha256, + ...(objectPath !== undefined ? { objectPath } : {}), + mimeType: entry.artifactRef.mimeType, + sizeBytes: entry.artifactRef.sizeBytes, + ...(entry.role !== undefined ? { role: entry.role } : {}), + ...(entry.disclosure !== undefined + ? { disclosure: entry.disclosure } + : {}), + }); + } + if (outputs.length === 0) return undefined; + return { executionId: match.executionId, outputs }; + }); + } catch (err) { + debugLogger.debug( + `findReusableOutputs failed: ${err instanceof Error ? err.message : err}`, + ); + return undefined; + } + } + + /** + * Read-side lookup by LOCATOR, for the harness only (design M §9.2.1: + * path/hash lookup is a harness capability; the model is confined to + * session handles). Returns the CURRENT version of the File recorded at + * this locator, whether or not the bytes are still on disk — which is + * the point: it lets a remembered-but-missing file be re-anchored into a + * session so its memory stays reachable. + */ + async findBindingByFileRef(fileRef: string): Promise< + | { + binding: MediaMemoryBinding; + mediaType: OmniModality; + sha256: string; + } + | undefined + > { + try { + return await this.store.read(undefined, (snapshot) => { + for (const file of Object.values(snapshot.files)) { + if (file.fileRef !== fileRef) continue; + const version = snapshot.versions[file.currentVersionId]; + if (!version) return undefined; + return { + binding: { + fileId: file.fileId, + fileVersionId: version.fileVersionId, + rootFileId: file.rootFileId, + }, + mediaType: version.mediaType, + sha256: version.sha256, + }; + } + return undefined; + }); + } catch (err) { + debugLogger.debug( + `findBindingByFileRef failed: ${err instanceof Error ? err.message : err}`, + ); + return undefined; + } + } + + /** + * Read-side lookup for callers that hold bytes but no identity (the + * reactive degradation ladder re-recognizes a stored object without + * knowing which memory version it is). Returns the binding of the + * newest version whose content hash matches, or undefined when memory + * has never seen the content (or the store is unreadable — logged, + * never thrown). + */ + async findBindingBySha256( + sha256: string, + ): Promise { + try { + return await this.store.read(undefined, (snapshot) => { + let found: { binding: MediaMemoryBinding; createdAt: string } | null = + null; + for (const version of Object.values(snapshot.versions)) { + if (version.sha256 !== sha256) continue; + const file = snapshot.files[version.fileId]; + if (!file) continue; + if (found && found.createdAt >= version.createdAt) continue; + found = { + binding: { + fileId: version.fileId, + fileVersionId: version.fileVersionId, + rootFileId: file.rootFileId, + }, + createdAt: version.createdAt, + }; + } + return found?.binding; + }); + } catch (err) { + debugLogger.debug( + `findBindingBySha256 failed: ${err instanceof Error ? err.message : err}`, + ); + return undefined; + } + } + + /** Injectable for tests via subclassing; records are stamped once per + * commit so all records of a transaction share one timestamp. */ + protected now(): string { + return new Date().toISOString(); + } +} + +/** Shared by both triggers: upsert the (file, version) pair for one + * recognized content state and move CURRENT_VERSION onto it. */ +function upsertRecognizedFile( + snapshot: MediaMemorySnapshot, + event: FileRecognizedEvent, + now: string, +): FileRecognizedCommit & { changed: boolean } { + // Policy derivatives are identified per root (see derivedFileIdFor); + // user/tool files by their locator alone. + const fileId = + event.origin === 'policy' && event.rootFileId !== undefined + ? derivedFileIdFor(event.fileRef, event.rootFileId) + : fileIdFor(event.fileRef); + const fileVersionId = versionIdFor(fileId, event.sha256); + let changed = false; + + let file: MediaFileRecord | undefined = snapshot.files[fileId]; + if (!file) { + file = { + fileId, + rootFileId: event.rootFileId ?? fileId, + fileRef: event.fileRef, + origin: event.origin, + currentVersionId: fileVersionId, + createdAt: now, + }; + snapshot.files[fileId] = file; + changed = true; + } + + let version: MediaFileVersionRecord | undefined = + snapshot.versions[fileVersionId]; + const created = !version; + if (!version) { + version = { + fileVersionId, + fileId, + sha256: event.sha256, + mediaType: event.mediaType, + metadata: event.metadata, + sizeBytes: event.sizeBytes, + mimeType: event.mimeType, + source: event.source, + recognition: event.recognition, + ...(event.parentVersionId !== undefined + ? { parentVersionId: event.parentVersionId } + : {}), + createdAt: now, + }; + snapshot.versions[fileVersionId] = version; + changed = true; + } + + // CURRENT_VERSION follows what is on disk NOW — a re-recognition of an + // older content state (user reverted the file) moves the pointer back. + if (file.currentVersionId !== fileVersionId) { + file.currentVersionId = fileVersionId; + changed = true; + } + + return { + fileId, + fileVersionId, + rootFileId: file.rootFileId, + created, + changed, + }; +} + +/** + * Content-identity reuse key of a recorded execution, or undefined when + * its source version is no longer in the graph. Recomputed from the + * record rather than persisted, so executions written before this key + * existed participate too. + */ +function reuseKeyOf( + snapshot: MediaMemorySnapshot, + execution: MediaPolicyExecutionRecord, +): string | undefined { + const sha = snapshot.versions[execution.sourceVersionId]?.sha256; + return sha === undefined ? undefined : `${sha}|${execution.omniConfigHash}`; +} + +/** + * The earliest execution whose content-identity key matches — one that + * already performed this exact computation on identical bytes, through + * another File. Deterministic (completion time, then id) so the same + * graph always attributes reuse to the same original. Reuse records are + * skipped as candidates: a chain always points at the ORIGINAL. + */ +function findReusableExecution( + snapshot: MediaMemorySnapshot, + reuseKey: string, + selfExecutionId: PolicyExecutionId, +): MediaPolicyExecutionRecord | undefined { + let best: MediaPolicyExecutionRecord | undefined; + for (const candidate of Object.values(snapshot.executions)) { + if (candidate.executionId === selfExecutionId) continue; + if (candidate.reusedExecutionId !== undefined) continue; + if (reuseKeyOf(snapshot, candidate) !== reuseKey) continue; + if ( + !best || + candidate.completedAt < best.completedAt || + (candidate.completedAt === best.completedAt && + candidate.executionId < best.executionId) + ) { + best = candidate; + } + } + return best; +} + +function commitExecution( + snapshot: MediaMemorySnapshot, + input: PolicySucceededInput, + maxInlineTextBytes: number, + now: string, +): { commit: PolicySucceededCommit; changed: boolean } { + // Per-File execution identity (M §11.2/§11.3 «每个 File 仍写入自己的 + // PolicyExecution 与 provenance … 不共享图节点»): keyed on the SOURCE + // VERSION, so two Files that happen to share bytes each record their + // own execution instead of the second one silently adopting the first's + // node (which left it with zero records of its own and rebound the + // first's derivatives under the second's root). + const sourceVersion = snapshot.versions[input.source.fileVersionId]; + const sourceSha = sourceVersion?.sha256 ?? input.source.fileVersionId; + const executionId = hashId( + 'x', + `${input.source.fileVersionId}|${input.omniConfigHash}`, + ); + // Content-identity reuse key (M §11.3): the same bytes under the same + // resolved tool configuration produce byte-identical output no matter + // which File they were derived through. Deliberately NOT the execution + // id — it spans Files, which is exactly what makes cross-File reuse + // possible while provenance stays separate. + const reuseKey = `${sourceSha}|${input.omniConfigHash}`; + + const mediaBindings = new Map(); + const existing = snapshot.executions[executionId]; + if (existing) { + // Idempotent replay: THIS File already has this execution (same + // source version, same configuration) — rebuild the bindings from its + // own recorded derivatives instead of duplicating nodes. + for (const entryId of existing.outputRefs) { + const entry = snapshot.entries[entryId]; + if (!entry?.derivedVersionId) continue; + const version = snapshot.versions[entry.derivedVersionId]; + if (!version) continue; + mediaBindings.set(version.sha256, { + fileId: version.fileId, + fileVersionId: version.fileVersionId, + rootFileId: + snapshot.files[version.fileId]?.rootFileId ?? input.source.rootFileId, + }); + } + return { + commit: { executionId, mediaBindings, created: false }, + changed: false, + }; + } + + // Cross-File reuse (M §11.3): another File already ran this exact + // computation on identical bytes. Record OUR own execution pointing at + // it via `reusedExecutionId` and materialize our own entry/version rows + // over the same content-addressed objects — no re-derivation, no shared + // graph nodes, no borrowed lineage. + const reused = findReusableExecution(snapshot, reuseKey, executionId); + + const outputRefs: MediaMemoryEntryId[] = []; + for (const [index, output] of input.outputs.entries()) { + const entryId = hashId('e', `${executionId}|${index}|${output.sha256}`); + outputRefs.push(entryId); + + let derivedVersionId: MediaFileVersionId | undefined; + if (output.kind === 'media') { + // Every derived media output becomes a policy-origin file with its + // own version, rooted at the source's root (M §7 lineage graph). + const commit = upsertRecognizedFile( + snapshot, + { + fileRef: output.objectPath, + sha256: output.sha256, + mediaType: output.mediaType, + metadata: output.metadata, + sizeBytes: output.sizeBytes, + mimeType: output.mimeType, + origin: 'policy', + rootFileId: input.source.rootFileId, + parentVersionId: input.source.fileVersionId, + source: { + protocol: 'managed', + locator: `sha256/${output.sha256}`, + }, + recognition: { + ingestionConfigHash: input.omniConfigHash, + detectorVersion: MEDIA_DETECTOR_VERSION, + probeStatus: 'complete', + } satisfies MediaVersionRecognition, + }, + now, + ); + derivedVersionId = commit.fileVersionId; + // Only the execution that FIRST materialized this content owns the + // provenance pointer. A later execution landing on the same version + // (identical bytes from a differently-configured run) would otherwise + // rewrite history, and the version would name an execution whose + // outputs it is not. + if (commit.created) { + snapshot.versions[commit.fileVersionId].producedByExecutionId = + executionId; + } + mediaBindings.set(output.sha256, { + fileId: commit.fileId, + fileVersionId: commit.fileVersionId, + rootFileId: commit.rootFileId, + }); + } + + const entry: NormalizedPolicyOutput = { + outputId: entryId, + kind: output.kind === 'media' ? 'derived_media' : 'policy_result', + ...(output.role !== undefined ? { role: output.role } : {}), + artifactRef: { + storage: 'managed', + managedId: `sha256/${output.sha256}`, + mimeType: output.mimeType, + sizeBytes: output.sizeBytes, + }, + ...(output.kind === 'text' + ? { inlineText: truncateUtf8(output.text, maxInlineTextBytes) } + : {}), + ...(output.disclosure !== undefined + ? { disclosure: output.disclosure } + : {}), + scope: {}, + channels: channelsFor( + output.kind === 'media' ? output.mediaType : undefined, + output.role, + ), + coverage: coverageFor(output.role), + parentVersionId: input.source.fileVersionId, + producedByExecutionId: executionId, + ...(derivedVersionId !== undefined ? { derivedVersionId } : {}), + createdAt: now, + }; + snapshot.entries[entryId] = entry; + } + + const execution: MediaPolicyExecutionRecord = { + executionId, + invocationId: input.invocationId, + sourceVersionId: input.source.fileVersionId, + rootFileId: input.source.rootFileId, + executionOrigin: input.executionOrigin, + toolName: input.toolName, + toolVersion: input.toolVersion, + finalArguments: input.finalArguments, + inputScope: {}, + omniConfigHash: input.omniConfigHash, + outputRefs, + ...(reused !== undefined ? { reusedExecutionId: reused.executionId } : {}), + startedAt: input.startedAt, + completedAt: input.completedAt, + }; + snapshot.executions[executionId] = execution; + + return { + commit: { executionId, mediaBindings, created: true }, + changed: true, + }; +} diff --git a/packages/core/src/services/media-memory/store.test.ts b/packages/core/src/services/media-memory/store.test.ts new file mode 100644 index 00000000000..880b07faa0d --- /dev/null +++ b/packages/core/src/services/media-memory/store.test.ts @@ -0,0 +1,252 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { MediaMemoryStore, MEDIA_MEMORY_FILE_NAME } from './store.js'; + +let root: string; +let store: MediaMemoryStore; + +beforeEach(async () => { + root = await fs.mkdtemp(path.join(os.tmpdir(), 'omni-memory-store-')); + store = new MediaMemoryStore(root); +}); + +afterEach(async () => { + await fs.chmod(root, 0o700).catch(() => {}); + await fs.chmod(store.filePath, 0o600).catch(() => {}); + await fs.rm(root, { recursive: true, force: true }); +}); + +// chmod-based denial tests are meaningless on Windows and as root. +const canDropPermissions = + process.platform !== 'win32' && + (typeof process.getuid !== 'function' || process.getuid() !== 0); + +async function listCorruptBackups(): Promise { + return (await fs.readdir(root)).filter((n) => + n.startsWith(`${MEDIA_MEMORY_FILE_NAME}.corrupt-`), + ); +} + +describe('MediaMemoryStore', () => { + it('starts from an empty snapshot when the document does not exist', async () => { + const counts = await store.read(undefined, (snapshot) => ({ + schemaVersion: snapshot.schemaVersion, + files: Object.keys(snapshot.files).length, + versions: Object.keys(snapshot.versions).length, + })); + expect(counts).toEqual({ schemaVersion: 1, files: 0, versions: 0 }); + // A pure read never creates the document. + await expect(fs.stat(store.filePath)).rejects.toMatchObject({ + code: 'ENOENT', + }); + }); + + it('persists a changed snapshot atomically and reloads it', async () => { + await store.transact(undefined, (snapshot) => { + snapshot.files['f1'] = { + fileId: 'f1', + rootFileId: 'f1', + fileRef: '/tmp/a.mp4', + origin: 'user', + currentVersionId: 'v1', + createdAt: '2026-08-11T00:00:00.000Z', + }; + return { result: undefined, changed: true }; + }); + const reloaded = new MediaMemoryStore(root); + const fileIds = await reloaded.read([], (snapshot) => + Object.keys(snapshot.files), + ); + expect(fileIds).toEqual(['f1']); + const stat = await fs.stat(store.filePath); + if (canDropPermissions) { + expect(stat.mode & 0o777).toBe(0o600); + } + }); + + // Rename replaces the inode; an in-place write keeps it. Meaningless on + // Windows, where fs.stat().ino is not a stable identity. + it.runIf(process.platform !== 'win32')( + 'replaces the document by rename instead of writing over it in place', + async () => { + const put = (fileId: string) => + store.transact(undefined, (snapshot) => { + snapshot.files[fileId] = { + fileId, + rootFileId: fileId, + fileRef: `/tmp/${fileId}.mp4`, + origin: 'user', + currentVersionId: 'v1', + createdAt: '2026-08-11T00:00:00.000Z', + }; + return { result: undefined, changed: true }; + }); + + await put('f1'); + const before = await fs.stat(store.filePath); + await put('f2'); + const after = await fs.stat(store.filePath); + + // An in-place rewrite truncates the live document first, so a + // concurrent reader (another store instance, another process on the + // same project) or a crash mid-write leaves half a JSON document — + // which the load path can only treat as corrupt, condemning the whole + // graph to a .corrupt backup and re-derivation. + expect(after.ino).not.toBe(before.ino); + expect( + JSON.parse(await fs.readFile(store.filePath, 'utf8')), + ).toMatchObject({ schemaVersion: 1, files: { f1: {}, f2: {} } }); + // The staging file is committed by the rename, never left behind. + expect( + (await fs.readdir(root)).filter((n) => n.endsWith('.tmp')), + ).toEqual([]); + }, + ); + + it('does not save when the mutator reports no change', async () => { + await store.transact(undefined, (snapshot) => { + snapshot.files['ghost'] = { + fileId: 'ghost', + rootFileId: 'ghost', + fileRef: 'x', + origin: 'user', + currentVersionId: 'v', + createdAt: 'now', + }; + return { result: undefined, changed: false }; + }); + await expect(fs.stat(store.filePath)).rejects.toMatchObject({ + code: 'ENOENT', + }); + }); + + it('backs up a corrupt document and rebuilds empty', async () => { + await fs.writeFile(store.filePath, 'not json at all'); + const files = await store.read(undefined, (snapshot) => + Object.keys(snapshot.files), + ); + expect(files).toEqual([]); + expect(await listCorruptBackups()).toHaveLength(1); + }); + + it('treats an unexpected shape (wrong schemaVersion) as corrupt', async () => { + await fs.writeFile( + store.filePath, + JSON.stringify({ schemaVersion: 99, files: {} }), + ); + const version = await store.read(undefined, (s) => s.schemaVersion); + expect(version).toBe(1); + expect(await listCorruptBackups()).toHaveLength(1); + }); + + it('prunes a malformed record value instead of blacking out every read', async () => { + // Envelope is valid, so the corrupt-backup self-heal never fires: a + // single non-object value used to surface as raw TypeErrors from every + // read path, caught into miss/empty — a PERMANENT global recall + // blackout for the whole project. + await fs.writeFile( + store.filePath, + JSON.stringify({ + schemaVersion: 1, + files: { f1: { fileId: 'f1', fileRef: '/a.mkv' }, f2: null }, + versions: { v1: 'not-an-object' }, + executions: {}, + entries: {}, + }), + ); + + const seen = await store.read(undefined, (s) => ({ + files: Object.keys(s.files), + versions: Object.keys(s.versions), + // Dereferences values exactly like the real index/lookup paths do — + // a surviving bad value would throw right here. + refs: Object.values(s.files).map((f) => f.fileRef), + })); + + expect(seen).toEqual({ files: ['f1'], versions: [], refs: ['/a.mkv'] }); + // Valid records survive, so the document is NOT condemned. + expect(await listCorruptBackups()).toHaveLength(0); + }); + + it('keeps at most two corrupt backups', async () => { + for (let i = 0; i < 4; i++) { + await fs.writeFile(store.filePath, `broken-${i}`); + await store.read(undefined, () => undefined); + // Distinct Date.now() suffixes. + await new Promise((resolve) => setTimeout(resolve, 2)); + } + expect((await listCorruptBackups()).length).toBeLessThanOrEqual(2); + }); + + it('discards the OLDEST corrupt backups, keeping the newest', async () => { + // Backups from earlier sessions, oldest first. + for (const stamp of ['1000000000000', '1000000000001', '1000000000002']) { + await fs.writeFile(`${store.filePath}.corrupt-${stamp}`, 'old'); + } + await fs.writeFile(store.filePath, 'not json at all'); + await store.read(undefined, () => undefined); + + // A backup is the only surviving evidence of what went wrong, and the + // corruption worth investigating is the one that just happened — + // retaining the oldest two would delete the fresh backup and keep + // documents from sessions nobody is debugging. + const stamps = (await listCorruptBackups()).map((name) => + name.slice(`${MEDIA_MEMORY_FILE_NAME}.corrupt-`.length), + ); + expect(stamps).toHaveLength(2); + expect(stamps.some((s) => Number(s) > 1_700_000_000_000)).toBe(true); + expect(stamps).toContain('1000000000002'); + expect(stamps).not.toContain('1000000000001'); + expect(stamps).not.toContain('1000000000000'); + }); + + it.runIf(canDropPermissions)( + 'returns unreadableResult and never saves when the document exists but cannot be read', + async () => { + await store.transact(undefined, (snapshot) => { + snapshot.entries['e1'] = { + outputId: 'e1', + kind: 'policy_result', + scope: {}, + channels: [], + coverage: { mode: 'complete', scope: {} }, + parentVersionId: 'v1', + producedByExecutionId: 'x1', + createdAt: 'now', + }; + return { result: undefined, changed: true }; + }); + await fs.chmod(store.filePath, 0o000); + const result = await store.transact('unreadable', (snapshot) => { + snapshot.entries = {}; + return { result: 'mutated', changed: true }; + }); + expect(result).toBe('unreadable'); + await fs.chmod(store.filePath, 0o600); + // The prior graph survived — a transient denial never wipes memory. + const entryCount = await store.read( + 0, + (s) => Object.keys(s.entries).length, + ); + expect(entryCount).toBe(1); + }, + ); + + it.runIf(canDropPermissions)( + 'surfaces save failures to the caller (transact rejects)', + async () => { + await fs.chmod(root, 0o500); + await expect( + store.transact(undefined, () => ({ result: undefined, changed: true })), + ).rejects.toThrow(); + }, + ); +}); diff --git a/packages/core/src/services/media-memory/store.ts b/packages/core/src/services/media-memory/store.ts new file mode 100644 index 00000000000..9284d87bd61 --- /dev/null +++ b/packages/core/src/services/media-memory/store.ts @@ -0,0 +1,205 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { serializeFileOperation } from '../../omni/json-cache-file.js'; +import { atomicWriteFile } from '../../utils/atomicFileWrite.js'; +import { createDebugLogger } from '../../utils/debugLogger.js'; +import type { MediaMemorySnapshot } from './types.js'; + +const debugLogger = createDebugLogger('omni:memory'); + +/** Keep at most this many `.corrupt-*` backups (newest wins) — same + * hygiene bound as the omni JSON caches. */ +const MAX_CORRUPT_BACKUPS = 2; + +export const MEDIA_MEMORY_FILE_NAME = 'memory.json'; + +function emptySnapshot(): MediaMemorySnapshot { + return { + schemaVersion: 1, + files: {}, + versions: {}, + executions: {}, + entries: {}, + }; +} + +function isPlainRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +/** + * Drop malformed RECORD VALUES from an otherwise well-shaped document. + * + * The envelope check above only proves the four collections are objects. + * A single non-object value inside one of them (a hand edit, a bad merge, + * a truncated sync) would surface as raw TypeErrors from every read path + * (`indexSnapshot` dereferences `version.parentVersionId`, + * `findBindingBySha256` reads `version.sha256`, …). Those throws are + * caught into "miss"/empty by the read wrappers — so ONE bad value turns + * every recall in the project into a permanent blackout, while the + * corrupt-backup self-heal never fires because the envelope is valid. + * + * Pruning individually costs only the pruned records (dangling references + * to them already degrade gracefully — a missing version reads as an + * `artifact_unavailable` gap) and keeps the never-fatal contract intact. + * Same defense as the sibling `OmniJsonCacheFile.load()`. + */ +function pruneMalformedRecords(snapshot: MediaMemorySnapshot): void { + const dropped: string[] = []; + for (const collection of [ + 'files', + 'versions', + 'executions', + 'entries', + ] as const) { + const records = snapshot[collection] as Record; + for (const [key, value] of Object.entries(records)) { + if (!isPlainRecord(value)) { + delete records[key]; + dropped.push(`${collection}/${key}`); + } + } + } + if (dropped.length > 0) { + // Name the records: a silent prune would leave an operator debugging + // "recall returns less than it should" with nothing to go on. + debugLogger.debug( + `dropped ${dropped.length} malformed memory record` + + `${dropped.length === 1 ? '' : 's'}: ${dropped.join(', ')}`, + ); + } +} + +/** + * v1 JSON persistence for the media-memory graph: ONE document + * (`.qwen/omni/memory.json`) holding every file/version/execution/entry + * record. Deliberately the same discipline as omni's JSON entry caches + * (S4 decision D2 — the backend is an internal detail; the collector and + * recall service only ever see {@link MediaMemorySnapshot}): + * + * - per-file serialized load-modify-save (in-process), sharing the omni + * file-operation chain so two store instances on one file never race; + * - atomic writes (tmp + rename, `noFollow`, 0600/0700 forced); + * - corrupt documents backed up as `.corrupt-` and rebuilt empty — + * memory loss costs re-derivation, never a broken pipeline; + * - unreadable-but-existing documents make the operation a no-op: a + * transient EACCES must never lead to a save that wipes the graph. + * + * Transaction semantics (M §12): one {@link transact} call is one atomic + * commit — the mutator runs against the full snapshot in memory and the + * document is rewritten in a single rename. Multi-record commits + * (OmniPolicySucceeded: execution + versions + entries + edge updates) + * therefore land all-or-nothing by construction. + */ +export class MediaMemoryStore { + readonly filePath: string; + + constructor(omniRootDir: string) { + this.filePath = path.join(omniRootDir, MEDIA_MEMORY_FILE_NAME); + } + + /** + * Run one serialized operation against the snapshot. `fn` returns the + * operation result plus whether it mutated the snapshot (triggering an + * atomic save). When the document exists but cannot be read, + * `unreadableResult` is returned and nothing is saved. + */ + async transact( + unreadableResult: R, + fn: ( + snapshot: MediaMemorySnapshot, + ) => + | { result: R; changed?: boolean } + | Promise<{ result: R; changed?: boolean }>, + ): Promise { + return serializeFileOperation(this.filePath, async () => { + const snapshot = await this.load(); + if (!snapshot) return unreadableResult; + const { result, changed } = await fn(snapshot); + if (changed) await this.save(snapshot); + return result; + }); + } + + /** Read-only view over the snapshot (recall, queries). */ + async read( + unreadableResult: R, + fn: (snapshot: MediaMemorySnapshot) => R | Promise, + ): Promise { + return this.transact(unreadableResult, async (snapshot) => ({ + result: await fn(snapshot), + })); + } + + private async load(): Promise { + let raw: string; + try { + raw = await fs.readFile(this.filePath, 'utf8'); + } catch (err) { + const code = (err as NodeJS.ErrnoException).code; + if (code === 'ENOENT' || code === 'ENOTDIR') return emptySnapshot(); + debugLogger.debug( + `memory read failed, operation skipped: ${err instanceof Error ? err.message : err}`, + ); + return null; + } + try { + const parsed = JSON.parse(raw) as MediaMemorySnapshot; + if ( + isPlainRecord(parsed) && + parsed.schemaVersion === 1 && + isPlainRecord(parsed.files) && + isPlainRecord(parsed.versions) && + isPlainRecord(parsed.executions) && + isPlainRecord(parsed.entries) + ) { + pruneMalformedRecords(parsed); + return parsed; + } + throw new Error('unexpected shape'); + } catch { + const backup = `${this.filePath}.corrupt-${Date.now()}`; + await fs.rename(this.filePath, backup).catch(() => {}); + await this.pruneCorruptBackups(); + debugLogger.debug(`corrupt memory document backed up to ${backup}`); + return emptySnapshot(); + } + } + + private async pruneCorruptBackups(): Promise { + const dir = path.dirname(this.filePath); + const prefix = `${path.basename(this.filePath)}.corrupt-`; + try { + const backups = (await fs.readdir(dir)) + .filter((n) => n.startsWith(prefix)) + .sort() + .reverse(); + for (const name of backups.slice(MAX_CORRUPT_BACKUPS)) { + await fs.rm(path.join(dir, name), { force: true }).catch(() => {}); + } + } catch { + // Hygiene only; never let it affect the read path. + } + } + + private async save(snapshot: MediaMemorySnapshot): Promise { + // Unlike the caches, a memory commit that cannot persist must SURFACE: + // the collector treats it as a collection failure (logged, delivery + // unaffected) rather than silently reporting success. + await fs.mkdir(path.dirname(this.filePath), { + recursive: true, + mode: 0o700, + }); + await atomicWriteFile(this.filePath, JSON.stringify(snapshot, null, 1), { + mode: 0o600, + forceMode: true, + noFollow: true, + }); + } +} diff --git a/packages/core/src/services/media-memory/types.ts b/packages/core/src/services/media-memory/types.ts new file mode 100644 index 00000000000..c7a1f155849 --- /dev/null +++ b/packages/core/src/services/media-memory/types.ts @@ -0,0 +1,274 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Multimodal media memory — persistent record types (upstream design M + * §5–§8). Everything here is data shape only: collection semantics live + * in the collector, persistence in the store, and read access in the + * recall service. Two invariants shape every type in this file: + * + * - Persistent identity is separate from filesystem reality. The model + * (and recall consumers) see opaque IDs, never real paths; locators are + * sanitized before they enter a record. + * - Only two collection triggers exist — FileRecognized and + * OmniPolicySucceeded — so every record traces back to a completed + * recognition or a fully-committed policy execution. + */ + +import type { MediaProbeResult } from '../../omni/ffmpeg.js'; +import type { OmniModality } from '../../omni/recognition.js'; + +// ─── Persistent IDs (M §5.1) ──────────────────────────────────────────── +// String aliases, not branded types: records cross a JSON boundary where +// brands cannot survive, and the alias names keep signatures readable. + +/** Stable identity of one logical media file (a fileRef the user or a + * tool introduced). Survives content changes — versions hang off it. */ +export type MediaFileId = string; + +/** Immutable identity of one content state (full SHA-256) of a file. */ +export type MediaFileVersionId = string; + +/** Identity of one memory entry (a normalized policy output or metadata + * record surfaced by recall). */ +export type MediaMemoryEntryId = string; + +/** Identity of one committed policy execution. */ +export type PolicyExecutionId = string; + +// ─── Scope / channel / coverage (M §5.4) ──────────────────────────────── + +/** Region of a media version an entry or execution covers. An empty + * object means the whole version. */ +export interface MediaScope { + temporal?: { startMs: number; endMs: number }; + spatial?: { + x: number; + y: number; + width: number; + height: number; + unit: 'normalized'; + }; + frameRange?: { start: number; end: number }; + streamIndexes?: number[]; + audioChannels?: number[]; +} + +/** Information channel an entry speaks for. */ +export type MediaChannel = + | 'technical_metadata' + | 'visual' + | 'acoustic' + | 'speech_text' + | 'onscreen_text'; + +/** How completely an entry covers its scope. */ +export interface MediaCoverage { + mode: 'complete' | 'continuous' | 'sampled' | 'partial' | 'summary'; + scope: MediaScope; +} + +// ─── File / version records (M §5.2) ──────────────────────────────────── + +/** Where a media file entered the system. */ +export type MediaFileOrigin = 'user' | 'tool' | 'policy'; + +export interface MediaFileRecord { + fileId: MediaFileId; + /** Harness-internal locator: absolute path for local user/tool files + * (identity = localPath + sha256, storage design S §4 — the bytes stay + * in place), managed object path for derivatives. Keys the idempotent + * FileRecognized upsert and lets recall detect deleted local files + * (artifact_unavailable gap). NEVER surfaced through recall payloads — + * the model only sees opaque IDs and sanitized sources. */ + fileRef: string; + /** Root of the derivation tree this file belongs to. A user/tool file + * is its own root; a policy derivative inherits its source's root. + * Bounds every graph traversal (M §7). */ + rootFileId: MediaFileId; + origin: MediaFileOrigin; + currentVersionId: MediaFileVersionId; + createdAt: string; +} + +/** Sanitized description of where a version's bytes came from. Never a + * raw filesystem path for user files — recall consumers must not learn + * real paths (M §5.2/§15). */ +export interface MediaVersionSource { + /** 'local' user/tool file, 'managed' promoted object, 'url' download. */ + protocol: 'local' | 'managed' | 'url'; + /** Sanitized locator: basename for local files, sha256-addressed + * object key for managed content, origin host for URLs. */ + locator: string; +} + +/** Recognition summary persisted with a version (M §5.2): enough to + * decide whether a past recognition is still trustworthy under today's + * configuration without re-probing. */ +export interface MediaVersionRecognition { + ingestionConfigHash: string; + detectorVersion: string; + probeStatus: 'complete' | 'partial' | 'unavailable'; +} + +export interface MediaFileVersionRecord { + fileVersionId: MediaFileVersionId; + fileId: MediaFileId; + /** Full content SHA-256 — the sole version-identity criterion (M §11). */ + sha256: string; + mediaType: OmniModality; + metadata: MediaProbeResult; + sizeBytes: number; + mimeType: string; + source: MediaVersionSource; + recognition: MediaVersionRecognition; + /** DERIVED_FROM edge: the source version a policy derived this from. */ + parentVersionId?: MediaFileVersionId; + /** PRODUCED_BY edge: the execution that produced this version. */ + producedByExecutionId?: PolicyExecutionId; + createdAt: string; +} + +// ─── Policy execution records (M §5.3) ────────────────────────────────── + +/** Who initiated the execution. Mirrors core's ToolExecutionOrigin but + * persists independently — wire types must stay free to evolve. */ +export type MediaExecutionOrigin = + | { + kind: 'fixed_policy'; + policyId: string; + stage: 'preprocessing' | 'transport_guard'; + } + | { kind: 'model' } + | { kind: 'client' }; + +export interface MediaPolicyExecutionRecord { + executionId: PolicyExecutionId; + /** Orchestrator staging invocation id (= scheduler callId). */ + invocationId: string; + /** EXECUTED_ON edge: the version the tool ran against. */ + sourceVersionId: MediaFileVersionId; + rootFileId: MediaFileId; + executionOrigin: MediaExecutionOrigin; + toolName: string; + /** Media-policy descriptor version of the tool implementation; absent + * when the tool declares none (the descriptor field is optional). */ + toolVersion?: string; + /** Arguments after defaults/locked/runtime resolution — the values the + * tool actually ran with (reserved runtime keys like inputPath are + * excluded: they are per-invocation filesystem details, not + * reproducible tool configuration). */ + finalArguments: Record; + /** Region of the source version the execution consumed. */ + inputScope: MediaScope; + /** Hash of the resolved omni configuration relevant to this run. */ + omniConfigHash: string; + /** HAS_OUTPUT edges. */ + outputRefs: MediaMemoryEntryId[]; + /** Set when this record is a reuse of an earlier execution (same + * content-identity cache key, M §11): points at the execution whose + * outputs were reused instead of duplicating nodes. */ + reusedExecutionId?: PolicyExecutionId; + startedAt: string; + completedAt: string; +} + +// ─── Memory entries (M §5.5 / §6.3) ───────────────────────────────────── + +/** Media-typed roles a v1 policy output may carry. Free-form strings are + * accepted from tools; these are the ones recall understands natively. */ +export type KnownMediaMemoryRole = + | 'transcript' + | 'ocr' + | 'caption' + | 'summary' + | 'keyframe' + | 'clip' + | 'extracted_audio'; + +/** Reference to a stored artifact backing a derived_media entry. */ +export interface MediaArtifactRef { + storage: 'managed' | 'workspace'; + /** Content-addressed object key when managed, as `sha256/` — the + * form the object store resolves and the only field an entry's bytes can + * be re-found by after the producing run is gone. */ + managedId?: string; + workspacePath?: string; + mimeType: string; + sizeBytes: number; +} + +/** One normalized policy output as committed to memory. Sourced ONLY + * from a PolicyArtifactBatch (M §6.3) — never llmContent/returnDisplay + * or hook-injected artifacts. */ +export interface NormalizedPolicyOutput { + outputId: MediaMemoryEntryId; + kind: 'derived_media' | 'policy_result'; + role?: string; + artifactRef?: MediaArtifactRef; + /** Bounded inline text (transcripts, OCR). Never larger than + * `collection.maxInlineTextBytes`. */ + inlineText?: string; + disclosure?: string; + scope: MediaScope; + channels: MediaChannel[]; + coverage: MediaCoverage; + /** Version of the SOURCE media this output describes. */ + parentVersionId: MediaFileVersionId; + producedByExecutionId: PolicyExecutionId; + /** For derived_media outputs: the fileVersionId of the derivative's + * own version record (the DERIVED_FROM child). */ + derivedVersionId?: MediaFileVersionId; + createdAt: string; +} + +// ─── Store snapshot shape (v1 JSON backend) ───────────────────────────── + +/** Whole-store snapshot persisted as one JSON document. v1 keeps the + * entire graph in one file (`.qwen/omni/memory.json`) — swapping the + * backend later changes only the store internals (S4 precedent D2). */ +export interface MediaMemorySnapshot { + schemaVersion: 1; + files: Record; + versions: Record; + executions: Record; + entries: Record; +} + +// ─── Collection inputs (collector-facing) ─────────────────────────────── + +/** Everything the FileRecognized trigger point knows about a file at the + * moment recognition completes (M §6.1 preconditions: mediaType decided, + * full SHA-256 available, metadata + definite probeStatus). */ +export interface FileRecognizedEvent { + /** Identity key for the logical file. For user/tool local files this + * is the absolute path (identity = localPath + sha256, storage design + * S §4 — the bytes are NOT copied into the object store); for managed + * derivatives it is the object path. Used only to derive/lookup the + * fileId — persisted records carry the sanitized `source` instead. */ + fileRef: string; + sha256: string; + mediaType: OmniModality; + metadata: MediaProbeResult; + sizeBytes: number; + mimeType: string; + origin: MediaFileOrigin; + source: MediaVersionSource; + recognition: MediaVersionRecognition; + /** For policy derivatives: lineage edges known at recognition time. */ + parentVersionId?: MediaFileVersionId; + /** For policy derivatives: root inherited from the source file. */ + rootFileId?: MediaFileId; +} + +/** Result of committing (or staging) a FileRecognized event. */ +export interface FileRecognizedCommit { + fileId: MediaFileId; + fileVersionId: MediaFileVersionId; + rootFileId: MediaFileId; + /** False when the version already existed (idempotent re-recognition). */ + created: boolean; +} diff --git a/packages/core/src/tools/tool-names.ts b/packages/core/src/tools/tool-names.ts index 3432b82b7e9..4db35a8d6fb 100644 --- a/packages/core/src/tools/tool-names.ts +++ b/packages/core/src/tools/tool-names.ts @@ -79,6 +79,9 @@ export const ToolNames = { OMNI_CLIP_VIDEO: 'omni_clip_video', OMNI_CONVERT_IMAGE: 'omni_convert_image', OMNI_TRANSCRIBE_AUDIO: 'omni_transcribe_audio', + // Omni memory recall (registered only when omni is enabled AND + // `omni.memory.recall.mode === 'active'` — D10 mutual exclusion). + OMNI_RECALL_MEDIA_MEMORY: 'omni_recall_media_memory', } as const; /** @@ -141,6 +144,7 @@ export const ToolDisplayNames = { OMNI_CLIP_VIDEO: 'ClipVideo', OMNI_CONVERT_IMAGE: 'ConvertImage', OMNI_TRANSCRIBE_AUDIO: 'TranscribeAudio', + OMNI_RECALL_MEDIA_MEMORY: 'RecallMediaMemory', } as const; // Migration from old tool names to new tool names diff --git a/packages/vscode-ide-companion/schemas/settings.schema.json b/packages/vscode-ide-companion/schemas/settings.schema.json index 71ebe6639c2..a8df2701a7f 100644 --- a/packages/vscode-ide-companion/schemas/settings.schema.json +++ b/packages/vscode-ide-companion/schemas/settings.schema.json @@ -3342,6 +3342,12 @@ "default": 0, "description": "Estimated-token ceiling for a single omni media input, checked at the delivery boundary using the versioned raw-resource estimator. 0 disables the token guard — the estimation formula is pending confirmation with the model provider; set a positive threshold to enforce fail-closed rejection." }, + "maxDurationSeconds": { + "type": "integer", + "minimum": 0, + "default": 0, + "description": "Duration ceiling in seconds for a single omni media input, checked at the delivery boundary. 0 disables it. Byte and token limits cannot express a provider duration cap: a long film downscaled under the byte ceiling still gets rejected by the API, after paying for the transcode. Set this and the guard omits it honestly instead." + }, "policies": { "description": "Guard policies keyed by policy id, run only when the final delivery set exceeds transport limits. Merged with system defaults by id. The merged set must cover image, video, and audio and must not be empty; every policy output must use source: omit.", "type": "object", @@ -3421,6 +3427,120 @@ } } } + }, + "memory": { + "description": "Persistent multimodal media memory (collection of recognized files and policy execution results, plus cross-session recall). Invalid values abort startup.", + "type": "object", + "properties": { + "collection": { + "description": "Collection-side budgets.", + "type": "object", + "properties": { + "maxInlineTextBytes": { + "type": "integer", + "minimum": 1, + "default": 65536, + "description": "Upper bound for inline text persisted on a memory entry (transcripts, OCR). Longer text is truncated on the entry; the stored artifact keeps the full content." + } + } + }, + "recall": { + "description": "Recall-side exposure and budgets.", + "type": "object", + "properties": { + "mode": { + "type": "string", + "enum": [ + "active", + "sideQuery" + ], + "default": "active", + "description": "Mutually exclusive recall exposure: \"active\" registers the recall tool for the model; \"sideQuery\" runs a passive selector before the main request instead." + }, + "maxEntries": { + "type": "integer", + "minimum": 1, + "default": 12, + "description": "Maximum entries one recall may return." + }, + "maxTextChars": { + "type": "integer", + "minimum": 1, + "default": 24000, + "description": "Total character budget across all text in one recall result." + }, + "kinds": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "metadata", + "derived_media", + "policy_result", + "execution" + ] + }, + "minItems": 1, + "description": "Entry kinds recall may surface. Replaces wholesale when set (never element-merged)." + }, + "includeHistoricalVersions": { + "description": "Whether recall may surface entries for non-current file versions by default.", + "type": "boolean", + "default": false + }, + "active": { + "description": "Budgets for the active recall tool.", + "type": "object", + "properties": { + "maxFilesPerCall": { + "type": "integer", + "minimum": 1, + "default": 8, + "description": "Maximum distinct files one recall tool call may query." + } + } + }, + "sideQuery": { + "description": "Budgets for the passive side-query selector.", + "type": "object", + "properties": { + "model": { + "type": [ + "string", + "null" + ], + "default": null, + "description": "Model for the passive selector; unset uses the side-query default (the configured fast model, falling back to the session model)." + }, + "timeoutMs": { + "type": "integer", + "minimum": 1, + "default": 30000, + "description": "Side-query timeout; on timeout the turn proceeds with an empty recall." + }, + "maxCandidateEntries": { + "type": "integer", + "minimum": 1, + "default": 100, + "description": "Maximum candidate entries shown to the selector." + }, + "maxSelectedEntries": { + "type": "integer", + "minimum": 1, + "default": 12, + "description": "Maximum entries the selector may pick. Must not exceed recall.maxEntries." + }, + "maxAttempts": { + "type": "integer", + "minimum": 1, + "default": 1, + "description": "Attempts allowed for the selector call itself (unparseable or schema-invalid output is retried up to this many times). A selection that parses but names entries outside the manifest is refused without a retry; the turn then proceeds with an empty recall." + } + } + } + } + } + } } } }, diff --git a/packages/web-shell/client/components/messages/toolFormatting.ts b/packages/web-shell/client/components/messages/toolFormatting.ts index 6433a483aff..9f226718ba0 100644 --- a/packages/web-shell/client/components/messages/toolFormatting.ts +++ b/packages/web-shell/client/components/messages/toolFormatting.ts @@ -66,6 +66,7 @@ export const TOOL_DISPLAY_NAMES: Record = { omni_clip_video: 'ClipVideo', omni_convert_image: 'ConvertImage', omni_transcribe_audio: 'TranscribeAudio', + omni_recall_media_memory: 'RecallMediaMemory', bash: 'Shell', shell: 'Shell Command', read: 'ReadFile', diff --git a/packages/web-shell/client/i18n.tsx b/packages/web-shell/client/i18n.tsx index 776635c81cc..65a591db726 100644 --- a/packages/web-shell/client/i18n.tsx +++ b/packages/web-shell/client/i18n.tsx @@ -2673,6 +2673,7 @@ const ZH: Messages = { 'toolName.omni_clip_video': '剪辑视频', 'toolName.omni_convert_image': '转换图像', 'toolName.omni_transcribe_audio': '转写音频', + 'toolName.omni_recall_media_memory': '召回媒体记忆', // web-shell-only wire aliases (see TOOL_DISPLAY_NAMES in toolFormatting.ts) 'toolName.bash': '运行命令', 'toolName.shell': 'Shell 命令',