From 3a9a88212dfc7d81822b3ac25d21b1c42355b6c4 Mon Sep 17 00:00:00 2001 From: Ilari Ahonen <77334791+CeamenDemon@users.noreply.github.com> Date: Fri, 11 Sep 2026 16:42:53 +0300 Subject: [PATCH 1/2] feat: add opt-in Context Atlas dispatcher exploration --- bin/context-atlas.ts | 113 +++++++++++++ bin/context-atlas/catalog.mjs | 207 ++++++++++++++++++++++++ bin/fm-test-run.sh | 12 +- docs/configuration.md | 4 + docs/context-atlas-design.md | 59 +++++++ docs/context-atlas.md | 35 ++++ docs/documentation-audiences.json | 12 ++ docs/verification/context-atlas.md | 99 ++++++++++++ docs/verification/runtime-backends.md | 4 + tests/assets/context-atlas-provider.ts | 103 ++++++++++++ tests/fm-context-atlas-live-e2e.test.sh | 59 +++++++ tests/fm-context-atlas.test.sh | 87 ++++++++++ tests/fm-pi-primary-types.test.sh | 5 +- 13 files changed, 796 insertions(+), 3 deletions(-) create mode 100644 bin/context-atlas.ts create mode 100644 bin/context-atlas/catalog.mjs create mode 100644 docs/context-atlas-design.md create mode 100644 docs/context-atlas.md create mode 100644 docs/verification/context-atlas.md create mode 100644 tests/assets/context-atlas-provider.ts create mode 100755 tests/fm-context-atlas-live-e2e.test.sh create mode 100755 tests/fm-context-atlas.test.sh diff --git a/bin/context-atlas.ts b/bin/context-atlas.ts new file mode 100644 index 00000000000..b691b4b2b95 --- /dev/null +++ b/bin/context-atlas.ts @@ -0,0 +1,113 @@ +/** + * Default-off Pi Context Atlas prototype. Explicit load only, never auto-install. + * From the selected Git repository root: + * pi -e /path/to/firstmate/bin/context-atlas.ts + * pi -e /path/to/firstmate/bin/context-atlas.ts --atlas-read --atlas-defer + * For a command-line prompt, put -- after the flags and before the quoted prompt. + * Optional --atlas-root /absolute/repo and --atlas-exclude prefix,prefix narrow scope. + * --atlas-read authorizes this extension's local, bounded text adapter, NOT a + * proxy for Pi read. Do not enable it where a read-tool-specific policy must + * mediate reads: that policy must independently authorize atlas too. + * --atlas-defer temporarily hides only originally active, original Pi built-in + * read/grep/find/ls tools. All other tools and their owners remain unchanged. + * /atlas-help prints this contract; /atlas-restore restores unchanged deferred + * definitions without removing any other active tools. Shutdown also restores. + * + * atlas({op:"resolve",q:"f:README.md"}) -> generation and identity-backed ref. + * atlas({op:"read",ref:"",gen:"",at:1,count:20}) + * atlas({op:"resolve",q:"t:read"}) then op:"activate" -> original tool next call. + * Other ops: catalog (q optional; at is 1-based pagination), inspect, refresh. + * q is a literal identity substring, optionally f: or t:, never a shell command. + * ref operations require gen. Refresh invalidates every earlier generation. + * No positional aliases, arbitrary execution, writes, symbols, diffs or network. + * Bounds: 8 catalog candidates, 100 read lines, 256 KiB file, 8 KiB JSON result, + * 20,000 Git inventory paths / 2 MiB inventory output / 5 seconds per Git call. + * Hidden/private/dependency/build/credential paths, ignored files (even tracked), + * symlinks and hardlinks are excluded; --atlas-exclude adds relative prefixes. + * The catalog reads metadata only, never contents. File reads need --atlas-read. + * This is not a sandbox or a secret scanner: do not authorize reads in a tree + * containing secrets disguised as ordinary source files or adversarial writers. + */ +import type { ExtensionAPI } from '@earendil-works/pi-coding-agent'; +import { Type } from 'typebox'; +import { StringEnum } from '@earendil-works/pi-ai'; +import { createAtlas, activatable, toolStamp } from './context-atlas/catalog.mjs'; + +const HELP = `Context Atlas: explicit-load experiment; no arbitrary tool execution. +Load: pi -e /path/to/firstmate/bin/context-atlas.ts [--atlas-read] [--atlas-defer] +For a command-line prompt, add -- before the quoted prompt, after all flags. +Scope: --atlas-root /absolute/git-root; --atlas-exclude relative/prefix,another +atlas ops: catalog, resolve, inspect, read, activate, refresh. +Resolve q is a literal identity substring (optional f: or t: prefix). +Use the returned ref and generation as ref/gen for inspect/read/activate. +at/count select 1-based catalog pages or read lines. Refresh invalidates old gen. +Limits: 8 candidates, 100 lines, 256 KiB files, 8 KiB JSON results. +Read needs --atlas-read and uses a separate local text adapter, not the read tool. +Existing read-specific policies do not automatically govern atlas; authorize it separately. +Only originally active built-in read/grep/find/ls can be deferred and reactivated. +Activation preserves the original name/schema/hooks for the model's next call. +/atlas-restore restores unchanged deferred tools, preserving other active tools. +Default exclusions cover ignored, hidden, private, credential, dependency and build paths. +No writes, shell execution, network, diffs or symbol parser. Not a sandbox or secret scanner. +See bin/context-atlas.ts header for the complete invocation contract.`; + +export default function (pi: ExtensionAPI) { + pi.registerFlag('atlas-root', { type: 'string', description: 'Selected Git root; defaults to cwd' }); + pi.registerFlag('atlas-exclude', { type: 'string', description: 'Additional excluded repository-relative prefixes, comma-separated' }); + pi.registerFlag('atlas-read', { type: 'boolean', default: false, description: 'Authorize Atlas local text reads separately from original tool policies' }); + pi.registerFlag('atlas-defer', { type: 'boolean', default: false, description: 'Defer originally active built-in read/grep/find/ls until Atlas activation' }); + let dispatch: ReturnType | undefined; + const deferred = new Map(); + function restore() { + const names = pi.getAllTools().filter(t => deferred.get(t.name) === toolStamp(t)).map(t => t.name); + pi.setActiveTools([...new Set([...pi.getActiveTools(), ...names])]); + deferred.clear(); + } + pi.on('session_start', (_event, ctx) => { + dispatch = undefined; + const root = pi.getFlag('atlas-root'); + const excludes = pi.getFlag('atlas-exclude'); + dispatch = createAtlas({ + root: typeof root === 'string' ? root : ctx.cwd, + read: pi.getFlag('atlas-read') === true, + exclusions: typeof excludes === 'string' ? excludes.split(',') : [], + tools: () => pi.getAllTools(), active: () => pi.getActiveTools(), + activate: (t) => { + if (deferred.get(t.name) !== toolStamp(t)) return false; + pi.setActiveTools([...new Set([...pi.getActiveTools(), t.name])]); + return pi.getActiveTools().includes(t.name); + }, + }); + if (pi.getFlag('atlas-defer') === true && pi.getActiveTools().includes('atlas')) { + const active = new Set(pi.getActiveTools()); + for (const t of pi.getAllTools()) if (active.has(t.name) && activatable(t)) deferred.set(t.name, toolStamp(t)); + pi.setActiveTools([...active].filter(n => !deferred.has(n))); + } + }); + pi.on('session_shutdown', () => { restore(); dispatch = undefined; }); + pi.registerCommand('atlas-help', { description: 'Context Atlas invocation and safety contract', handler: async () => { + pi.sendMessage({ customType: 'atlas-help', content: HELP, display: true }); + } }); + pi.registerCommand('atlas-restore', { description: 'Restore unchanged Atlas-deferred tools', handler: async (_args, ctx) => { + if (!ctx.isIdle()) { ctx.ui.notify('Wait for the current turn before restoring tools.', 'warning'); return; } + restore(); + ctx.ui.notify('Unchanged Atlas-deferred tools restored.', 'info'); + } }); + pi.registerTool({ + name: 'atlas', label: 'Context Atlas', + description: 'Index files/tools: catalog/resolve q (literal, optional f:/t:). inspect/read/activate require returned ref and generation as gen. Read at/count lines; activate original read-only tool for next call. refresh invalidates gen. Max 8 candidates, 100 lines, 8 KiB JSON. No writes or shell.', + parameters: Type.Object({ + op: StringEnum(['catalog', 'resolve', 'inspect', 'read', 'activate', 'refresh']), + q: Type.Optional(Type.String({ maxLength: 400 })), + ref: Type.Optional(Type.String({ maxLength: 400 })), + gen: Type.Optional(Type.String({ maxLength: 400 })), + at: Type.Optional(Type.Integer({ minimum: 1 })), + count: Type.Optional(Type.Integer({ minimum: 1, maximum: 100 })), + }, { additionalProperties: false }), + async execute(_id, params, signal) { + if (!dispatch) throw new Error('Atlas unavailable: selected root must be a readable Git root; see /atlas-help.'); + const result = dispatch(params, signal); + return { content: [{ type: 'text', text: JSON.stringify(result) }], details: result }; + }, + }); +} diff --git a/bin/context-atlas/catalog.mjs b/bin/context-atlas/catalog.mjs new file mode 100644 index 00000000000..2fe5ae1ff5c --- /dev/null +++ b/bin/context-atlas/catalog.mjs @@ -0,0 +1,207 @@ +// Context Atlas's read-only dispatcher. The Pi entry point owns invocation help. +import { execFileSync } from 'node:child_process'; +import { createHash, randomBytes } from 'node:crypto'; +import { constants, lstatSync, realpathSync, openSync, fstatSync, readSync, closeSync } from 'node:fs'; +import { resolve, relative, sep, extname } from 'node:path'; + +const MAX_FILE = 256 * 1024; +const MAX_RESULT = 8192; +const MAX_ITEMS = 8; +const READ_TOOLS = new Set(['read', 'grep', 'find', 'ls']); +const TEXT = new Set(['.md', '.txt', '.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs', '.py', '.sh', '.json', '.yaml', '.yml', '.toml', '.css', '.html', '.rs', '.go', '.c', '.h', '.cpp', '.java', '.rb']); +const DENIED = /^(?:data|state|config|projects|node_modules|vendor|dist|build|target|coverage|env|venv|__pycache__|credentials?|secrets?)$/i; +const SENSITIVE = /(?:^|[._-])(?:env|environment|auth|credentials?|secrets?|tokens?|passwords?|id_rsa|id_ed25519|id_ecdsa|id_dsa)(?:$|[._-])|\.(?:pem|key|p12|pfx|keystore|lock|log|map|sqlite|db)$/i; +const hash = value => createHash('sha256').update(value).digest('hex'); +const bytes = value => Buffer.byteLength(JSON.stringify(value)); +const stamp = s => [s.dev, s.ino, s.mode, s.nlink, s.size, s.mtimeNs, s.ctimeNs].join(':'); +const metadata = t => ({ name: t.name, description: t.description, parameters: t.parameters, promptGuidelines: t.promptGuidelines, sourceInfo: t.sourceInfo }); +const toolStamp = t => hash(JSON.stringify(metadata(t))); +const activatable = t => READ_TOOLS.has(t.name) && t.sourceInfo?.source === 'builtin' && t.sourceInfo?.path === ``; + +// No caller-controlled shell or argv. Ignore Git environment redirection and +// fsmonitor hooks; ls-files and check-ignore neither run filters nor publish. +function git(root, args, input) { + const env = Object.fromEntries(Object.entries(process.env).filter(([k]) => !k.startsWith('GIT_'))); + return execFileSync('git', ['-c', 'core.fsmonitor=false', '-C', root, ...args], { + env: { ...env, GIT_OPTIONAL_LOCKS: '0', GIT_TERMINAL_PROMPT: '0' }, + encoding: 'utf8', input, maxBuffer: 2 * 1024 * 1024, timeout: 5000, + stdio: ['pipe', 'pipe', 'pipe'], + }); +} + +/** + * @param {{root: string, read?: boolean, exclusions?: string[], + * tools: () => import('@earendil-works/pi-coding-agent').ToolInfo[], + * active: () => string[], + * activate: (tool: import('@earendil-works/pi-coding-agent').ToolInfo) => boolean}} options + */ +export function createAtlas({ root, read = false, exclusions = [], tools, active, activate }) { + const canonicalRoot = realpathSync(root); + if (realpathSync(git(canonicalRoot, ['rev-parse', '--show-toplevel']).trim()) !== canonicalRoot) { + throw new Error('atlas_root_must_be_git_toplevel'); + } + if (exclusions.some(p => typeof p !== 'string' || !p || p.startsWith('/') || p.split('/').some(c => !c || c === '.' || c === '..'))) { + throw new Error('atlas_invalid_exclusion'); + } + let generation = null; + let entries = new Map(); + let ordered = []; + + function allowed(path) { + return typeof path === 'string' && path.length <= 400 && !path.includes('\\') && !/[\x00-\x1f\x7f]/.test(path) && + path.split('/').every(p => p && !p.startsWith('.') && !DENIED.test(p) && !SENSITIVE.test(p)) && + !exclusions.some(p => path === p || path.startsWith(p + '/')); + } + function inventory() { + const paths = [...new Set(git(canonicalRoot, ['ls-files', '-z', '--cached', '--others', '--exclude-standard']).split('\0').filter(Boolean))]; + if (paths.length > 20000) throw new Error('atlas_inventory_too_large'); + if (!paths.length) return []; + let ignored = ''; + try { + ignored = git(canonicalRoot, ['check-ignore', '--no-index', '-z', '--stdin'], paths.join('\0') + '\0'); + } catch (error) { + if (error.status !== 1) throw new Error('atlas_inventory_unavailable'); + } + const excluded = new Set(ignored.split('\0')); + return paths.filter(p => allowed(p) && !excluded.has(p)).sort(); + } + function fileStat(path) { + if (!allowed(path)) throw new Error('atlas_path_denied'); + const absolute = resolve(canonicalRoot, path); + if (relative(canonicalRoot, absolute).split(sep).join('/') !== path || realpathSync(absolute) !== absolute) throw new Error('atlas_path_denied'); + // Reject symlinks at every level, including an in-root directory alias. + let current = canonicalRoot; + for (const part of path.split('/')) { + current = resolve(current, part); + if (lstatSync(current).isSymbolicLink()) throw new Error('atlas_path_denied'); + } + const stat = lstatSync(absolute, { bigint: true }); + if (!stat.isFile() || stat.nlink !== 1n) throw new Error('atlas_path_denied'); + return stat; + } + function refresh() { + const next = []; + for (const path of inventory()) { + let stat; + try { stat = fileStat(path); } catch { continue; } + const version = stamp(stat); + next.push({ kind: 'file', identity: path, handle: 'f:' + hash(canonicalRoot + '/' + path).slice(0, 20) + '.' + hash(version).slice(0, 12), version, size: Number(stat.size) }); + } + for (const t of tools()) { + if (t.name === 'atlas') continue; + if (typeof t.name !== 'string' || t.name.length > 200 || !t.sourceInfo || !t.parameters) throw new Error('atlas_tool_metadata_unavailable'); + const version = toolStamp(t); + next.push({ kind: 'tool', identity: t.name, handle: 't:' + version.slice(0, 32), version }); + } + if (next.length > 22000 || new Set(next.map(e => e.handle)).size !== next.length) throw new Error('atlas_catalog_collision_or_limit'); + entries = new Map(next.map(e => [e.handle, e])); + ordered = next.sort((a, b) => a.handle.localeCompare(b.handle)); + generation = randomBytes(8).toString('hex'); + } + const item = e => ({ kind: e.kind, identity: e.identity, ref: e.handle }); + function current(e) { + if (e.kind === 'tool') { + const t = tools().find(t => t.name === e.identity); + if (!t) return 'unavailable'; + return toolStamp(t) === e.version ? 'fresh' : 'stale'; + } + try { + if (!inventory().includes(e.identity)) return 'excluded'; + return stamp(fileStat(e.identity)) === e.version ? 'fresh' : 'stale'; + } catch { return 'unavailable'; } + } + function readLines(e, at, count) { + if (!read) return { outcome: 'read_not_authorized' }; + if (!(TEXT.has(extname(e.identity).toLowerCase()) || /(?:^|\/)(?:README|LICENSE|Makefile)$/.test(e.identity))) return { outcome: 'unsupported_file_type' }; + if (e.size > MAX_FILE) return { outcome: 'file_too_large', fileBytes: e.size }; + const fd = openSync(resolve(canonicalRoot, e.identity), constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + try { + if (stamp(fstatSync(fd, { bigint: true })) !== e.version) return { outcome: 'stale_handle', freshness: 'stale' }; + const buffer = Buffer.alloc(e.size + 1); + let size = 0; + while (size < buffer.length) { + const n = readSync(fd, buffer, size, buffer.length - size, null); + if (!n) break; + size += n; + } + if (size !== e.size || stamp(fstatSync(fd, { bigint: true })) !== e.version || stamp(fileStat(e.identity)) !== e.version) return { outcome: 'stale_handle', freshness: 'stale' }; + const data = buffer.subarray(0, size); + if (data.includes(0)) return { outcome: 'unsupported_file_type' }; + let text; + try { text = new TextDecoder('utf-8', { fatal: true }).decode(data); } catch { return { outcome: 'unsupported_encoding' }; } + const lines = text.split('\n'); + if (text.endsWith('\n')) lines.pop(); + if (at > lines.length && !(at === 1 && lines.length === 0)) return { outcome: 'line_out_of_range', totalLines: lines.length }; + const content = lines.slice(at - 1, at - 1 + count).join('\n'); + return { outcome: 'read', content, at, lines: Math.min(count, Math.max(0, lines.length - at + 1)), totalLines: lines.length, more: at - 1 + count < lines.length, contentBytes: Buffer.byteLength(content) }; + } finally { closeSync(fd); } + } + + // All callers, including Pi's tool_call argument patches, get strict validation. + return function dispatch(request, signal) { + let result = { generation, identity: null, selection: 'none', freshness: 'not_checked', outcome: 'invalid_request', truncated: false }; + try { + signal?.throwIfAborted(); + const p = request; + if (!p || typeof p !== 'object' || Array.isArray(p) || Object.keys(p).some(k => !['op', 'q', 'ref', 'gen', 'at', 'count'].includes(k)) || + !['catalog', 'resolve', 'inspect', 'read', 'activate', 'refresh'].includes(p.op) || + ['q', 'ref', 'gen'].some(k => p[k] !== undefined && (typeof p[k] !== 'string' || p[k].length > 400)) || + ['at', 'count'].some(k => p[k] !== undefined && (!Number.isSafeInteger(p[k]) || p[k] < 1)) || (p.count !== undefined && p.count > 100)) return finish(result); + if (p.op === 'refresh') { + refresh(); + return finish({ ...result, generation, outcome: 'refreshed', selection: 'explicit_refresh', entries: entries.size }); + } + if (!generation) refresh(); + result.generation = generation; + if (p.gen !== undefined && p.gen !== generation) return finish({ ...result, outcome: 'stale_snapshot', freshness: 'stale' }); + if (p.op === 'catalog' || p.op === 'resolve') { + if (p.op === 'resolve' && !p.q) return finish(result); + const q = p.q ?? ''; + const kind = q.startsWith('f:') ? 'file' : q.startsWith('t:') ? 'tool' : null; + const term = kind ? q.slice(2) : q; + const pool = ordered.filter(e => !kind || e.kind === kind); + const exact = pool.filter(e => e.identity === term); + const matches = exact.length ? exact : pool.filter(e => e.identity.toLowerCase().includes(term.toLowerCase())); + const start = (p.at ?? 1) - 1; + const candidates = matches.slice(start, start + Math.min(p.count ?? MAX_ITEMS, MAX_ITEMS)).map(item); + result = { ...result, selection: exact.length ? 'exact_identity' : 'literal_substring', outcome: matches.length === 0 ? 'not_found' : p.op === 'catalog' ? 'catalog' : matches.length === 1 ? 'resolved' : 'ambiguous', candidates, total: matches.length, more: start + candidates.length < matches.length }; + if (matches.length === 1) result.identity = matches[0].identity; + return finish(result); + } + if (!p.gen) return finish({ ...result, outcome: 'generation_required' }); + const e = entries.get(p.ref); + if (!e) return finish({ ...result, outcome: 'unknown_handle' }); + result = { ...result, identity: e.identity, selection: 'identity_handle', freshness: current(e) }; + if (result.freshness !== 'fresh') return finish({ ...result, outcome: 'stale_handle' }); + if (p.op === 'inspect') { + const info = e.kind === 'file' ? { fileBytes: e.size, readAuthorized: read } : metadata(tools().find(t => t.name === e.identity)); + return finish({ ...result, outcome: 'inspected', info }); + } + if (p.op === 'read' && e.kind === 'file') return finish({ ...result, ...readLines(e, p.at ?? 1, p.count ?? 40) }); + if (p.op === 'activate' && e.kind === 'tool') { + const t = tools().find(t => t.name === e.identity); + if (!activatable(t)) return finish({ ...result, outcome: 'execution_disallowed' }); + // Activation keeps the original definition and its execution hooks. + // It never executes the tool, and only restores an operator-deferred tool. + if (!active().includes(t.name) && !activate(t)) return finish({ ...result, outcome: 'activation_not_authorized' }); + return finish({ ...result, outcome: 'active_for_next_call', tool: t.name }); + } + return finish({ ...result, outcome: 'execution_disallowed' }); + } catch (error) { + return finish({ ...result, outcome: signal?.aborted ? 'cancelled' : 'unavailable', error: error.message?.startsWith('atlas_') ? error.message : 'atlas_operation_unavailable' }); + } + }; +} + +// Count exact model-facing UTF-8 JSON bytes, including the count itself. Oversize +// structured payloads are refused, not cut into invalid JSON or hidden on disk. +function finish(result) { + if (bytes(result) > MAX_RESULT - 80) { + result = { generation: result.generation, identity: result.identity, selection: result.selection, freshness: result.freshness, outcome: 'output_too_large', truncated: true, omittedBytes: bytes(result) }; + } + result.outputBytes = 0; + while (result.outputBytes !== bytes(result)) result.outputBytes = bytes(result); + return result; +} + +export { activatable, toolStamp }; diff --git a/bin/fm-test-run.sh b/bin/fm-test-run.sh index fcfa4b4f43e..bbd9b94d85c 100755 --- a/bin/fm-test-run.sh +++ b/bin/fm-test-run.sh @@ -270,7 +270,7 @@ family_for_basename() { fm-bearings-board.test.sh|\ fm-brief.test.sh|fm-vendor-auth-probe.test.sh|\ fm-calm-pi-extension.test.sh|fm-cd-pretool-check.test.sh|\ - fm-classify-decision-key.test.sh|\ + fm-classify-decision-key.test.sh|fm-context-atlas.test.sh|\ fm-composer-ghost.test.sh|fm-composer-lib.test.sh|\ fm-crew-state.test.sh|fm-captain-hold-lifecycle.test.sh|\ fm-documentation-audiences.test.sh|fm-ensure-agents-md.test.sh|fm-grok-harness.test.sh|\ @@ -334,7 +334,7 @@ family_for_basename() { fm-cmux-claude-composer-live-e2e.test.sh|\ fm-composer-matrix-live-e2e.test.sh|\ fm-codex-continuity-live-e2e.test.sh|fm-grok-continuity-live-e2e.test.sh|\ - fm-cursor-primary-live-e2e.test.sh|\ + fm-cursor-primary-live-e2e.test.sh|fm-context-atlas-live-e2e.test.sh|\ fm-grok-stop-live-e2e.test.sh|fm-harness-adapter-instructions-live-e2e.test.sh|\ fm-harness-liveness-drift-live-e2e.test.sh|\ fm-muse-signals-live-e2e.test.sh|fm-rovo-signals-live-e2e.test.sh|\ @@ -1302,6 +1302,14 @@ families_for_changed_path() { # resolution in the caller; emit a marker family of __script__ printf '%s\n' "__script__:$(basename "$path")" ;; + bin/context-atlas.ts|bin/context-atlas/*) + printf '%s\n' __script__:fm-context-atlas.test.sh + printf '%s\n' __script__:fm-context-atlas-live-e2e.test.sh + printf '%s\n' __script__:fm-pi-primary-types.test.sh + ;; + tests/assets/context-atlas-provider.ts) + printf '%s\n' __script__:fm-context-atlas-live-e2e.test.sh + ;; bin/fm-test-run.sh|bin/fm-test-isolation-proof.sh) # Deliberately the WHOLE family, not just the two contract tests. This # runner executes every pure-contract-unit script, so a change to it is diff --git a/docs/configuration.md b/docs/configuration.md index eb79f0a64b6..55bcc27eae3 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -25,6 +25,10 @@ Wake, watcher, away-mode, and Relay-specific state mechanics remain with their n `AGENTS.md` retains the run-once and read-once operator rules, lock-refusal safety, installation consent, and direct-report recovery boundaries because those facts apply at every session start. Ordinary dead-direct-report recovery is owned by `stuck-crewmate-recovery`, while persistent-secondmate recovery is owned by `secondmate-provisioning`. +## Optional Pi Context Atlas + +[Context Atlas](context-atlas.md) is a default-off, explicitly loaded file/tool catalog experiment; its entry point owns invocation mechanics, and it stores no persistent configuration. + ## Pi Calm preference (config/calm) The Pi Calm extension stores the captain's home-local presentation choice in gitignored `config/calm` under the effective Firstmate home, resolved from `FM_HOME`, then `FM_ROOT_OVERRIDE`, then the tracked code root derived from the extension path, or under `FM_CONFIG_OVERRIDE` when that test and specialized-setup override is present. diff --git a/docs/context-atlas-design.md b/docs/context-atlas-design.md new file mode 100644 index 00000000000..41dd74e89c6 --- /dev/null +++ b/docs/context-atlas-design.md @@ -0,0 +1,59 @@ +# Context Atlas design + +[Operator guidance](context-atlas.md) owns adoption and supported limits; the header and help in [`bin/context-atlas.ts`](../bin/context-atlas.ts) own exact invocation mechanics. +The Pi entry point owns registration and active-tool restoration, while [`bin/context-atlas/catalog.mjs`](../bin/context-atlas/catalog.mjs) owns the deterministic dispatcher contract. +Neither is in an auto-discovered extension directory or referenced by a production launcher. +There is no daemon, persistence format, remote service, dependency installation, or general workflow engine. + +## Public Pi boundary + +The installed extension documentation, SDK documentation, public declarations, and dynamic-tools, kimi-deferred-tools, and tools examples establish the boundary: + +| API | Sufficient for | Not sufficient for | +| --- | --- | --- | +| `registerTool` | One compact typed Atlas entry point | Automatically inheriting another tool's policy | +| `getAllTools` | Name, description, parameter schema, guidelines, source provenance | Executable definitions or a generic invocation API | +| `getActiveTools` / `setActiveTools` | Deferring selected tools and additively restoring the original definition | Arbitrary execution or authority to activate an operator-disabled tool | +| SDK tool factories | Constructing a new known built-in implementation | Recovering another extension's registered implementation or preserving its invocation hooks through a direct call | +| Public custom provider API | Observing tools, prompt text, and real tool results in an offline Pi smoke | Proving remote provider serialization, token billing, or model selection quality | + +Atlas therefore implements deterministic resolution plus activation for original, originally active, built-in read-only tools. +All configured tools can be cataloged, but only the explicit built-in allowlist can be activated, and only if Atlas itself deferred the same metadata identity. +A custom tool named `read` is not treated as the built-in implementation. +Existing tools retain their original typed entry point and execute normally on the next Pi call. +There is no private Pi API, executable definition extraction, or arbitrary shell/argv path. + +The optional file reader is a separate, explicitly authorized read-only adapter, not a proxy call to Pi's `read` tool. +Atlas tool calls go through normal Pi tool events; policies matching only another tool name do not apply automatically. +This distinction is intentional and is reinforced at the read authorization flag and operator guidance. + +## Identity and freshness + +A file reference combines a hash of its canonical repository-root/relative-path identity with a hash of its filesystem freshness evidence. +Freshness evidence includes device, inode, mode, link count, size, and nanosecond modification/change timestamps, so indexing never opens file contents. +Every reference operation also requires the returned random snapshot generation; refresh, reload, and a replacement session invalidate earlier generations. +There are no positional aliases and no fallback from an unknown reference to a path. +A tool reference hashes the public name, schema, description, guidelines, and source metadata; a metadata change or disappearance refuses the old reference. +This cannot attest hidden changes to a tool's implementation that Pi does not expose. + +Discovery uses bounded, fixed-argument Git inventory and ignore queries, including ignore rules for already tracked files. +Paths must pass the default exclusions and operator-added relative prefixes, remain canonical within the selected root, and be regular single-link files without symlink components. +Non-Git directories are refused rather than silently falling back to an unrestricted filesystem walk. +Lookup yields metadata only; focused line reads require the separate read grant, supported UTF-8 text, and the file-size bound. +Before a read, the dispatcher rechecks inventory membership and freshness, opens without following a final symlink, and compares descriptor/path metadata before returning any bytes. +These checks protect ordinary stale references and path mistakes, not an adversarial same-user filesystem race or deliberately disguised secrets. + +Responses carry concrete identity or bounded candidates, generation, selection reason, freshness verdict, action outcome, truncation state, and exact UTF-8 JSON output bytes. +Discovery explicitly reports freshness as not checked; inspection and action recheck the selected identity. +Ambiguous literal matches return candidates rather than selecting the highest-ranked guess. +Oversized structured responses are replaced with a bounded refusal, never invalid JSON, a hidden full-output file, or silent truncation. +Ordinary Pi schema-validation and extension-initialization errors remain Pi errors, outside the dispatcher response envelope. + +## Verification + +The public dispatcher regression is [`tests/fm-context-atlas.test.sh`](../tests/fm-context-atlas.test.sh). +The real explicitly loaded Pi guard and measurement command are owned by [`tests/fm-context-atlas-live-e2e.test.sh`](../tests/fm-context-atlas-live-e2e.test.sh). +That guard substitutes only the model provider, not the extension host, tool registration, execution loop, active-tool updates, original read policy, or shutdown. +It runs without model tokens and records absent Pi explicitly through the shared live-test gate. +The existing [`Pi type check`](../tests/fm-pi-primary-types.test.sh) includes the optional entry point. +Current measurements and inspected compatibility axes belong in the [maintainer verification record](verification/context-atlas.md). diff --git a/docs/context-atlas.md b/docs/context-atlas.md new file mode 100644 index 00000000000..10561f5e9fa --- /dev/null +++ b/docs/context-atlas.md @@ -0,0 +1,35 @@ +# Context Atlas experiment + +Context Atlas is an optional Pi prototype for finding allowed repository files and configured tools through one compact catalog. +It is not a replacement for Firstmate's existing tools or an automatic optimization. +Nothing loads it during ordinary primary, worker, or secondmate startup. + +## Trying it + +Use the explicit-load instructions in [`bin/context-atlas.ts`](../bin/context-atlas.ts), then its `/atlas-help` command for the operation contract, flags, and limits. +No package installation or persistent settings change is needed. +Use a selected Git repository root, not a subdirectory or a filesystem-wide root. +Additional exclusions can narrow that selection. + +The initial catalog contains metadata, not file contents. +Reading eligible text requires separate explicit authorization for Atlas's local reader. +If your environment has restrictions attached specifically to the original `read` tool, do not enable Atlas reads until those restrictions also authorize the new tool. +Hidden, ignored, private, credential, dependency, and build paths are excluded conservatively, but this is not a secret scanner or an operating-system sandbox. +Secrets disguised as ordinary source files and hostile concurrent filesystem writers are outside its protection. + +Tool resolution does not invoke arbitrary tools. +Optional deferral only hides originally active Pi built-in read-only tools; selecting one makes the original tool available for the model's next call, with its original schema and execution checks. +Mutation, publication, credential, lifecycle, and merge operations remain with their existing owners. +The prototype does not alter those tools or grant authority through a handle. +The restore command and normal shutdown restore unchanged deferred definitions without removing other active tools. +Do not combine deferral with another extension that independently manages the same active-tool selection. + +## What the exploration establishes + +The [measurement record](verification/context-atlas.md) compares broad discovery, focused discovery, already-known tools, and Pi's default tool set. +The experiment reduces initial schema bytes when several read-only tools can be deferred, but adds overhead when the default tool set is already small. +A focused native file lookup also returns fewer bytes than Atlas's structured results. +No token-billing, provider-cache, model-reasoning, or production latency improvement is claimed. +Keep it optional unless measurements on your actual workflow justify it. + +The [design owner](context-atlas-design.md) records the public API boundary, identity and freshness model, and verification entry points. diff --git a/docs/documentation-audiences.json b/docs/documentation-audiences.json index 03124cd1628..a8e98a8e53d 100644 --- a/docs/documentation-audiences.json +++ b/docs/documentation-audiences.json @@ -296,6 +296,14 @@ "path": "docs/calm.md", "audience": "operator-current" }, + { + "path": "docs/context-atlas.md", + "audience": "operator-current" + }, + { + "path": "docs/context-atlas-design.md", + "audience": "maintainer-architecture" + }, { "path": "docs/cd-guard.md", "audience": "maintainer-architecture" @@ -432,6 +440,10 @@ "path": "docs/turnend-guard.md", "audience": "operator-current" }, + { + "path": "docs/verification/context-atlas.md", + "audience": "maintainer-verification" + }, { "path": "docs/verification/dispatch-auth.md", "audience": "maintainer-verification" diff --git a/docs/verification/context-atlas.md b/docs/verification/context-atlas.md new file mode 100644 index 00000000000..dddea6b7071 --- /dev/null +++ b/docs/verification/context-atlas.md @@ -0,0 +1,99 @@ +# Context Atlas verification + +This is the active empirical record for the [optional prototype](../context-atlas.md), not a claim of production token savings. +The [design owner](../context-atlas-design.md) owns implementation boundaries. + +## Real Pi proof + +Verified on 2026-09-11 with Pi 0.85.1 on Linux using its real CLI, explicit extension loading, and a local scripted provider. +No credential, network call, installation, global configuration write, or model token was needed. +The provider fixture registers no tool and supplies deterministic calls; Pi itself validates, dispatches, executes, and delivers the results. +The command refreshes the exact candidate under test, rather than a copied or installed extension: + +```sh +FM_CONTEXT_ATLAS_LIVE=1 bin/fm-test-run.sh tests/fm-context-atlas-live-e2e.test.sh +``` + +Exact terminal success line: + +```text +ok - Pi 0.85.1: exact Atlas candidate loaded, real tool calls correct, original read policy preserved, active tools restored, clean exit (offline scripted provider; no model tokens) +``` + +The guard asserts that Atlas is the only added tool, deferral leaves all other original tools active, activation exposes the unchanged original read schema on the next request, a read-specific policy still blocks a subsequent original read call, an allowed original read succeeds, and shutdown restores the original set without changing definitions. +The non-deferred case verifies preservation without activating the optimization. +Every flow reads the independently known declaration `export const refundLimit = 42;` from the selected line in a fixture with 60 unrelated source files. +The test's header owns exact CLI flags and evidence-retention mechanics, including the option terminator before a command-line prompt. + +## Measurement + +Baseline capture preceded implementation using Pi's public tool factories and a native bounded read. +The repeatable paired measurements below use real Pi requests and tool results; all bytes are UTF-8, not estimated tokens. +Schema bytes are JSON of active tool name, description, and parameters at the provider boundary; prompt bytes are the separate system prompt. +Result bytes include every returned content block, including Atlas's identity/freshness envelope. +Latency spans the fixture provider's startup through shutdown, excluding CLI module loading and any network/model reasoning; it is a single local observation, not a benchmark distribution. + +Observed paired run: + +| Flow | Initial/final schema bytes | Prompt bytes | Discovery calls | Result bytes | Duplicate reads | Tool calls | Latency ms | Correct | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- | +| Native broad find + focused read | 4754 / 4754 | 2724 | 1 | 6017 | 0 | 2 | 54.990 | yes | +| Native focused find + focused read | 4754 / 4754 | 2724 | 1 | 159 | 0 | 2 | 50.413 | yes | +| Native already-known read tool | 4754 / 4754 | 2724 | 0 | 110 | 0 | 1 | 37.018 | yes | +| Atlas file resolve + focused read, deferral enabled | 2771 / 2771 | 2544 | 1 | 729 | 0 | 2 | 46.818 | yes | +| Atlas tool resolve + activation + original read policy probe + allowed read | 2771 / 3425 | 2544 | 1 | 754 | 0 | 4 | 56.868 | yes | +| Atlas file flow, no deferral | 5462 / 5462 | 2724 | 1 | 729 | 0 | 2 | 47.059 | yes | +| Pi default four tools, already-known read | 2717 / 2717 | 2622 | 0 | 110 | 0 | 1 | 23.571 | yes | +| Atlas with Pi default four tools, deferral enabled | 2771 / 2771 | 2544 | 1 | 729 | 0 | 2 | 43.978 | yes | + +The seven-tool setup reduces initial active schema bytes by 41.7%, from 4754 to 2771, while original-read activation grows that set again to 3425. +Against the four-tool default, Atlas instead increases schema bytes by 54; without deferral it adds 708 bytes to the seven-tool set. +Broad discovery output falls from 6017 to 729 bytes, but focused native discovery is substantially smaller at 159 bytes. +The safety-probe row deliberately includes one blocked read call and its error result; it is not a matched latency comparison against the single-call native row. +There is no measured discovery-call or duplicate-read improvement in these flows, and resolving/activating an already-known tool adds calls. +The result supports a removable, opt-in exploration for larger tool sets, not default activation or a universal token-saving claim. +Remote provider serialization, caching, billed tokens, natural-language selection accuracy, TUI pixel layout, and non-Linux filesystem behavior were not measured. + +## Portable failure coverage + +```sh +bin/fm-test-run.sh tests/fm-context-atlas.test.sh +``` + +Exact output: + +```text +ok - Atlas public dispatcher: identity, freshness, exclusions, authority, bounds and refusal cases +``` + +Coverage includes stale generation and file metadata, same-size rewrites, newly ignored tracked paths, replaced directory symlinks, out-of-root/ignored/private paths, hardlinks, unknown handles, ambiguity and bounded pagination, invalid line ranges, UTF-8, oversized/binary output, absent or changed tools, read authorization, custom read overrides, and disallowed execution. +The existing Pi type-check entry point includes Atlas and passed with TypeScript 5.9.3 against Pi 0.85.1: + +```sh +bin/fm-test-run.sh tests/fm-pi-primary-types.test.sh +``` + +```text +ok - tracked Pi extensions pass strict no-emit typecheck against Pi 0.85.1 +``` + +## Compatibility inspection + +No startup, supervisor instruction, hook, shared runtime adapter, package manifest, or existing extension changes to load Atlas. +The affected shared runner only classifies/selects its new tests; existing primary/runtime integration behavior is not modified. + +| Axis inspected | Integration surface | Applicability | +| --- | --- | --- | +| Pi and pi-signed primary/secondmate | `.pi/extensions/`, `bin/fm-spawn.sh` Pi launch branches | Ordinary startup unaffected; optional explicit Pi path verified above; signed wrapper not installed or claimed tested | +| Pi workers | Generated per-task extension and explicit `-e` launch in `bin/fm-spawn.sh` | Unaffected; Atlas is not added to generated extensions or launch arguments | +| Claude | `.claude/settings.json` | Not applicable; hooks do not discover TypeScript under `bin/` | +| Codex | `.codex/hooks.json` | Not applicable; startup and pre-tool/stop hooks unchanged | +| OpenCode | `.opencode/plugins/` | Not applicable; plugin discovery and shell checks unchanged | +| Grok | `.grok/hooks/` | Not applicable; configured command hooks unchanged | +| Kimi | Harness reference and global-hook launch integration | Not applicable; no primary guard or worker-hook changes | +| Cursor | `.cursor/hooks.json` | Not applicable; explicit shell/session hooks unchanged | +| OMP | `.omp/extensions/` and launch branches | Not applicable; its separate discovery path does not import Atlas | +| Gemini, Muse, Rovo workers | Harness detection and launch branches | Not applicable; no entry point for this Pi-only extension added | +| tmux, Herdr, zellij, Orca, cmux | `bin/backends/` adapter headers and launch boundary | Not applicable; Atlas neither controls endpoints nor changes session-provider operations | + +No runtime backend was started, stopped, restarted, or otherwise exercised for this experiment. diff --git a/docs/verification/runtime-backends.md b/docs/verification/runtime-backends.md index acaaf0708f7..e1d0a7bf051 100644 --- a/docs/verification/runtime-backends.md +++ b/docs/verification/runtime-backends.md @@ -6,6 +6,10 @@ This record contains reusable version-scoped evidence for active runtime guarant The backend guides own current setup, safety boundaries, and limitations. Exact task chronology, branch names, temporary homes, local paths, process ids, thread ids, and delivery transcripts remain in private reports or PR evidence. +## Optional Pi Context Atlas + +The [Context Atlas verification record](context-atlas.md) owns the 2026-09-11 Pi 0.85.1 explicit-load proof, token-free refresh command, measurements, and unaffected harness/backend inspection. + ## tmux Foreground-process behavior was verified on 2026-07-07 with tmux 3.6a on macOS. diff --git a/tests/assets/context-atlas-provider.ts b/tests/assets/context-atlas-provider.ts new file mode 100644 index 00000000000..7dda96f6fb4 --- /dev/null +++ b/tests/assets/context-atlas-provider.ts @@ -0,0 +1,103 @@ +// Token-free scripted provider driving the REAL Pi tool loop, not a fake host. +// Only the test script explicitly loads this fixture; it registers no tools. +import assert from 'node:assert/strict'; +import { writeFileSync } from 'node:fs'; +import { createAssistantMessageEventStream } from '@earendil-works/pi-ai'; +import type { ExtensionAPI } from '@earendil-works/pi-coding-agent'; + +export default function (pi: ExtensionAPI) { + const mode = process.env.ATLAS_CASE!; + const target = 'src/payments/refund.ts'; + const report: any = { mode, schemaBytes: [], promptBytes: [], calls: [], resultBytes: 0, discoveryCalls: 0, duplicateReads: 0, correct: false }; + const started = performance.now(); + const allNames = mode.endsWith('default') ? ['bash', 'edit', 'read', 'write'] : ['bash', 'edit', 'find', 'grep', 'ls', 'read', 'write']; + const original = new Map(); + let step = 0; + let handle: any; + let previousCall: any; + const readKeys = new Set(); + pi.on('session_start', () => { + const all = pi.getAllTools(); + assert.ok(all.every(t => !('execute' in t)), 'ToolInfo is metadata, not execution'); + assert.deepEqual(all.filter(t => t.sourceInfo.source !== 'builtin').map(t => t.name), mode.startsWith('atlas') ? ['atlas'] : []); + for (const t of all) if (t.sourceInfo.source === 'builtin') original.set(t.name, JSON.stringify(t)); + const expected = mode.startsWith('atlas') && mode !== 'atlas-preserve' ? ['atlas','bash','edit','write'] : [...allNames, ...(mode === 'atlas-preserve' ? ['atlas'] : [])]; + assert.deepEqual(pi.getActiveTools().sort(), expected.sort()); + report.startup = true; + }); + pi.on('tool_call', e => { + report.calls.push(e.toolName); + if (e.toolName === 'read' && e.input.offset === 3) return { block: true, reason: 'ATLAS_TEST_ORIGINAL_READ_POLICY' }; + }); + pi.on('session_shutdown', () => { + assert.deepEqual(pi.getActiveTools().sort(), [...allNames,...(mode.startsWith('atlas') ? ['atlas'] : [])].sort()); + for (const t of pi.getAllTools()) if (original.has(t.name)) assert.equal(JSON.stringify(t), original.get(t.name)); + report.restored = true; + report.latencyMs = Number((performance.now()-started).toFixed(3)); + writeFileSync(process.env.ATLAS_REPORT!, JSON.stringify(report)); + }); + pi.registerProvider('atlas-test', { + baseUrl: 'http://127.0.0.1/unused', apiKey: 'offline-test-only', api: 'atlas-test-api', + models: [{ id:'scripted',name:'Atlas deterministic tool client',reasoning:false,input:['text'],cost:{input:0,output:0,cacheRead:0,cacheWrite:0},contextWindow:64000,maxTokens:1024 }], + streamSimple(model, context) { + const stream = createAssistantMessageEventStream(); + queueMicrotask(() => { + const out: any = {role:'assistant',content:[],api:model.api,provider:model.provider,model:model.id,usage:{input:0,output:0,cacheRead:0,cacheWrite:0,totalTokens:0,cost:{input:0,output:0,cacheRead:0,cacheWrite:0,total:0}},stopReason:'stop',timestamp:Date.now()}; + try { + const schema = context.tools?.map(({name,description,parameters})=>({name,description,parameters})) ?? []; + report.schemaBytes.push(Buffer.byteLength(JSON.stringify(schema))); + report.promptBytes.push(Buffer.byteLength(context.systemPrompt ?? '')); + const last: any = context.messages.filter(m=>m.role==='toolResult').at(-1); + let result: any; + if (previousCall) { + assert.ok(last, 'tool result delivered to next request'); + report.resultBytes += Buffer.byteLength(JSON.stringify(last.content)); + result = last.content.filter((b:any)=>b.type==='text').map((b:any)=>b.text).join('\n'); + if (previousCall.name==='atlas') result=JSON.parse(result); + if (previousCall.name==='find' || (previousCall.name==='atlas' && previousCall.arguments.op==='resolve')) report.discoveryCalls++; + if (previousCall.name==='read' || (previousCall.name==='atlas' && previousCall.arguments.op==='read')) { + const key=JSON.stringify(previousCall.arguments); + if(readKeys.has(key)) report.duplicateReads++; + readKeys.add(key); + } + } + let name = 'atlas'; let args: any; + if (mode.startsWith('baseline')) { + const knownTarget = mode==='baseline-tool' || mode==='baseline-default'; + if (step===0 && !knownTarget) { name='find';args={pattern:mode==='baseline-focused'?'**/refund.ts':'**/*',limit:200}; } + else if ((step===1 && !knownTarget) || (step===0 && knownTarget)) { + if (!knownTarget) assert.ok(result.includes(target)); + name='read'; args={path:target,offset:2,limit:1}; + } else { assert.ok(result.includes('export const refundLimit = 42;'));report.correct=true; } + } else if (mode==='atlas-tool') { + if(step===0) args={op:'resolve',q:'t:read'}; + else if(step===1) {assert.equal(result.outcome,'resolved');handle={ref:result.candidates[0].ref,gen:result.generation};args={op:'activate',...handle};} + else if(step===2) {assert.equal(result.outcome,'active_for_next_call');assert.ok(schema.some(t=>t.name==='read'));name='read';args={path:target,offset:3,limit:1};} + else if(step===3) {assert.equal(last.isError,true);assert.ok(result.includes('ATLAS_TEST_ORIGINAL_READ_POLICY'));name='read';args={path:target,offset:2,limit:1};report.policyPreserved=true;} + else {assert.ok(result.includes('export const refundLimit = 42;'));report.correct=true;} + } else { + if(step===0) args={op:'resolve',q:'f:refund.ts'}; + else if(step===1) {assert.equal(result.outcome,'resolved');assert.equal(result.identity,target);args={op:'read',ref:result.candidates[0].ref,gen:result.generation,at:2,count:1};} + else {assert.equal(result.outcome,'read');assert.equal(result.content,'export const refundLimit = 42;');report.correct=true;} + } + stream.push({type:'start',partial:out}); + if(args) { + assert.ok(schema.some(t=>t.name===name), 'requested tool is active'); + previousCall={type:'toolCall',id:'call-'+step,name,arguments:args}; + out.content=[previousCall];out.stopReason='toolUse'; + stream.push({type:'toolcall_start',contentIndex:0,partial:out}); + stream.push({type:'toolcall_delta',contentIndex:0,delta:JSON.stringify(args),partial:out}); + stream.push({type:'toolcall_end',contentIndex:0,toolCall:previousCall,partial:out}); + } else { + out.content=[{type:'text',text:'ATLAS_SMOKE_CORRECT'}]; + stream.push({type:'text_start',contentIndex:0,partial:out}); + stream.push({type:'text_delta',contentIndex:0,delta:'ATLAS_SMOKE_CORRECT',partial:out}); + stream.push({type:'text_end',contentIndex:0,content:'ATLAS_SMOKE_CORRECT',partial:out}); + } + step++;stream.push({type:'done',reason:out.stopReason,message:out});stream.end(); + } catch(e) {out.stopReason='error';out.errorMessage=String(e);stream.push({type:'error',reason:'error',error:out});stream.end();} + }); + return stream; + }, + }); +} diff --git a/tests/fm-context-atlas-live-e2e.test.sh b/tests/fm-context-atlas-live-e2e.test.sh new file mode 100755 index 00000000000..bdac3389459 --- /dev/null +++ b/tests/fm-context-atlas-live-e2e.test.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# Real explicitly loaded Pi CLI with a token-free scripted provider. Measures +# public prompt/schema and navigation cost without credentials or network. +# Refresh: FM_CONTEXT_ATLAS_LIVE=1 bin/fm-test-run.sh tests/fm-context-atlas-live-e2e.test.sh +# Optional ATLAS_EVIDENCE_DIR retains the eight JSON metric records at that path. +set -eu +# shellcheck source=tests/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +fm_live_gate default-on FM_CONTEXT_ATLAS_LIVE pi node git +TMP_ROOT=$(fm_test_tmproot fm-context-atlas-live) +trap fm_test_cleanup EXIT +mkdir -p "$TMP_ROOT/repo/src/payments" "$TMP_ROOT/agent" +git -C "$TMP_ROOT/repo" init -q +printf '%s\n' '// Refund policy' 'export const refundLimit = 42;' '// original read policy test' > "$TMP_ROOT/repo/src/payments/refund.ts" +for i in $(seq 1 60); do + mkdir -p "$TMP_ROOT/repo/src/component-$i/internal" + printf '%s\n' 'export const unrelated = true;' > "$TMP_ROOT/repo/src/component-$i/internal/implementation.ts" +done +version=$(pi --version) +for mode in baseline-broad baseline-focused baseline-tool atlas-read atlas-tool atlas-preserve baseline-default atlas-default; do + tools=read,bash,edit,write,grep,find,ls,atlas + case "$mode" in *-default) tools=read,bash,edit,write,atlas ;; esac + extensions=() + flags=() + case "$mode" in + atlas*) + extensions=(-e "$ROOT/bin/context-atlas.ts") + flags=(--atlas-read) + [ "$mode" = atlas-preserve ] || flags+=(--atlas-defer) + ;; + esac + ( + cd "$TMP_ROOT/repo" + PI_CODING_AGENT_DIR="$TMP_ROOT/agent" PI_OFFLINE=1 PI_TELEMETRY=0 \ + ATLAS_CASE="$mode" ATLAS_REPORT="$TMP_ROOT/$mode.json" \ + pi --mode json --no-approve --no-session --no-context-files --no-extensions \ + --no-skills --no-prompt-templates --no-themes \ + "${extensions[@]}" -e "$ROOT/tests/assets/context-atlas-provider.ts" \ + --tools "$tools" \ + --model atlas-test/scripted "${flags[@]}" -- 'Read the refundLimit declaration.' + ) > "$TMP_ROOT/$mode.log" 2>&1 || { tail -50 "$TMP_ROOT/$mode.log" >&2; fail "Pi $version: $mode failed"; } + grep -Fq ATLAS_SMOKE_CORRECT "$TMP_ROOT/$mode.log" || { tail -50 "$TMP_ROOT/$mode.log" >&2; fail "Pi $version: $mode did not finish correctly"; } +done +node --input-type=module - "$TMP_ROOT" "$version" <<'JS' +import assert from 'node:assert/strict'; +import {readFileSync} from 'node:fs'; +const [tmp,version]=process.argv.slice(2); +for(const mode of ['baseline-broad','baseline-focused','baseline-tool','atlas-read','atlas-tool','atlas-preserve','baseline-default','atlas-default']) { + const r=JSON.parse(readFileSync(tmp+'/'+mode+'.json','utf8')); + assert.equal(r.startup,true);assert.equal(r.correct,true);assert.equal(r.restored,true); + if(mode==='atlas-tool')assert.equal(r.policyPreserved,true); + console.log(JSON.stringify({mode,schemaBytes:r.schemaBytes[0],finalSchemaBytes:r.schemaBytes.at(-1),promptBytes:r.promptBytes[0],discoveryCalls:r.discoveryCalls,resultBytes:r.resultBytes,duplicateReads:r.duplicateReads,toolCalls:r.calls.length,latencyMs:r.latencyMs,correct:r.correct})); +} +console.log(`ok - Pi ${version}: exact Atlas candidate loaded, real tool calls correct, original read policy preserved, active tools restored, clean exit (offline scripted provider; no model tokens)`); +JS +if [ -n "${ATLAS_EVIDENCE_DIR:-}" ]; then + mkdir -p "$ATLAS_EVIDENCE_DIR" + cp "$TMP_ROOT/"*.json "$ATLAS_EVIDENCE_DIR/" +fi diff --git a/tests/fm-context-atlas.test.sh b/tests/fm-context-atlas.test.sh new file mode 100755 index 00000000000..fd8affe6b4e --- /dev/null +++ b/tests/fm-context-atlas.test.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# Portable public-dispatch contract: identity, freshness, read authority, bounds, +# exclusions and activation refusal. No Pi or model credentials required. +set -eu +# shellcheck source=tests/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +TMP_ROOT=$(fm_test_tmproot fm-context-atlas) +trap fm_test_cleanup EXIT +node --input-type=module - "$ROOT" "$TMP_ROOT" <<'JS' +import assert from 'node:assert/strict'; +import {mkdirSync,writeFileSync,symlinkSync,linkSync,renameSync,rmSync,utimesSync,statSync} from 'node:fs'; +import {execFileSync} from 'node:child_process'; +import {pathToFileURL} from 'node:url'; +const [root,tmp] = process.argv.slice(2); +const {createAtlas} = await import(pathToFileURL(root+'/bin/context-atlas/catalog.mjs')); +const repo=tmp+'/repo'; mkdirSync(repo); +const git=(...args)=>execFileSync('git',['-C',repo,...args]); git('init','-q'); +const empty=createAtlas({root:repo,tools:()=>[],active:()=>[],activate:()=>false}); +assert.equal(empty({op:'catalog'}).outcome,'not_found'); +const put=(path,text)=>{mkdirSync(repo+'/'+path.split('/').slice(0,-1).join('/'),{recursive:true});writeFileSync(repo+'/'+path,text);}; +put('src/alpha.ts','first\nsecond\nthird\n'); put('docs/alpha.md','guide\n'); +put('.gitignore','ignored/\ntracked-ignore.txt\n'); +put('tracked-ignore.txt','hidden'); git('add','-f','tracked-ignore.txt'); +for (const p of ['ignored/a.ts','.env','env.json','src/auth.json','state/public.md','node_modules/a.ts','dist/a.ts','excluded/a.ts','.git/invented','credentials.txt']) put(p,'DO_NOT_READ'); +put('big.txt','x'.repeat(270000)); put('long.txt','x'.repeat(9000)); put('binary.txt','x\0y'); +put('utf.txt','é\n☃\n'); put('empty.txt',''); +writeFileSync(tmp+'/outside.txt','OUTSIDE'); +symlinkSync(tmp+'/outside.txt',repo+'/link.txt'); symlinkSync(tmp,repo+'/escape'); +linkSync(tmp+'/outside.txt',repo+'/hard.txt'); +let list=['read','write'].map(name=>({name,description:name,parameters:{type:'object'},sourceInfo:{source:'builtin',path:``}})); +let active=[]; let invoked=0; +const options={root:repo,read:true,exclusions:['excluded'],tools:()=>list,active:()=>active,activate:t=>{active.push(t.name);invoked++;return true;}}; +const atlas=createAtlas(options); +const call=p=>{const r=atlas(p);assert.equal(r.outputBytes,Buffer.byteLength(JSON.stringify(r)));assert.ok(r.outputBytes<=8192);return r;}; +const resolve=q=>{const r=call({op:'resolve',q});assert.equal(r.outcome,'resolved',q);return {ref:r.candidates[0].ref,gen:r.generation};}; +const read=(h,extra={})=>call({op:'read',...h,...extra}); +const alpha=resolve('f:src/alpha.ts'); +assert.equal(read(alpha,{at:2,count:1}).content,'second'); +assert.equal(call({op:'inspect',...alpha}).info.readAuthorized,true); +assert.equal(call({op:'resolve',q:'alpha'}).outcome,'ambiguous'); +assert.equal(call({op:'resolve',q:'f:alpha'}).candidates.length,2); +assert.equal(call({op:'read',ref:alpha.ref}).outcome,'generation_required'); +assert.equal(read({...alpha,gen:'old'}).outcome,'stale_snapshot'); +assert.equal(read({...alpha,ref:'f:unknown'}).outcome,'unknown_handle'); +assert.equal(read(alpha,{at:0}).outcome,'invalid_request'); +assert.equal(read(alpha,{count:101}).outcome,'invalid_request'); +assert.equal(call({op:'execute',command:'touch BAD'}).outcome,'invalid_request'); +assert.equal(call({op:'read',...alpha,argv:['rm']}).outcome,'invalid_request'); +for (const p of ['ignored/a.ts','tracked-ignore.txt','.env','env.json','src/auth.json','state/public.md','node_modules/a.ts','dist/a.ts','excluded/a.ts','.git/invented','credentials.txt','link.txt','escape/outside.txt','hard.txt','../outside.txt',tmp+'/outside.txt']) assert.equal(call({op:'resolve',q:'f:'+p}).outcome,'not_found',p); +assert.equal(read(resolve('f:big.txt')).outcome,'file_too_large'); +assert.equal(read(resolve('f:long.txt')).outcome,'output_too_large'); +assert.equal(read(resolve('f:binary.txt')).outcome,'unsupported_file_type'); +assert.equal(read(resolve('f:utf.txt'),{at:2,count:1}).content,'☃'); +assert.equal(read(resolve('f:empty.txt')).content,''); +assert.equal(read(alpha,{at:20}).outcome,'line_out_of_range'); +const denied=createAtlas({...options,read:false}); const dr=denied({op:'resolve',q:'f:src/alpha.ts'}); +assert.equal(denied({op:'read',ref:dr.candidates[0].ref,gen:dr.generation}).outcome,'read_not_authorized'); +// Metadata-only indexing must not read an oversized, binary or private file. +// Freshness catches an equal-size rewrite even when mtime is restored. +const s=statSync(repo+'/src/alpha.ts');put('src/alpha.ts','other\nsecond\nthird\n');utimesSync(repo+'/src/alpha.ts',s.atime,s.mtime); +assert.equal(read(alpha).outcome,'stale_handle'); +call({op:'refresh'}); assert.equal(read(alpha).outcome,'stale_snapshot'); +const fresh=resolve('f:src/alpha.ts'); assert.notEqual(fresh.ref,alpha.ref); +put('.gitignore','ignored/\ntracked-ignore.txt\nsrc/alpha.ts\n'); assert.equal(read(fresh).freshness,'excluded'); +put('.gitignore','ignored/\ntracked-ignore.txt\n'); +const guide=resolve('f:docs/alpha.md');renameSync(repo+'/docs',repo+'/old-docs');symlinkSync(tmp,repo+'/docs');assert.equal(read(guide).outcome,'stale_handle'); +const t=resolve('t:read'); assert.equal(call({op:'activate',...t}).outcome,'active_for_next_call');assert.equal(invoked,1); +assert.equal(call({op:'activate',...resolve('t:write')}).outcome,'execution_disallowed'); assert.equal(invoked,1); +list[0]={...list[0],parameters:{type:'object',required:['path']}};assert.equal(call({op:'activate',...t}).freshness,'stale'); +list=[]; assert.equal(call({op:'inspect',...t}).freshness,'unavailable'); +assert.equal(call({op:'activate',...fresh}).outcome,'execution_disallowed'); +const noActivation=createAtlas({...options,tools:()=>[{name:'read',description:'custom override',parameters:{type:'object'},sourceInfo:{source:'extension',path:'/custom'}}],activate:()=>{throw new Error('must not activate');}}); +const override=noActivation({op:'resolve',q:'t:read'});assert.equal(noActivation({op:'activate',ref:override.candidates[0].ref,gen:override.generation}).outcome,'execution_disallowed'); +const disabled=createAtlas({...options,tools:()=>[{name:'read',description:'read',parameters:{type:'object'},sourceInfo:{source:'builtin',path:''}}],active:()=>[],activate:()=>false}); +const off=disabled({op:'resolve',q:'t:read'});assert.equal(disabled({op:'activate',ref:off.candidates[0].ref,gen:off.generation}).outcome,'activation_not_authorized'); +const hugeTool=createAtlas({...options,tools:()=>[{name:'huge',description:'x'.repeat(10000),parameters:{type:'object'},sourceInfo:{source:'sdk',path:'fixture'}}]}); +const huge=hugeTool({op:'resolve',q:'t:huge'});assert.equal(hugeTool({op:'inspect',ref:huge.candidates[0].ref,gen:huge.generation}).outcome,'output_too_large'); +const abort=new AbortController();abort.abort();assert.equal(atlas({op:'catalog'},abort.signal).outcome,'cancelled'); +for(let i=0;i<12;i++)put('page-'+i+'.txt','ok');call({op:'refresh'}); +const page=call({op:'catalog',q:'page-',count:100});assert.equal(page.candidates.length,8);assert.equal(page.more,true); +const next=call({op:'catalog',q:'page-',at:9,gen:page.generation});assert.equal(next.candidates.length,4); +assert.equal(new Set([...page.candidates,...next.candidates].map(c=>c.ref)).size,12); +assert.throws(()=>createAtlas({...options,root:repo+'/src'}),/toplevel/); +rmSync(repo+'/.git',{recursive:true}); assert.throws(()=>createAtlas(options)); +assert.equal(read({ref:page.candidates[0].ref,gen:page.generation}).freshness,'unavailable'); +console.log('ok - Atlas public dispatcher: identity, freshness, exclusions, authority, bounds and refusal cases'); +JS diff --git a/tests/fm-pi-primary-types.test.sh b/tests/fm-pi-primary-types.test.sh index 4746be3e107..4d244e629f0 100755 --- a/tests/fm-pi-primary-types.test.sh +++ b/tests/fm-pi-primary-types.test.sh @@ -26,7 +26,9 @@ cleanup() { } trap cleanup EXIT -mkdir -p "$TMP_ROOT/lib" "$TMP_ROOT/node_modules/@earendil-works" "$TMP_ROOT/node_modules/@types" +mkdir -p "$TMP_ROOT/lib" "$TMP_ROOT/context-atlas" "$TMP_ROOT/node_modules/@earendil-works" "$TMP_ROOT/node_modules/@types" +cp "$ROOT/bin/context-atlas.ts" "$TMP_ROOT/context-atlas.ts" +cp "$ROOT/bin/context-atlas/catalog.mjs" "$TMP_ROOT/context-atlas/catalog.mjs" cp "$ROOT/.pi/extensions/fm-branch-supervision.ts" "$TMP_ROOT/fm-branch-supervision.ts" cp "$ROOT/.pi/extensions/fm-calm.ts" "$TMP_ROOT/fm-calm.ts" cp "$ROOT/.pi/extensions/fm-primary-pi-watch.ts" "$TMP_ROOT/fm-primary-pi-watch.ts" @@ -53,6 +55,7 @@ cat > "$TMP_ROOT/tsconfig.json" <<'JSON' { "compilerOptions": { "allowImportingTsExtensions": true, + "allowJs": true, "module": "NodeNext", "moduleResolution": "NodeNext", "noEmit": true, From 45c9997e45c06668c8e9edef74a75b8267b5076c Mon Sep 17 00:00:00 2001 From: Ilari Ahonen <77334791+CeamenDemon@users.noreply.github.com> Date: Fri, 11 Sep 2026 17:26:46 +0300 Subject: [PATCH 2/2] feat: reuse Atlas snapshots for compact query reads --- bin/context-atlas.ts | 7 ++++- bin/context-atlas/catalog.mjs | 30 +++++++++++++------ docs/context-atlas-design.md | 5 +++- docs/context-atlas.md | 5 ++-- docs/verification/context-atlas.md | 40 +++++++++++++++---------- tests/assets/context-atlas-provider.ts | 14 ++++++++- tests/fm-context-atlas-live-e2e.test.sh | 19 ++++++++---- tests/fm-context-atlas.test.sh | 20 +++++++++++++ 8 files changed, 106 insertions(+), 34 deletions(-) diff --git a/bin/context-atlas.ts b/bin/context-atlas.ts index b691b4b2b95..3c9241141bc 100644 --- a/bin/context-atlas.ts +++ b/bin/context-atlas.ts @@ -15,6 +15,10 @@ * * atlas({op:"resolve",q:"f:README.md"}) -> generation and identity-backed ref. * atlas({op:"read",ref:"",gen:"",at:1,count:20}) + * With a known generation, read may instead take q:"f:". + * Exactly one snapshot match reads immediately; ambiguity returns candidates. + * q and ref are mutually exclusive for reads. New files never retarget a query + * inside an older snapshot; selected-file freshness is checked before reading. * atlas({op:"resolve",q:"t:read"}) then op:"activate" -> original tool next call. * Other ops: catalog (q optional; at is 1-based pagination), inspect, refresh. * q is a literal identity substring, optionally f: or t:, never a shell command. @@ -40,6 +44,7 @@ Scope: --atlas-root /absolute/git-root; --atlas-exclude relative/prefix,another atlas ops: catalog, resolve, inspect, read, activate, refresh. Resolve q is a literal identity substring (optional f: or t: prefix). Use the returned ref and generation as ref/gen for inspect/read/activate. +Read may use a file q instead of ref, but still requires gen and one snapshot match. at/count select 1-based catalog pages or read lines. Refresh invalidates old gen. Limits: 8 candidates, 100 lines, 256 KiB files, 8 KiB JSON results. Read needs --atlas-read and uses a separate local text adapter, not the read tool. @@ -95,7 +100,7 @@ export default function (pi: ExtensionAPI) { } }); pi.registerTool({ name: 'atlas', label: 'Context Atlas', - description: 'Index files/tools: catalog/resolve q (literal, optional f:/t:). inspect/read/activate require returned ref and generation as gen. Read at/count lines; activate original read-only tool for next call. refresh invalidates gen. Max 8 candidates, 100 lines, 8 KiB JSON. No writes or shell.', + description: 'Catalog/resolve literal q (f:/t:). Read ref+gen or file q+gen, at/count lines. Inspect/activate ref+gen; activate original tool next call. Refresh invalidates gen. Read-only, 8 KiB max.', parameters: Type.Object({ op: StringEnum(['catalog', 'resolve', 'inspect', 'read', 'activate', 'refresh']), q: Type.Optional(Type.String({ maxLength: 400 })), diff --git a/bin/context-atlas/catalog.mjs b/bin/context-atlas/catalog.mjs index 2fe5ae1ff5c..6245d8d8aed 100644 --- a/bin/context-atlas/catalog.mjs +++ b/bin/context-atlas/catalog.mjs @@ -154,24 +154,36 @@ export function createAtlas({ root, read = false, exclusions = [], tools, active if (!generation) refresh(); result.generation = generation; if (p.gen !== undefined && p.gen !== generation) return finish({ ...result, outcome: 'stale_snapshot', freshness: 'stale' }); - if (p.op === 'catalog' || p.op === 'resolve') { + const queryRead = p.op === 'read' && p.q !== undefined; + if (queryRead && (p.ref !== undefined || !p.q)) return finish(result); + if (queryRead && !p.gen) return finish({ ...result, outcome: 'generation_required' }); + let selected; + let selectionReason; + if (p.op === 'catalog' || p.op === 'resolve' || queryRead) { if (p.op === 'resolve' && !p.q) return finish(result); const q = p.q ?? ''; const kind = q.startsWith('f:') ? 'file' : q.startsWith('t:') ? 'tool' : null; const term = kind ? q.slice(2) : q; - const pool = ordered.filter(e => !kind || e.kind === kind); + const pool = ordered.filter(e => (!kind || e.kind === kind) && (!queryRead || e.kind === 'file')); const exact = pool.filter(e => e.identity === term); const matches = exact.length ? exact : pool.filter(e => e.identity.toLowerCase().includes(term.toLowerCase())); - const start = (p.at ?? 1) - 1; - const candidates = matches.slice(start, start + Math.min(p.count ?? MAX_ITEMS, MAX_ITEMS)).map(item); - result = { ...result, selection: exact.length ? 'exact_identity' : 'literal_substring', outcome: matches.length === 0 ? 'not_found' : p.op === 'catalog' ? 'catalog' : matches.length === 1 ? 'resolved' : 'ambiguous', candidates, total: matches.length, more: start + candidates.length < matches.length }; - if (matches.length === 1) result.identity = matches[0].identity; - return finish(result); + selectionReason = exact.length ? 'exact_identity' : 'literal_substring'; + if (queryRead && matches.length === 1) { + selected = matches[0]; + } else { + // For a query read, at/count are line controls, never candidate paging. + const start = queryRead ? 0 : (p.at ?? 1) - 1; + const limit = queryRead ? MAX_ITEMS : Math.min(p.count ?? MAX_ITEMS, MAX_ITEMS); + const candidates = matches.slice(start, start + limit).map(item); + result = { ...result, selection: selectionReason, outcome: matches.length === 0 ? 'not_found' : p.op === 'catalog' ? 'catalog' : matches.length === 1 ? 'resolved' : 'ambiguous', candidates, total: matches.length, more: start + candidates.length < matches.length }; + if (matches.length === 1) result.identity = matches[0].identity; + return finish(result); + } } if (!p.gen) return finish({ ...result, outcome: 'generation_required' }); - const e = entries.get(p.ref); + const e = selected ?? entries.get(p.ref); if (!e) return finish({ ...result, outcome: 'unknown_handle' }); - result = { ...result, identity: e.identity, selection: 'identity_handle', freshness: current(e) }; + result = { ...result, identity: e.identity, selection: selected ? 'snapshot_' + selectionReason : 'identity_handle', freshness: current(e), ...(selected ? { ref: e.handle } : {}) }; if (result.freshness !== 'fresh') return finish({ ...result, outcome: 'stale_handle' }); if (p.op === 'inspect') { const info = e.kind === 'file' ? { fileBytes: e.size, readAuthorized: read } : metadata(tools().find(t => t.name === e.identity)); diff --git a/docs/context-atlas-design.md b/docs/context-atlas-design.md index 41dd74e89c6..6017bf1f395 100644 --- a/docs/context-atlas-design.md +++ b/docs/context-atlas-design.md @@ -45,7 +45,10 @@ These checks protect ordinary stale references and path mistakes, not an adversa Responses carry concrete identity or bounded candidates, generation, selection reason, freshness verdict, action outcome, truncation state, and exact UTF-8 JSON output bytes. Discovery explicitly reports freshness as not checked; inspection and action recheck the selected identity. -Ambiguous literal matches return candidates rather than selecting the highest-ranked guess. +Literal queries prefer an exact identity, otherwise use case-insensitive substring matching; ambiguity returns candidates rather than a guessed winner. +A file query can combine resolution and reading only within an explicitly named existing snapshot and with exactly one match. +It uses the stored identity and ordinary selected-file freshness checks, never retargets to a newly indexed path, and still requires the separate read grant. +This avoids an extra lookup for a subsequent file without changing tool execution or introducing another cache. Oversized structured responses are replaced with a bounded refusal, never invalid JSON, a hidden full-output file, or silent truncation. Ordinary Pi schema-validation and extension-initialization errors remain Pi errors, outside the dispatcher response envelope. diff --git a/docs/context-atlas.md b/docs/context-atlas.md index 10561f5e9fa..7647583cbf5 100644 --- a/docs/context-atlas.md +++ b/docs/context-atlas.md @@ -12,6 +12,7 @@ Use a selected Git repository root, not a subdirectory or a filesystem-wide root Additional exclusions can narrow that selection. The initial catalog contains metadata, not file contents. +After acquiring a snapshot, reading a uniquely matching file query can avoid a separate discovery call without silently selecting newly added files. Reading eligible text requires separate explicit authorization for Atlas's local reader. If your environment has restrictions attached specifically to the original `read` tool, do not enable Atlas reads until those restrictions also authorize the new tool. Hidden, ignored, private, credential, dependency, and build paths are excluded conservatively, but this is not a secret scanner or an operating-system sandbox. @@ -27,8 +28,8 @@ Do not combine deferral with another extension that independently manages the sa ## What the exploration establishes The [measurement record](verification/context-atlas.md) compares broad discovery, focused discovery, already-known tools, and Pi's default tool set. -The experiment reduces initial schema bytes when several read-only tools can be deferred, but adds overhead when the default tool set is already small. -A focused native file lookup also returns fewer bytes than Atlas's structured results. +The experiment reduces initial schema bytes when several read-only tools can be deferred, with a smaller measured saving for the default four-tool set. +A two-file flow also avoids one separate discovery call by reusing its snapshot, but a focused native file lookup still returns fewer bytes than Atlas's structured results. No token-billing, provider-cache, model-reasoning, or production latency improvement is claimed. Keep it optional unless measurements on your actual workflow justify it. diff --git a/docs/verification/context-atlas.md b/docs/verification/context-atlas.md index dddea6b7071..aa7ede5b609 100644 --- a/docs/verification/context-atlas.md +++ b/docs/verification/context-atlas.md @@ -22,7 +22,9 @@ ok - Pi 0.85.1: exact Atlas candidate loaded, real tool calls correct, original The guard asserts that Atlas is the only added tool, deferral leaves all other original tools active, activation exposes the unchanged original read schema on the next request, a read-specific policy still blocks a subsequent original read call, an allowed original read succeeds, and shutdown restores the original set without changing definitions. The non-deferred case verifies preservation without activating the optimization. -Every flow reads the independently known declaration `export const refundLimit = 42;` from the selected line in a fixture with 60 unrelated source files. +All eleven flows read the independently known declaration `export const refundLimit = 42;` from the selected line in a fixture with 60 unrelated source files. +The three two-file flows also read `export const chargeLimit = 7;` and compare native discovery, separate Atlas resolution, and snapshot query/read. +The guard rejects default-tool schema overhead and requires the snapshot flow to finish correctly with one discovery-only call and three total calls. The test's header owns exact CLI flags and evidence-retention mechanics, including the option terminator before a command-line prompt. ## Measurement @@ -31,26 +33,33 @@ Baseline capture preceded implementation using Pi's public tool factories and a The repeatable paired measurements below use real Pi requests and tool results; all bytes are UTF-8, not estimated tokens. Schema bytes are JSON of active tool name, description, and parameters at the provider boundary; prompt bytes are the separate system prompt. Result bytes include every returned content block, including Atlas's identity/freshness envelope. +Discovery counts are standalone model-facing tool calls, not internal searches; duplicate reads count repeated identical read arguments. Latency spans the fixture provider's startup through shutdown, excluding CLI module loading and any network/model reasoning; it is a single local observation, not a benchmark distribution. Observed paired run: -| Flow | Initial/final schema bytes | Prompt bytes | Discovery calls | Result bytes | Duplicate reads | Tool calls | Latency ms | Correct | +| Flow | Initial/final schema bytes | Prompt bytes | Discovery-only calls | Result bytes | Duplicate reads | Tool calls | Latency ms | Correct | | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- | -| Native broad find + focused read | 4754 / 4754 | 2724 | 1 | 6017 | 0 | 2 | 54.990 | yes | -| Native focused find + focused read | 4754 / 4754 | 2724 | 1 | 159 | 0 | 2 | 50.413 | yes | -| Native already-known read tool | 4754 / 4754 | 2724 | 0 | 110 | 0 | 1 | 37.018 | yes | -| Atlas file resolve + focused read, deferral enabled | 2771 / 2771 | 2544 | 1 | 729 | 0 | 2 | 46.818 | yes | -| Atlas tool resolve + activation + original read policy probe + allowed read | 2771 / 3425 | 2544 | 1 | 754 | 0 | 4 | 56.868 | yes | -| Atlas file flow, no deferral | 5462 / 5462 | 2724 | 1 | 729 | 0 | 2 | 47.059 | yes | -| Pi default four tools, already-known read | 2717 / 2717 | 2622 | 0 | 110 | 0 | 1 | 23.571 | yes | -| Atlas with Pi default four tools, deferral enabled | 2771 / 2771 | 2544 | 1 | 729 | 0 | 2 | 43.978 | yes | - -The seven-tool setup reduces initial active schema bytes by 41.7%, from 4754 to 2771, while original-read activation grows that set again to 3425. -Against the four-tool default, Atlas instead increases schema bytes by 54; without deferral it adds 708 bytes to the seven-tool set. -Broad discovery output falls from 6017 to 729 bytes, but focused native discovery is substantially smaller at 159 bytes. +| Native broad find + focused read | 4754 / 4754 | 2724 | 1 | 5989 | 0 | 2 | 40.693 | yes | +| Native focused find + focused read | 4754 / 4754 | 2724 | 1 | 159 | 0 | 2 | 45.686 | yes | +| Native already-known read tool | 4754 / 4754 | 2724 | 0 | 110 | 0 | 1 | 26.183 | yes | +| Atlas file resolve + focused read, deferral enabled | 2672 / 2672 | 2544 | 1 | 729 | 0 | 2 | 54.149 | yes | +| Atlas tool resolve + activation + original read policy probe + allowed read | 2672 / 3326 | 2544 | 1 | 754 | 0 | 4 | 52.371 | yes | +| Atlas file flow, no deferral | 5363 / 5363 | 2724 | 1 | 729 | 0 | 2 | 45.683 | yes | +| Pi default four tools, already-known read | 2717 / 2717 | 2622 | 0 | 110 | 0 | 1 | 24.713 | yes | +| Atlas with Pi default four tools, deferral enabled | 2672 / 2672 | 2544 | 1 | 729 | 0 | 2 | 44.235 | yes | +| Native focused discovery + reads, two files | 4754 / 4754 | 2724 | 2 | 317 | 0 | 4 | 55.941 | yes | +| Atlas separate resolve + reads, two files | 2672 / 2672 | 2544 | 2 | 1458 | 0 | 4 | 48.368 | yes | +| Atlas snapshot query/read for second file | 2672 / 2672 | 2544 | 1 | 1130 | 0 | 3 | 50.248 | yes | + +The seven-tool setup reduces initial active schema bytes by 43.8%, from 4754 to 2672, while original-read activation grows that set again to 3326. +Against the four-tool default, Atlas saves 45 schema bytes (1.7%); without deferral it still adds 609 bytes to the seven-tool set. +This reduction comes from compact description wording, not removing typed fields or safety validation. +Broad discovery output falls from 5989 to 729 bytes, but focused native discovery is substantially smaller at 159 bytes. +For two files, reusing the first lookup's snapshot generation combines the second file's resolution and read: discovery-only calls fall from two to one, total calls from four to three, and Atlas result bytes from 1458 to 1130. +The combined call still performs an internal lookup and freshness checks; it does not avoid that work, and native focused results remain smaller at 317 bytes. The safety-probe row deliberately includes one blocked read call and its error result; it is not a matched latency comparison against the single-call native row. -There is no measured discovery-call or duplicate-read improvement in these flows, and resolving/activating an already-known tool adds calls. +There is no measured duplicate-read improvement, and resolving/activating an already-known tool still adds calls. The result supports a removable, opt-in exploration for larger tool sets, not default activation or a universal token-saving claim. Remote provider serialization, caching, billed tokens, natural-language selection accuracy, TUI pixel layout, and non-Linux filesystem behavior were not measured. @@ -67,6 +76,7 @@ ok - Atlas public dispatcher: identity, freshness, exclusions, authority, bounds ``` Coverage includes stale generation and file metadata, same-size rewrites, newly ignored tracked paths, replaced directory symlinks, out-of-root/ignored/private paths, hardlinks, unknown handles, ambiguity and bounded pagination, invalid line ranges, UTF-8, oversized/binary output, absent or changed tools, read authorization, custom read overrides, and disallowed execution. +Query reads additionally cover missing/stale generations, conflicting selectors, ambiguity without contents, file-only selection, and refusal to retarget after the original file disappears even when a new matching file exists. The existing Pi type-check entry point includes Atlas and passed with TypeScript 5.9.3 against Pi 0.85.1: ```sh diff --git a/tests/assets/context-atlas-provider.ts b/tests/assets/context-atlas-provider.ts index 7dda96f6fb4..2d4272c27c5 100644 --- a/tests/assets/context-atlas-provider.ts +++ b/tests/assets/context-atlas-provider.ts @@ -62,7 +62,19 @@ export default function (pi: ExtensionAPI) { } } let name = 'atlas'; let args: any; - if (mode.startsWith('baseline')) { + if (mode==='baseline-twofiles') { + if(step===0) {name='find';args={pattern:'**/refund.ts'};} + else if(step===1) {assert.ok(result.includes(target));name='read';args={path:target,offset:2,limit:1};} + else if(step===2) {assert.ok(result.includes('export const refundLimit = 42;'));name='find';args={pattern:'**/charge.ts'};} + else if(step===3) {assert.ok(result.includes('src/payments/charge.ts'));name='read';args={path:'src/payments/charge.ts',offset:2,limit:1};} + else {assert.ok(result.includes('export const chargeLimit = 7;'));report.correct=true;} + } else if (mode==='atlas-twofiles-legacy' || mode==='atlas-twofiles-snapshot') { + if(step===0) args={op:'resolve',q:'f:refund.ts'}; + else if(step===1) {assert.equal(result.outcome,'resolved');handle={ref:result.candidates[0].ref,gen:result.generation};args={op:'read',...handle,at:2,count:1};} + else if(step===2) {assert.equal(result.content,'export const refundLimit = 42;');args=mode==='atlas-twofiles-snapshot'?{op:'read',q:'f:charge.ts',gen:handle.gen,at:2,count:1}:{op:'resolve',q:'f:charge.ts',gen:handle.gen};} + else if(step===3 && mode==='atlas-twofiles-legacy') {assert.equal(result.outcome,'resolved');args={op:'read',ref:result.candidates[0].ref,gen:result.generation,at:2,count:1};} + else {assert.equal(result.content,'export const chargeLimit = 7;');report.correct=true;} + } else if (mode.startsWith('baseline')) { const knownTarget = mode==='baseline-tool' || mode==='baseline-default'; if (step===0 && !knownTarget) { name='find';args={pattern:mode==='baseline-focused'?'**/refund.ts':'**/*',limit:200}; } else if ((step===1 && !knownTarget) || (step===0 && knownTarget)) { diff --git a/tests/fm-context-atlas-live-e2e.test.sh b/tests/fm-context-atlas-live-e2e.test.sh index bdac3389459..02ee6423c41 100755 --- a/tests/fm-context-atlas-live-e2e.test.sh +++ b/tests/fm-context-atlas-live-e2e.test.sh @@ -2,7 +2,7 @@ # Real explicitly loaded Pi CLI with a token-free scripted provider. Measures # public prompt/schema and navigation cost without credentials or network. # Refresh: FM_CONTEXT_ATLAS_LIVE=1 bin/fm-test-run.sh tests/fm-context-atlas-live-e2e.test.sh -# Optional ATLAS_EVIDENCE_DIR retains the eight JSON metric records at that path. +# Optional ATLAS_EVIDENCE_DIR retains the per-flow JSON metric records at that path. set -eu # shellcheck source=tests/lib.sh . "$(dirname "${BASH_SOURCE[0]}")/lib.sh" @@ -12,14 +12,17 @@ trap fm_test_cleanup EXIT mkdir -p "$TMP_ROOT/repo/src/payments" "$TMP_ROOT/agent" git -C "$TMP_ROOT/repo" init -q printf '%s\n' '// Refund policy' 'export const refundLimit = 42;' '// original read policy test' > "$TMP_ROOT/repo/src/payments/refund.ts" +printf '%s\n' '// Charge policy' 'export const chargeLimit = 7;' > "$TMP_ROOT/repo/src/payments/charge.ts" for i in $(seq 1 60); do mkdir -p "$TMP_ROOT/repo/src/component-$i/internal" printf '%s\n' 'export const unrelated = true;' > "$TMP_ROOT/repo/src/component-$i/internal/implementation.ts" done version=$(pi --version) -for mode in baseline-broad baseline-focused baseline-tool atlas-read atlas-tool atlas-preserve baseline-default atlas-default; do +for mode in baseline-broad baseline-focused baseline-tool atlas-read atlas-tool atlas-preserve baseline-default atlas-default baseline-twofiles atlas-twofiles-legacy atlas-twofiles-snapshot; do tools=read,bash,edit,write,grep,find,ls,atlas case "$mode" in *-default) tools=read,bash,edit,write,atlas ;; esac + prompt='Read the refundLimit declaration.' + case "$mode" in *twofiles*) prompt='Read the refundLimit and chargeLimit declarations.' ;; esac extensions=() flags=() case "$mode" in @@ -37,7 +40,7 @@ for mode in baseline-broad baseline-focused baseline-tool atlas-read atlas-tool --no-skills --no-prompt-templates --no-themes \ "${extensions[@]}" -e "$ROOT/tests/assets/context-atlas-provider.ts" \ --tools "$tools" \ - --model atlas-test/scripted "${flags[@]}" -- 'Read the refundLimit declaration.' + --model atlas-test/scripted "${flags[@]}" -- "$prompt" ) > "$TMP_ROOT/$mode.log" 2>&1 || { tail -50 "$TMP_ROOT/$mode.log" >&2; fail "Pi $version: $mode failed"; } grep -Fq ATLAS_SMOKE_CORRECT "$TMP_ROOT/$mode.log" || { tail -50 "$TMP_ROOT/$mode.log" >&2; fail "Pi $version: $mode did not finish correctly"; } done @@ -45,12 +48,18 @@ node --input-type=module - "$TMP_ROOT" "$version" <<'JS' import assert from 'node:assert/strict'; import {readFileSync} from 'node:fs'; const [tmp,version]=process.argv.slice(2); -for(const mode of ['baseline-broad','baseline-focused','baseline-tool','atlas-read','atlas-tool','atlas-preserve','baseline-default','atlas-default']) { - const r=JSON.parse(readFileSync(tmp+'/'+mode+'.json','utf8')); +const records={}; +for(const mode of ['baseline-broad','baseline-focused','baseline-tool','atlas-read','atlas-tool','atlas-preserve','baseline-default','atlas-default','baseline-twofiles','atlas-twofiles-legacy','atlas-twofiles-snapshot']) { + const r=JSON.parse(readFileSync(tmp+'/'+mode+'.json','utf8'));records[mode]=r; assert.equal(r.startup,true);assert.equal(r.correct,true);assert.equal(r.restored,true); if(mode==='atlas-tool')assert.equal(r.policyPreserved,true); console.log(JSON.stringify({mode,schemaBytes:r.schemaBytes[0],finalSchemaBytes:r.schemaBytes.at(-1),promptBytes:r.promptBytes[0],discoveryCalls:r.discoveryCalls,resultBytes:r.resultBytes,duplicateReads:r.duplicateReads,toolCalls:r.calls.length,latencyMs:r.latencyMs,correct:r.correct})); } +assert.ok(records['atlas-default'].schemaBytes[0] <= records['baseline-default'].schemaBytes[0], 'no default-tool schema overhead'); +assert.ok(records['atlas-read'].schemaBytes[0] < records['baseline-broad'].schemaBytes[0], 'seven-tool schema reduction'); +assert.equal(records['atlas-twofiles-snapshot'].discoveryCalls,1); +assert.equal(records['atlas-twofiles-legacy'].discoveryCalls,2); +assert.equal(records['atlas-twofiles-snapshot'].calls.length,3); console.log(`ok - Pi ${version}: exact Atlas candidate loaded, real tool calls correct, original read policy preserved, active tools restored, clean exit (offline scripted provider; no model tokens)`); JS if [ -n "${ATLAS_EVIDENCE_DIR:-}" ]; then diff --git a/tests/fm-context-atlas.test.sh b/tests/fm-context-atlas.test.sh index fd8affe6b4e..4ec2517b069 100755 --- a/tests/fm-context-atlas.test.sh +++ b/tests/fm-context-atlas.test.sh @@ -36,6 +36,17 @@ const resolve=q=>{const r=call({op:'resolve',q});assert.equal(r.outcome,'resolve const read=(h,extra={})=>call({op:'read',...h,...extra}); const alpha=resolve('f:src/alpha.ts'); assert.equal(read(alpha,{at:2,count:1}).content,'second'); +const queryRead=(q,gen=alpha.gen,extra={})=>call({op:'read',q,gen,...extra}); +assert.equal(call({op:'read',q:'f:src/alpha.ts'}).outcome,'generation_required'); +assert.equal(queryRead('f:src/alpha.ts',alpha.gen,{at:2,count:1}).content,'second'); +assert.equal(queryRead('f:src/alpha.ts').ref,alpha.ref); +assert.equal(queryRead('f:src/alpha.ts').selection,'snapshot_exact_identity'); +const ambiguous=queryRead('f:alpha',alpha.gen,{at:999,count:1}); +assert.equal(ambiguous.outcome,'ambiguous');assert.equal(ambiguous.candidates.length,2);assert.equal(ambiguous.content,undefined); +assert.equal(queryRead('f:src/alpha.ts','old').outcome,'stale_snapshot'); +assert.equal(queryRead('f:src/alpha.ts',alpha.gen,{ref:alpha.ref}).outcome,'invalid_request'); +assert.equal(queryRead('t:read').outcome,'not_found'); +assert.equal(queryRead('../outside.txt').outcome,'not_found'); assert.equal(call({op:'inspect',...alpha}).info.readAuthorized,true); assert.equal(call({op:'resolve',q:'alpha'}).outcome,'ambiguous'); assert.equal(call({op:'resolve',q:'f:alpha'}).candidates.length,2); @@ -55,10 +66,12 @@ assert.equal(read(resolve('f:empty.txt')).content,''); assert.equal(read(alpha,{at:20}).outcome,'line_out_of_range'); const denied=createAtlas({...options,read:false}); const dr=denied({op:'resolve',q:'f:src/alpha.ts'}); assert.equal(denied({op:'read',ref:dr.candidates[0].ref,gen:dr.generation}).outcome,'read_not_authorized'); +assert.equal(denied({op:'read',q:'f:src/alpha.ts',gen:dr.generation}).outcome,'read_not_authorized'); // Metadata-only indexing must not read an oversized, binary or private file. // Freshness catches an equal-size rewrite even when mtime is restored. const s=statSync(repo+'/src/alpha.ts');put('src/alpha.ts','other\nsecond\nthird\n');utimesSync(repo+'/src/alpha.ts',s.atime,s.mtime); assert.equal(read(alpha).outcome,'stale_handle'); +assert.equal(queryRead('f:src/alpha.ts').outcome,'stale_handle'); call({op:'refresh'}); assert.equal(read(alpha).outcome,'stale_snapshot'); const fresh=resolve('f:src/alpha.ts'); assert.notEqual(fresh.ref,alpha.ref); put('.gitignore','ignored/\ntracked-ignore.txt\nsrc/alpha.ts\n'); assert.equal(read(fresh).freshness,'excluded'); @@ -76,6 +89,13 @@ const off=disabled({op:'resolve',q:'t:read'});assert.equal(disabled({op:'activat const hugeTool=createAtlas({...options,tools:()=>[{name:'huge',description:'x'.repeat(10000),parameters:{type:'object'},sourceInfo:{source:'sdk',path:'fixture'}}]}); const huge=hugeTool({op:'resolve',q:'t:huge'});assert.equal(hugeTool({op:'inspect',ref:huge.candidates[0].ref,gen:huge.generation}).outcome,'output_too_large'); const abort=new AbortController();abort.abort();assert.equal(atlas({op:'catalog'},abort.signal).outcome,'cancelled'); +put('src/rebind.ts','original');call({op:'refresh'}); +const rebind=resolve('f:src/rebind.ts'); +put('src/new-rebind.ts','replacement'); +assert.equal(queryRead('f:rebind.ts',rebind.gen).content,'original'); +rmSync(repo+'/src/rebind.ts'); +const removed=queryRead('f:rebind.ts',rebind.gen); +assert.equal(removed.outcome,'stale_handle');assert.equal(removed.identity,'src/rebind.ts');assert.equal(removed.content,undefined); for(let i=0;i<12;i++)put('page-'+i+'.txt','ok');call({op:'refresh'}); const page=call({op:'catalog',q:'page-',count:100});assert.equal(page.candidates.length,8);assert.equal(page.more,true); const next=call({op:'catalog',q:'page-',at:9,gen:page.generation});assert.equal(next.candidates.length,4);