diff --git a/benchmarks/search/manifest.json b/benchmarks/search/manifest.json new file mode 100644 index 00000000..9d616d21 --- /dev/null +++ b/benchmarks/search/manifest.json @@ -0,0 +1,128 @@ +{ + "schemaVersion": "dotaios-search-benchmark/v1", + "benchmarkId": "search-baseline-2026-08-13", + "referenceMachine": { + "identifier": "mac16-1-m4-10c-16gb", + "modelIdentifier": "Mac16,1", + "chip": "Apple M4", + "cpuCores": 10, + "memoryGiB": 16, + "operatingSystem": "macOS 26.6.1 build 25G76 (Darwin 25.6.0)", + "powerProfile": { + "source": "battery", + "lowPowerMode": false, + "requireIdleMachine": true + } + }, + "runtime": { + "supportedNodeMajors": [ + 20, + 22 + ], + "baselineNode": "22.22.3" + }, + "corpus": { + "generator": { + "version": "dotaios-search-fixture-v1", + "seed": 20260813, + "fixedMtime": "2026-08-13T00:00:00.000Z" + }, + "fileCounts": [ + 500, + 2500, + 10000 + ], + "layouts": { + "shallow": { + "kind": "bucketed", + "directoryDepth": 1, + "bucketCount": 2 + }, + "nested": { + "kind": "tree", + "directoryDepth": 3, + "branchingFactor": 8 + } + }, + "distributions": { + "prose": { + "kind": "representative-prose", + "targetBytes": { + "min": 768, + "max": 1280 + }, + "vocabularySize": 192, + "frontmatterEvery": 4 + }, + "high-entropy": { + "kind": "adversarial-high-entropy", + "targetBytes": { + "min": 768, + "max": 1280 + }, + "tokenLength": 18, + "frontmatterEvery": 4 + } + }, + "scenarioMatrix": [ + { + "layout": "shallow", + "distribution": "prose" + }, + { + "layout": "nested", + "distribution": "high-entropy" + } + ] + }, + "queries": [ + { + "id": "no-hit", + "text": "zqxj-unfindable-20260813", + "expectation": { + "kind": "none", + "hitCount": 0 + } + }, + { + "id": "low-hit", + "text": "controlled-peregrine-benchmark-needle", + "expectation": { + "kind": "fixed-indices", + "fileIndices": [ + 3, + 17, + 101, + 307 + ], + "hitCount": 4 + } + }, + { + "id": "high-hit", + "text": "common-benchmark-marker-needle", + "expectation": { + "kind": "modulo", + "modulo": 25, + "remainder": 0, + "resultLimit": 20 + } + } + ], + "protocol": { + "coldSamples": 3, + "warmupSamples": 3, + "measuredSamples": 20, + "concurrency": 32, + "resultLimit": 20, + "rssPollIntervalMs": 5, + "coldDefinition": "fresh request-scoped reader before harness warm-up; operating-system file cache is uncontrolled", + "warmDefinition": "fresh request-scoped reader after warm-up in the same Node process", + "rawReadControl": { + "enabled": true, + "method": "open-handle-read-all", + "validation": "exact file count and byte total", + "concurrency": 32 + } + } +} diff --git a/docs/adr/0003-keep-canonical-memory-separate-from-derived-views.md b/docs/adr/0003-keep-canonical-memory-separate-from-derived-views.md index 66f1f72a..69c43dc5 100644 --- a/docs/adr/0003-keep-canonical-memory-separate-from-derived-views.md +++ b/docs/adr/0003-keep-canonical-memory-separate-from-derived-views.md @@ -15,7 +15,7 @@ host-specific views without making their loss or corruption a loss of memory. | --- | --- | --- | --- | | Durable user context | User-authored files under `context/`, project records, decisions, and daily memory | The person directly, or a command they explicitly invoke for that exact record | Working context, search results, generated summaries | | Source material | Provenance-bearing files under `vault/` and other explicit imports | Explicit ingest/capture commands; later edits remain the person's | Search snippets, source indexes | -| Recent event and signal memory | Append-only records under `memory/` | Explicit capture/log workflows and configured local automations with a named write contract | Bounded startup selection, search results, archives produced by explicit maintenance | +| Recent event and signal memory | Append-only live records and bounded archive shards under `memory/` | Explicit capture/log workflows and configured local automations with a named write contract; maintenance may move complete records into canonical cold-storage shards | Bounded startup selection and search results | | Session evidence | Readable session Markdown under `memory/sessions//` | Explicit save/import or a separately enabled host-capture workflow | `memory/sessions/index.jsonl`, working-context selections, search results | | Managed scaffold | Files or marked regions DotAIOS can prove it owns | Previewed setup, activation, migration, repair, disconnect, or removal operations | Installation inventory and health reports | | Operational evidence | Receipts, recovery metadata, locks, metrics, and quarantine material | The exact operation that owns the artifact | Status and doctor summaries | @@ -35,6 +35,14 @@ the Markdown durably before its index entry becomes visible, reconcile must recover orphans without deleting evidence, and delete must prove ownership of the exact canonical file before changing either representation. +For event and signal memory, an archive shard is canonical cold storage once +maintenance removes those records from the live file. It is not a disposable +index or cache: search must include it, crash recovery must preserve exact +record multiplicity, and maintenance may delete or replace it only through the +documented ownership-checked archive protocol. Rotation markers, transaction +envelopes, and format witnesses remain operational evidence; they prove or +recover a state transition but are not user memory themselves. + Rejected alternatives: - Treat the session index as authoritative: a torn or tampered row could hide diff --git a/docs/advanced-memory.md b/docs/advanced-memory.md index 23c6e96d..4bd2d9c9 100644 --- a/docs/advanced-memory.md +++ b/docs/advanced-memory.md @@ -32,6 +32,30 @@ The audit never deletes memory. By default it follows DotAIOS memory routing: the last 50 `memory/events.jsonl` entries plus today/yesterday signal files. Use `--all-memory` only when you want a deeper forensic pass over older history. +Older events and trimmed signals remain canonical JSONL. DotAIOS keeps each +active archive below 2 MiB, rotating complete records into immutable numbered +files (`events-archive.000001.jsonl`, for example). Search reads those shards +automatically; there is no index to rebuild and no archived record becomes +write-only. A single record may occupy its own shard up to the 4 MiB safe-read +ceiling. Anything larger stops maintenance before the source is removed. +Rotation uses a durable format witness and crash marker. An ambiguous archive +created by the older markerless rotator is left byte-for-byte unchanged and +reports `DOTAIOS_ARCHIVE_LEGACY_RECOVERY_REQUIRED` for explicit inspection; +DotAIOS never guesses whether an identical shard prefix is a retry or a +legitimate duplicate. A restart also repairs the narrowly proven two-name +hard-link state left when a process dies during exclusive publication, while +rejecting unrelated links. + +Execution-time runway check (2026-08-13): the live AIOS event archive is +199,021 bytes / 725 lines and the signal archive is 66,177 bytes / 203 lines. +At the observed Git-visible rates of roughly 30 event lines/day and 10 signal +lines/day since 2026-07-27, the coarse per-file runways were about 480 and +1,240 days under the former 4 MiB single-file limit, or about 230 and 620 days +to the new 2 MiB rotation point from this snapshot. Rotation is therefore a +durability bound, not a reason to add a database or persistent search index. +The existing signal archive was an eligible legacy 0644 file; the next locked +maintenance run narrows that exact safe case to 0600 before publication. + `--write-queue` writes proposed skill patches to `memory/skill-patches/queue.md` with stable IDs, so cleanup or compaction does not duplicate the same lesson. If the queue is intentionally capped, the report diff --git a/docs/architecture.md b/docs/architecture.md index 3f0594c4..6047089a 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -122,6 +122,101 @@ a bounded `operational.migration` sibling; selection, and `resolve_skill` routes workflow intent. There are no compatibility aliases. +### On-demand search + +Markdown search is a request-scoped safe corpus transaction. The evidence +reader enumerates eligible regular files and returns transaction-owned +`filePath`, UTF-8 `content`, and `mtimeMs` observations to one callback. +Canonical matching, snippet construction, whole-corpus IDF statistics, recency +ranking, stable ordering, and the result limit all run inside that callback. +The search promise cannot resolve until the evidence reader has performed its +final root, ancestor, and observed-directory generation validation; a changed +generation rejects the request without publishing a partial result. + +The transaction preserves each logical corpus boundary and its source policy: +daily and inbox notes remain separate from memory streams, plugin search accepts +Markdown plus `manifest.json`, project search reads only the resolved selector, +and an external vault remains its own explicitly authorized root. Hidden and +secret-like entries remain ineligible. Linked, non-regular, changed, invalid +UTF-8, unauthorized, misconfigured, or unexpectedly unreadable observed +evidence rejects the whole request. + +Resource ceilings are different. One request-owned discovery transaction uses +phase-local fair ledgers before metadata inspection or catalog reads can spend +the shared, non-releasable physical ledger. Half of each currently available +byte, file, and entry ceiling is reserved as equal protected shares; unused +capacity is redistributed in declared order. The same rule is applied to the +bounded catalog discovery needed for exact JSONL entry counts or session +membership. Session discovery replays the public reverse-order filters, query, +and limit, so it retains each body at most once and never charges a body that a +title, agent, or project hit makes unnecessary. Retained catalog/body bytes are +never reread. Only scopes whose remaining work fits have their ordinary content +read and are tokenized and ranked. Otherwise the whole scope is omitted so +partial-corpus IDF and ranking are never presented as complete. Every inspected +file plus each directory, ancestor, and root observation remains +transaction-owned and is revalidated before results resolve; all phase readers +and prepared capabilities close on success or failure. + +Successful search arrays retain their iterable group shape and expose frozen, +non-enumerable `scope` and `omissions` metadata. Omissions use the five primary +closed reason codes `file_too_large`, `directory_entries_exceeded`, +`aggregate_bytes_exceeded`, `file_count_exceeded`, and +`entry_count_exceeded`; the explicit aggregate-remainder reason is +`omissions_truncated`; contain bounded counts and path-free recovery text; and +are capped at 32 records plus one defensive aggregate remainder. A directory +ceiling is `partially_enumerated`; other ceiling omissions are `not_searched`. +Every observed directory, including a directory stopped at its ceiling, is +revalidated after ranking and before results resolve. The CLI prints valid +results to stdout, warnings to stderr, and exits 2 for incomplete searches. +Exit 0 is complete, including zero hits, while integrity and configuration +failures remain exit 1. + +Search writes no index, cache, manifest, or other derived state. Each request +enumerates the current canonical files, so additions, edits, and deletions are +visible on the next request. The optimization amortizes repeated containment +checks only for the lifetime of that request; the AIOS folder remains the sole +search authority. + +### Bounded memory archives + +Event compaction and stale-signal trimming keep the unsuffixed +`events-archive.jsonl` and `signals-archive.jsonl` files as active append +targets. Before an append would cross 2 MiB, maintenance publishes complete +JSONL records into immutable, zero-padded shards such as +`events-archive.000001.jsonl`. Numbered shards are searched in numeric order, +then the active archive. Exact retry overlap is deduplicated before corpus +statistics and ranking, so an interruption cannot turn one event into two +search results. + +One valid record above 2 MiB but no larger than the 4 MiB evidence-file ceiling +occupies a shard by itself. A larger record stops maintenance before the live +event generation is replaced or a stale signal source is removed. The pending +batch remains recovery authority until shard and active-file publication have +been fsynced. A durable `*.rotation-format` witness separates new +marker-protocol generations from ambiguous overlap left by the older +markerless rotator. If the witness is absent and the newest shard is an exact +prefix of the active archive, maintenance fails before mutation with +`DOTAIOS_ARCHIVE_LEGACY_RECOVERY_REQUIRED`; an operator can inspect both +authoritative copies instead of DotAIOS guessing whether equal records are a +retry or legitimate duplicates. Each shard is created exclusively at mode 0600 +and is never overwritten; active and pending files must be owned, regular, +single-link files. Maintenance narrowly secures an eligible legacy 0644 active +archive to 0600, but rejects links, wrong ownership, broader modes, and unsafe +pre-existing shard targets. + +Exclusive publication links an owned UUID temporary into its final name. If a +real process death leaves those two names on the same inode, restart recovery +removes only the single proven temporary, fsyncs the directory, and revalidates +the final file as the same owned, mode-0600, single-link object. Any different +hard-link state remains fatal. + +Search observes the memory directory before reading the numbered generation +and revalidates it before results resolve. A concurrent rotation therefore +returns the complete old generation, the complete new generation, or a fatal +source-changed retry—never an accepted mixture. Eventually, many valid shards +can exhaust the request-wide search ceiling; the resource-ceiling contract +above then reports the whole memory scope as an explicit omission. + ## Vault `vault/` is long-term knowledge, loaded on demand. Users may keep it inside `~/aios/vault` or configure an external `vault_path` in `aios.json`, such as an Obsidian vault. diff --git a/docs/benchmarks/2026-08-13-search-baseline.md b/docs/benchmarks/2026-08-13-search-baseline.md new file mode 100644 index 00000000..4e3f4818 --- /dev/null +++ b/docs/benchmarks/2026-08-13-search-baseline.md @@ -0,0 +1,199 @@ +# Search benchmark baseline — 2026-08-13 + +This is the pre-optimization receipt for the current safe, contained Markdown +search path. It is a comparison authority for U5/U6, not a performance pass: +the 10,000-file baseline misses the under-one-second R1 gate while returning the +exact controlled results. + +## Authority and protocol + +- Manifest: `benchmarks/search/manifest.json` +- Manifest SHA-256: `b6c38cb5920f91b0a84c66be8181f6c14f7a1c73360fa4f07993b32a7704d55a` +- Generator: `dotaios-search-fixture-v1`, seed `20260813`, fixed mtime + `2026-08-13T00:00:00.000Z` +- Reference machine: `mac16-1-m4-10c-16gb` (`Mac16,1`, Apple M4, 10 cores, + 16 GiB RAM), macOS 26.6.1 build 25G76 / Darwin 25.6.0 +- Runtime: Node 22.22.3, arm64. The manifest and deterministic test also fix + Node 20/22 as the supported comparison majors. +- Power profile: battery, no Low Power Mode, idle-machine requirement. The + battery was at 69%, discharging, with no AC charger at the start. +- Per operation: 3 pre-warm-up samples, 3 warm-up samples, then 20 measured + samples. The warm median/p95 are the decision statistics; the cold columns + describe the three fresh-reader pre-warm-up observations (their p95 is the + maximum of three), not a replacement for R1's 20-sample warmed gate. +- “Cold” means a fresh request-scoped evidence reader before harness warm-up; + operating-system file cache state is intentionally uncontrolled. “Warm” uses + a fresh reader after warm-up in the same Node process. +- Peak RSS is the maximum absolute process RSS observed for any sample in that + row, not a heap delta. Operation counts are deterministic per sample. +- Every timed search sample was accepted only after exact ordered source + equality. The raw control opened and read every inventory file and validated + the exact file count and byte total. After timing, the harness re-hashed the + actual fixture inventory before returning the report. +- A supplemental `raw-search-control-v1` uses the same frozen sampling fields + without adding a manifest field or changing its receipt. That control is + fixed by the harness schema as + `harness-schema-v1-reusing-frozen-manifest-sampling`: it uses the existing + 3 cold / 3 warm-up / 20 measured protocol for each query. It was measured in + a later control-only pass on the same machine, Node, battery power source, and + disabled Low Power Mode (battery 56% at the start). No safe contained sample + or bytes-only raw-read sample was rerun. +- The fixture and raw baseline reports used for this document were generated + in an external temporary directory outside the repository. The committed + reports under `docs/benchmarks/reports/` are separate repository artifacts. + +The nested topology is a bounded, shared three-level tree with branching factor +8 (at most 512 leaf directory chains), rather than a unique ancestor chain per +file. The shallow topology uses two shared buckets. The matrix pairs +shallow/prose and nested/high-entropy at 500, 2,500, and 10,000 files. + +## Immutable fixture receipts + +| Fixture | Source bytes | Inventory SHA-256 | +| --- | ---: | --- | +| 500 / shallow / prose | 543,978 | `db6ab0454118d7cd1e2c54c3519db2018fddcba75e124105f13f1345dc525306` | +| 500 / nested / high-entropy | 519,370 | `16d7f2dc55c48230a2a65d71e93453245cd4bfd6b562701cd5c5cc95c3a5fe19` | +| 2,500 / shallow / prose | 2,731,459 | `229da6c47d147660684099dacb22af3362a3467b2e607a6823dd7d8bc582a910` | +| 2,500 / nested / high-entropy | 2,582,784 | `42178a1a3d649a5223662730f7373ce1505289feefbb88dd2afad3918194e537` | +| 10,000 / shallow / prose | 10,886,041 | `50a3e8026807256715a8f7ea5ffcb8d35c8a074d04c08badbf4e27c6119008c6` | +| 10,000 / nested / high-entropy | 10,319,743 | `cc818211f579b4c54fcacfaa42ed195be052d156c5161c1ccdf2f23cdd1bf9a8` | + +Changing a corpus, query, machine, runtime, or sampling field changes the +manifest receipt. Regenerating a file, path, byte, or fixed mtime changes its +inventory receipt. An old row must not be compared after either hash changes. + +## Timings and file operations + +Times are milliseconds. `RSS MiB` is the maximum across cold and warm samples. + +| Fixture | Operation | Cold median | Cold p95 | Warm median | Warm p95 | RSS MiB | lstat | realpath | open | +| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| 500 / shallow / prose | no-hit | 115.45 | 138.57 | 105.74 | 111.40 | 111.7 | 18,554 | 6,018 | 500 | +| 500 / shallow / prose | low-hit | 106.16 | 107.50 | 106.48 | 109.07 | 112.5 | 18,554 | 6,018 | 500 | +| 500 / shallow / prose | high-hit | 106.52 | 107.03 | 106.58 | 107.91 | 112.8 | 18,554 | 6,018 | 500 | +| 500 / shallow / prose | raw-read | 7.24 | 7.40 | 6.85 | 7.32 | 115.1 | 0 | 0 | 500 | +| 500 / nested / high-entropy | no-hit | 938.10 | 969.52 | 932.91 | 942.54 | 115.4 | 56,174 | 13,432 | 500 | +| 500 / nested / high-entropy | low-hit | 936.43 | 937.50 | 944.89 | 958.30 | 114.9 | 56,174 | 13,432 | 500 | +| 500 / nested / high-entropy | high-hit | 950.66 | 951.17 | 952.01 | 958.66 | 115.1 | 56,174 | 13,432 | 500 | +| 500 / nested / high-entropy | raw-read | 8.25 | 9.53 | 6.94 | 7.48 | 115.5 | 0 | 0 | 500 | +| 2,500 / shallow / prose | no-hit | 539.00 | 548.46 | 525.63 | 560.88 | 114.5 | 92,554 | 30,018 | 2,500 | +| 2,500 / shallow / prose | low-hit | 529.74 | 531.23 | 526.73 | 535.64 | 114.4 | 92,554 | 30,018 | 2,500 | +| 2,500 / shallow / prose | high-hit | 528.83 | 532.53 | 522.84 | 528.05 | 114.5 | 92,554 | 30,018 | 2,500 | +| 2,500 / shallow / prose | raw-read | 33.97 | 34.82 | 32.40 | 32.97 | 117.2 | 0 | 0 | 2,500 | +| 2,500 / nested / high-entropy | no-hit | 4,362.47 | 4,364.22 | 4,357.28 | 4,415.01 | 133.0 | 214,558 | 53,510 | 2,500 | +| 2,500 / nested / high-entropy | low-hit | 4,412.81 | 4,435.83 | 4,424.79 | 4,473.72 | 134.0 | 214,558 | 53,510 | 2,500 | +| 2,500 / nested / high-entropy | high-hit | 4,448.70 | 4,464.04 | 4,423.31 | 4,467.79 | 134.3 | 214,558 | 53,510 | 2,500 | +| 2,500 / nested / high-entropy | raw-read | 35.63 | 37.20 | 33.95 | 34.25 | 131.6 | 0 | 0 | 2,500 | +| 10,000 / shallow / prose | no-hit | 2,090.84 | 2,097.96 | 2,069.22 | 2,089.91 | 124.6 | 370,054 | 120,018 | 10,000 | +| 10,000 / shallow / prose | low-hit | 2,109.78 | 2,118.19 | 2,086.30 | 2,103.04 | 123.9 | 370,054 | 120,018 | 10,000 | +| 10,000 / shallow / prose | high-hit | 2,089.16 | 2,096.40 | 2,090.27 | 2,109.22 | 124.2 | 370,054 | 120,018 | 10,000 | +| 10,000 / shallow / prose | raw-read | 133.07 | 138.83 | 132.52 | 135.39 | 126.4 | 0 | 0 | 10,000 | +| 10,000 / nested / high-entropy | no-hit | 15,102.39 | 15,229.67 | 15,151.46 | 15,219.07 | 390.0 | 807,058 | 203,510 | 10,000 | +| 10,000 / nested / high-entropy | low-hit | 15,025.84 | 15,169.58 | 15,126.71 | 15,253.17 | 391.6 | 807,058 | 203,510 | 10,000 | +| 10,000 / nested / high-entropy | high-hit | 15,043.19 | 15,272.44 | 15,213.70 | 15,588.76 | 392.4 | 807,058 | 203,510 | 10,000 | +| 10,000 / nested / high-entropy | raw-read | 139.18 | 145.01 | 138.47 | 161.99 | 346.0 | 0 | 0 | 10,000 | + +## Unsafe benchmark-only raw-search control + +This is the residual-logic comparison for U5/U6. It is deliberately unsafe and +benchmark-only: `searchMarkdownDir` still runs the canonical matching, +snippet-building, corpus-statistics, and ranking code unchanged, but its reader +does not perform containment, ancestor, directory-generation, or budget +validation. `listFiles` returns only paths from the immutable receipt inventory; +`readText(..., { returnSnapshot: true })` uses an `O_NOFOLLOW` open, handle read, +and `fstat`, then returns the file content and mtime to canonical search. The harness +verifies the fixture inventory before and after the matrix and validates exact +ordered results before accepting every sample. This reader lives only in +`scripts/bench-search.mjs`; it is not a production reader or a safe design +candidate. + +Times are milliseconds. `RSS MiB` is the maximum across cold and warm samples. +Every row reports zero `lstat` and `realpath` operations; `open` is exactly one +per fixture file. + +| Fixture | Query | Cold median | Cold p95 | Warm median | Warm p95 | RSS MiB | open | +| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | +| 500 / shallow / prose | no-hit | 21.10 | 22.04 | 17.66 | 18.59 | 104.7 | 500 | +| 500 / shallow / prose | low-hit | 19.01 | 19.16 | 18.17 | 18.68 | 105.1 | 500 | +| 500 / shallow / prose | high-hit | 18.32 | 18.92 | 18.18 | 18.80 | 115.4 | 500 | +| 500 / nested / high-entropy | no-hit | 21.48 | 22.42 | 17.82 | 19.37 | 118.1 | 500 | +| 500 / nested / high-entropy | low-hit | 19.41 | 19.89 | 18.50 | 19.38 | 121.0 | 500 | +| 500 / nested / high-entropy | high-hit | 18.51 | 19.58 | 18.66 | 19.58 | 121.3 | 500 | +| 2,500 / shallow / prose | no-hit | 91.97 | 101.71 | 89.06 | 90.88 | 127.1 | 2,500 | +| 2,500 / shallow / prose | low-hit | 92.91 | 94.24 | 92.44 | 93.98 | 127.3 | 2,500 | +| 2,500 / shallow / prose | high-hit | 92.16 | 92.69 | 92.38 | 93.94 | 127.0 | 2,500 | +| 2,500 / nested / high-entropy | no-hit | 99.30 | 107.37 | 94.65 | 98.34 | 138.6 | 2,500 | +| 2,500 / nested / high-entropy | low-hit | 98.16 | 99.65 | 97.61 | 98.91 | 139.3 | 2,500 | +| 2,500 / nested / high-entropy | high-hit | 98.12 | 99.28 | 98.45 | 107.01 | 139.3 | 2,500 | +| 10,000 / shallow / prose | no-hit | 361.55 | 373.33 | 360.98 | 364.56 | 179.4 | 10,000 | +| 10,000 / shallow / prose | low-hit | 377.75 | 380.63 | 372.26 | 374.39 | 179.4 | 10,000 | +| 10,000 / shallow / prose | high-hit | 374.41 | 375.01 | 373.03 | 375.67 | 180.2 | 10,000 | +| 10,000 / nested / high-entropy | no-hit | 433.46 | 436.53 | 421.78 | 433.64 | 407.6 | 10,000 | +| 10,000 / nested / high-entropy | low-hit | 436.42 | 447.08 | 436.25 | 446.53 | 356.6 | 10,000 | +| 10,000 / nested / high-entropy | high-hit | 437.22 | 447.55 | 439.98 | 455.68 | 397.9 | 10,000 | + +## Exact controlled results + +Every fixture returned these expectations on every cold, warm-up, and measured +sample: + +- `no-hit` (`zqxj-unfindable-20260813`): exactly `[]`. +- `low-hit` (`controlled-peregrine-benchmark-needle`): exactly four sources, + ordered lexically by their generated path. At 10,000 files the shallow order + is `vault/bucket-01/note-00003.md`, `vault/bucket-01/note-00017.md`, + `vault/bucket-01/note-00101.md`, `vault/bucket-01/note-00307.md`. The nested + order is `vault/branch-00/branch-00/branch-03/note-00003.md`, + `vault/branch-00/branch-02/branch-01/note-00017.md`, + `vault/branch-01/branch-04/branch-05/note-00101.md`, + `vault/branch-04/branch-06/branch-03/note-00307.md`. +- `high-hit` (`common-benchmark-marker-needle`): exactly the first 20 ordered + sources whose numeric file index is divisible by 25. This produces 20 + validated results at every count/topology without allowing a truncated or + empty run to pass. + +The full exact paths are reproducible from the manifest and are embedded in +each external fixture receipt. Their order is also covered by the deterministic +generator test; changing a query expectation invalidates the manifest receipt. + +## Decision + +The pre-change safe path fails R1 at 10,000 files in both representative matrix +cells: warm p95 is about 2.11 seconds for shallow/prose and 15.59 seconds for +nested/high-entropy. Raw-read warm p95 is 135.39 ms and 161.99 ms respectively. +Unsafe raw-search warm p95 is 364.56–375.67 ms and 433.64–455.68 ms +respectively, so canonical matching, snippets, corpus statistics, and ranking +remain below R1 when isolated from containment. This control is explanatory +only: its unsafe reader can never count as the optimized safe result. +The gap is accompanied by 370,054/120,018 and 807,058/203,510 +`lstat`/`realpath` calls for 10,000 safe reads, while `open` remains exactly one +per file. This makes containment/path-validation multiplication, not raw source +reading or result validation, the falsifiable optimization target. U5/U6 must +preserve the exact outputs and safety contract while reducing that overhead; +an empty result, reordered result, or changed receipt is a failed benchmark. + +## Reproduction + +Use an empty destination outside the repository. Example: + +```bash +node scripts/bench-search.mjs receipt +node scripts/bench-search.mjs generate \ + --output /tmp/dotaios-search-10000-shallow-prose \ + --receipt /tmp/dotaios-search-10000-shallow-prose.receipt.json \ + --count 10000 --layout shallow --distribution prose +node scripts/bench-search.mjs run \ + --fixture /tmp/dotaios-search-10000-shallow-prose \ + --receipt /tmp/dotaios-search-10000-shallow-prose.receipt.json \ + --output /tmp/dotaios-search-10000-shallow-prose.report.json +node scripts/bench-search.mjs raw-search \ + --fixture /tmp/dotaios-search-10000-shallow-prose \ + --receipt /tmp/dotaios-search-10000-shallow-prose.receipt.json \ + --output /tmp/dotaios-search-10000-shallow-prose.raw-search.report.json +``` + +Repeat with `--layout nested --distribution high-entropy` and with counts 500 +and 2500. The harness exits nonzero on manifest/fixture receipt mismatch, +unsafe or changed inventory, search error, empty controlled output, exact-order +mismatch, unstable sample output, or raw-read file/byte mismatch. +The `raw-search` command runs only the unsafe benchmark-only residual control; +it does not rerun the contained or bytes-only controls. diff --git a/docs/benchmarks/2026-08-13-search-final.md b/docs/benchmarks/2026-08-13-search-final.md new file mode 100644 index 00000000..704c6edf --- /dev/null +++ b/docs/benchmarks/2026-08-13-search-final.md @@ -0,0 +1,107 @@ +# Search benchmark final receipt — 2026-08-13 + +This is the post-review release receipt for request-scoped safe search. It +supersedes the earlier U6 implementation receipt after the request-wide budget, +final file-generation, and archive retry-provenance fixes. Every measured +sample was accepted only after inventory and exact ordered-result validation. + +## Authority and protocol + +- Manifest: `benchmarks/search/manifest.json` +- Manifest SHA-256: `b6c38cb5920f91b0a84c66be8181f6c14f7a1c73360fa4f07993b32a7704d55a` +- Full reports: [`reports/`](./reports/), named + `2026-08-13---.report.json` +- Machine: `Mac16,1`, Apple M4, 10 cores, 16 GiB; macOS 26.6.1 / Darwin + 25.6.0; Node 22.22.3 arm64. +- Each operation used 3 cold samples, 3 warm-up samples, and 20 measured warm + samples with a fresh request-scoped reader for every safe sample. +- Safe and full unsafe canonical-search samples validated exact ordered paths + and output SHA-256 before their duration was accepted. Safe-corpus and + bytes-only samples validated exact file count and byte total. Fixture + inventories were hashed before and after each run. + +The full unsafe canonical-search control retains matching, snippets, corpus +statistics, ranking, fixture selection, queries, limits, and result validation; +it omits containment only. The bytes-only `raw-read` control remains an +informational lower bound under the accepted +[performance amendment](../plans/2026-08-13-002-search-performance-gate-amendment.md). + +## End-to-end safe search + +Times are milliseconds. RSS is the largest cold or warm process RSS observed +for that query. + +| Fixture | Query | Cold median | Cold p95 | Warm median | Warm p95 | RSS MiB | lstat | realpath | open | +| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| 500 / shallow / prose | no-hit | 41.56 | 45.73 | 27.86 | 33.00 | 112.25 | 1,032 | 510 | 500 | +| 500 / shallow / prose | low-hit | 29.56 | 30.09 | 27.24 | 32.69 | 117.95 | 1,032 | 510 | 500 | +| 500 / shallow / prose | high-hit | 28.54 | 29.10 | 27.91 | 29.80 | 137.39 | 1,032 | 510 | 500 | +| 500 / nested / high-entropy | no-hit | 189.14 | 209.63 | 177.98 | 181.80 | 138.95 | 9,417 | 2,217 | 500 | +| 500 / nested / high-entropy | low-hit | 181.27 | 181.72 | 178.69 | 181.22 | 140.88 | 9,417 | 2,217 | 500 | +| 500 / nested / high-entropy | high-hit | 178.02 | 179.70 | 179.38 | 181.11 | 141.47 | 9,417 | 2,217 | 500 | +| 2,500 / shallow / prose | no-hit | 139.81 | 159.15 | 133.15 | 135.46 | 138.06 | 5,032 | 2,510 | 2,500 | +| 2,500 / shallow / prose | low-hit | 145.44 | 154.89 | 135.89 | 138.63 | 138.23 | 5,032 | 2,510 | 2,500 | +| 2,500 / shallow / prose | high-hit | 136.34 | 138.19 | 135.23 | 139.36 | 138.45 | 5,032 | 2,510 | 2,500 | +| 2,500 / nested / high-entropy | no-hit | 328.41 | 338.79 | 303.40 | 310.75 | 252.30 | 13,610 | 4,256 | 2,500 | +| 2,500 / nested / high-entropy | low-hit | 306.03 | 310.97 | 306.56 | 337.06 | 218.78 | 13,610 | 4,256 | 2,500 | +| 2,500 / nested / high-entropy | high-hit | 310.97 | 323.86 | 307.03 | 313.11 | 219.69 | 13,610 | 4,256 | 2,500 | +| 10,000 / shallow / prose | no-hit | 536.22 | 560.91 | 533.65 | 551.29 | 196.23 | 20,032 | 10,010 | 10,000 | +| 10,000 / shallow / prose | low-hit | 544.01 | 545.65 | 544.50 | 553.32 | 202.69 | 20,032 | 10,010 | 10,000 | +| 10,000 / shallow / prose | high-hit | 544.93 | 612.68 | 546.44 | 556.88 | 203.58 | 20,032 | 10,010 | 10,000 | +| 10,000 / nested / high-entropy | no-hit | 816.74 | 846.10 | 805.27 | 820.61 | 448.63 | 28,610 | 11,756 | 10,000 | +| 10,000 / nested / high-entropy | low-hit | 832.96 | 901.37 | 813.54 | 844.55 | 446.73 | 28,610 | 11,756 | 10,000 | +| 10,000 / nested / high-entropy | high-hit | 817.33 | 893.53 | 817.52 | 835.11 | 450.06 | 28,610 | 11,756 | 10,000 | + +## Relative and informational controls + +| Fixture | Safe warm p95, no/low/high | Full unsafe warm p95, no/low/high | Safe/unsafe ratio, no/low/high | Safe-corpus p95 | Raw-read p95 | +| --- | ---: | ---: | ---: | ---: | ---: | +| 500 / shallow / prose | 33.00 / 32.69 / 29.80 | 19.93 / 20.11 / 20.48 | 1.66 / 1.63 / 1.45 | 21.43 | 8.00 | +| 500 / nested / high-entropy | 181.80 / 181.22 / 181.11 | 20.65 / 24.04 / 21.24 | 8.80 / 7.54 / 8.53 | 173.31 | 7.91 | +| 2,500 / shallow / prose | 135.46 / 138.63 / 139.36 | 101.63 / 105.10 / 102.95 | 1.33 / 1.32 / 1.35 | 93.24 | 37.68 | +| 2,500 / nested / high-entropy | 310.75 / 337.06 / 313.11 | 109.65 / 111.84 / 123.48 | 2.83 / 3.01 / 2.54 | 254.32 | 38.00 | +| 10,000 / shallow / prose | 551.29 / 553.32 / 556.88 | 391.06 / 404.83 / 416.80 | 1.41 / 1.37 / 1.34 | 406.20 | 145.45 | +| 10,000 / nested / high-entropy | 820.61 / 844.55 / 835.11 | 493.24 / 506.84 / 523.92 | 1.66 / 1.67 / 1.59 | 551.26 | 149.60 | + +The relative gap is topology-sensitive because the safe path revalidates roots, +ancestors, directories, and prepared file generations while the unsafe control +does none of that. It is recorded as a diagnostic; it does not replace R1, +parity, safety, regression, or scaling gates. + +## Gate verdicts + +- **R1 / AE1: PASS.** All six 10,000-file warm p95 values are below 1,000 ms. + Worst is 844.55 ms with 155.45 ms headroom. +- **Exact parity: PASS.** Every safe result SHA-256 equals both the frozen U8 + authority and the corresponding full unsafe canonical-search control. +- **500-file regression: PASS.** Worst shallow p95 is 33.00 ms and worst nested + p95 is 181.80 ms. Both remain below the frozen contained + [500-file baselines](./2026-08-13-search-baseline.md): shallow + 111.40 / 109.07 / 107.91 ms and nested 942.54 / 958.30 / 958.66 ms for + no-hit / low-hit / high-hit, plus the larger of 20% or 50 ms. +- **Relative diagnostic: EXCEPTION.** The 10,000-file nested safe/full-unsafe + ratios are 1.66x / 1.67x / 1.59x for no-hit / low-hit / high-hit. The governing + [performance amendment](../plans/2026-08-13-002-search-performance-gate-amendment.md) + supersedes the original 1.5x proxy gate and makes this comparator diagnostic: + the nested safe path performs the required containment and final-generation + validation omitted by the full unsafe control, while R1, parity, safety, + regression, and scaling all pass. +- **U5 bytes-only diagnostic: EXCEPTION.** On the 10,000-file nested fixture, + safe-corpus p95 minus raw-read p95 is +401.66 ms (551.26 - 149.60), above the + original raw-read-plus-150 ms proxy. The same amendment supersedes that clause + because raw-read is a bytes-only lower bound that omits traversal, + containment, generation validation, decoding, and canonical search work; it + is informational and is not a release gate. +- **Operation scaling: PASS.** Accepted file paths remain exactly one `lstat`, + one `realpath`, and one `open` per file. Between 2,500 and 10,000 at fixed + topology, totals add exactly two `lstat`, one `realpath`, and one `open` per + additional file. Shallow request overhead stays 32 `lstat` / 10 `realpath`; + saturated nested request overhead stays 8,610 / 1,756. No observed-directory + scan occurs inside the per-file mapper. +- **Safety and request scope: PASS.** Differential and adversarial tests cover + final file mutation, directory/root generation, path and link safety, + deterministic fair discovery, reader closure on every exit, session + query/filter/limit replay, and whole-scope ceiling omissions. +- **Persistent index: DEFERRED.** The request-scoped safe scan meets R1 without + adding a cache, database, embedding, vector, graph, daemon, or derived search + authority. diff --git a/docs/benchmarks/2026-08-13-search-optimized.md b/docs/benchmarks/2026-08-13-search-optimized.md new file mode 100644 index 00000000..f919520a --- /dev/null +++ b/docs/benchmarks/2026-08-13-search-optimized.md @@ -0,0 +1,214 @@ +# Search benchmark optimized receipt — 2026-08-13 + +> Historical U6 implementation receipt. It is superseded by the +> [post-review final receipt](./2026-08-13-search-final.md), which includes the +> request-wide budget and final file-generation fixes plus checked-in raw JSON +> reports. Do not use the latency or operation values below as release evidence. + +This is the U6 measurement receipt for request-scoped safe bulk search. It uses +the frozen U8 authority unchanged and records both passing gates and observed +exceptions; an empty, reordered, unstable, errored, or inventory-mismatched +sample was rejected by the harness. + +## Authority and protocol + +- Manifest: `benchmarks/search/manifest.json` +- Manifest SHA-256: `b6c38cb5920f91b0a84c66be8181f6c14f7a1c73360fa4f07993b32a7704d55a` +- Fixtures and receipts: `/tmp/dotaios-search-baseline-20260813.yA3y7M` +- Full optimized JSON reports: `/tmp/dotaios-search-u6-final-*.report.json` +- Machine: `Mac16,1`, Apple M4, 10 cores, 16 GiB; macOS 26.6.1 build + 25G76 / Darwin 25.6.0; Node 22.22.3 arm64. +- Power: battery, Low Power Mode disabled; 44%, discharging when recorded after + the matrix. The harness ran on the frozen reference machine and profile. +- Each operation used 3 cold samples, 3 warm-up samples, and 20 measured warm + samples. A fresh request-scoped reader was used for each safe sample. +- Peak RSS is the largest absolute process RSS in the cold or warm samples. +- Every safe and unsafe canonical-search sample validated the exact ordered + result before its duration was accepted. The fixture inventory was hashed + before and after the matrix. The bytes-only and safe-corpus controls validated + exact file count and byte total before accepting a sample. + +The `raw-search` rows are the full unsafe U8 comparison: they run canonical +matching, snippets, corpus statistics, and ranking, omitting containment only. +The `raw-read` rows are the deliberately unlike bytes-only informational lower +bound. `safe-corpus` measures U5 enumeration, safe UTF-8 reads, callback +consumption, and final generation validation without canonical search work. + +## End-to-end safe search + +Times are milliseconds. + +| Fixture | Query | Cold median | Cold p95 | Warm median | Warm p95 | RSS MiB | lstat | realpath | open | +| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| 500 / shallow / prose | no-hit | 34.91 | 41.29 | 27.23 | 28.07 | 110.97 | 1,020 | 506 | 500 | +| 500 / shallow / prose | low-hit | 28.98 | 29.16 | 27.70 | 28.86 | 122.25 | 1,020 | 506 | 500 | +| 500 / shallow / prose | high-hit | 26.75 | 26.79 | 27.79 | 28.64 | 122.75 | 1,020 | 506 | 500 | +| 500 / nested / high-entropy | no-hit | 112.25 | 125.21 | 102.72 | 105.07 | 129.73 | 3,865 | 1,075 | 500 | +| 500 / nested / high-entropy | low-hit | 106.10 | 107.34 | 103.75 | 106.67 | 129.73 | 3,865 | 1,075 | 500 | +| 500 / nested / high-entropy | high-hit | 104.16 | 104.32 | 103.63 | 106.94 | 129.94 | 3,865 | 1,075 | 500 | +| 2,500 / shallow / prose | no-hit | 140.24 | 157.01 | 131.74 | 139.05 | 137.48 | 5,020 | 2,506 | 2,500 | +| 2,500 / shallow / prose | low-hit | 135.67 | 137.60 | 134.73 | 137.85 | 137.81 | 5,020 | 2,506 | 2,500 | +| 2,500 / shallow / prose | high-hit | 136.15 | 136.66 | 136.02 | 138.47 | 137.83 | 5,020 | 2,506 | 2,500 | +| 2,500 / nested / high-entropy | no-hit | 240.05 | 257.38 | 226.09 | 238.91 | 229.88 | 7,930 | 3,088 | 2,500 | +| 2,500 / nested / high-entropy | low-hit | 232.49 | 235.30 | 228.82 | 232.74 | 201.55 | 7,930 | 3,088 | 2,500 | +| 2,500 / nested / high-entropy | high-hit | 229.98 | 230.15 | 228.73 | 231.46 | 201.83 | 7,930 | 3,088 | 2,500 | +| 10,000 / shallow / prose | no-hit | 626.94 | 716.81 | 544.71 | 568.40 | 196.44 | 20,020 | 10,006 | 10,000 | +| 10,000 / shallow / prose | low-hit | 576.57 | 598.39 | 557.53 | 566.00 | 196.52 | 20,020 | 10,006 | 10,000 | +| 10,000 / shallow / prose | high-hit | 562.01 | 564.11 | 555.19 | 569.29 | 201.88 | 20,020 | 10,006 | 10,000 | +| 10,000 / nested / high-entropy | no-hit | 757.91 | 785.28 | 748.63 | 767.48 | 433.78 | 22,930 | 10,588 | 10,000 | +| 10,000 / nested / high-entropy | low-hit | 770.44 | 778.80 | 755.23 | 766.35 | 434.28 | 22,930 | 10,588 | 10,000 | +| 10,000 / nested / high-entropy | high-hit | 758.90 | 793.22 | 771.03 | 818.54 | 434.48 | 22,930 | 10,588 | 10,000 | + +## Full unsafe canonical-search control + +Each row performed exactly one `open` per file and zero `lstat`/`realpath`. + +| Fixture | Query | Cold median | Cold p95 | Warm median | Warm p95 | RSS MiB | +| --- | --- | ---: | ---: | ---: | ---: | ---: | +| 500 / shallow / prose | no-hit | 19.75 | 19.88 | 19.25 | 19.62 | 137.27 | +| 500 / shallow / prose | low-hit | 19.67 | 19.90 | 19.87 | 21.42 | 145.67 | +| 500 / shallow / prose | high-hit | 20.01 | 20.47 | 19.86 | 20.31 | 145.73 | +| 500 / nested / high-entropy | no-hit | 20.07 | 20.12 | 19.43 | 20.51 | 132.86 | +| 500 / nested / high-entropy | low-hit | 20.42 | 20.63 | 20.75 | 25.10 | 132.64 | +| 500 / nested / high-entropy | high-hit | 20.50 | 23.25 | 20.55 | 30.53 | 146.59 | +| 2,500 / shallow / prose | no-hit | 97.91 | 98.85 | 95.41 | 104.64 | 138.31 | +| 2,500 / shallow / prose | low-hit | 96.15 | 105.91 | 98.11 | 99.95 | 138.31 | +| 2,500 / shallow / prose | high-hit | 99.72 | 101.05 | 98.65 | 99.64 | 138.38 | +| 2,500 / nested / high-entropy | no-hit | 105.99 | 108.27 | 105.50 | 110.82 | 202.70 | +| 2,500 / nested / high-entropy | low-hit | 108.56 | 112.78 | 109.91 | 111.79 | 200.94 | +| 2,500 / nested / high-entropy | high-hit | 108.37 | 110.64 | 111.41 | 133.47 | 200.94 | +| 10,000 / shallow / prose | no-hit | 391.11 | 398.40 | 387.39 | 391.05 | 203.86 | +| 10,000 / shallow / prose | low-hit | 399.36 | 400.90 | 399.90 | 404.18 | 203.86 | +| 10,000 / shallow / prose | high-hit | 403.78 | 406.80 | 402.17 | 423.49 | 203.86 | +| 10,000 / nested / high-entropy | no-hit | 497.36 | 503.57 | 493.99 | 504.41 | 489.48 | +| 10,000 / nested / high-entropy | low-hit | 514.03 | 514.49 | 509.24 | 517.55 | 447.09 | +| 10,000 / nested / high-entropy | high-hit | 511.83 | 516.43 | 508.93 | 522.15 | 495.97 | + +## Safe-corpus and bytes-only controls + +Times are milliseconds. Safe-corpus rows include the same safe operation totals +as end-to-end search. Raw-read rows have zero `lstat`/`realpath` and one `open` +per file. + +| Fixture | Control | Cold median | Cold p95 | Warm median | Warm p95 | RSS MiB | lstat | realpath | open | +| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| 500 / shallow / prose | safe-corpus | 20.39 | 20.45 | 18.59 | 19.07 | 148.17 | 1,020 | 506 | 500 | +| 500 / shallow / prose | raw-read | 7.65 | 8.10 | 7.15 | 7.46 | 147.91 | 0 | 0 | 500 | +| 500 / nested / high-entropy | safe-corpus | 94.26 | 95.39 | 93.69 | 98.96 | 138.56 | 3,865 | 1,075 | 500 | +| 500 / nested / high-entropy | raw-read | 7.99 | 8.59 | 7.32 | 7.64 | 138.50 | 0 | 0 | 500 | +| 2,500 / shallow / prose | safe-corpus | 91.17 | 93.45 | 90.61 | 91.62 | 141.25 | 5,020 | 2,506 | 2,500 | +| 2,500 / shallow / prose | raw-read | 36.28 | 36.75 | 35.11 | 35.74 | 140.69 | 0 | 0 | 2,500 | +| 2,500 / nested / high-entropy | safe-corpus | 204.14 | 211.85 | 188.74 | 205.08 | 189.67 | 7,930 | 3,088 | 2,500 | +| 2,500 / nested / high-entropy | raw-read | 40.36 | 41.97 | 37.78 | 43.28 | 189.52 | 0 | 0 | 2,500 | +| 10,000 / shallow / prose | safe-corpus | 373.26 | 379.31 | 374.75 | 394.82 | 205.00 | 20,020 | 10,006 | 10,000 | +| 10,000 / shallow / prose | raw-read | 145.70 | 169.20 | 145.97 | 151.01 | 204.50 | 0 | 0 | 10,000 | +| 10,000 / nested / high-entropy | safe-corpus | 471.01 | 478.09 | 473.22 | 500.71 | 419.91 | 22,930 | 10,588 | 10,000 | +| 10,000 / nested / high-entropy | raw-read | 150.03 | 163.63 | 149.04 | 151.78 | 356.63 | 0 | 0 | 10,000 | + +## Exact results and operation scaling + +Every cold, warm-up, measured, safe, and raw-search sample returned the same +ordered sources and result hash. At 10,000 files: + +- `no-hit`: exactly `[]`. +- Shallow `low-hit`: `vault/bucket-01/note-00003.md`, + `vault/bucket-01/note-00017.md`, `vault/bucket-01/note-00101.md`, + `vault/bucket-01/note-00307.md`. +- Nested `low-hit`: `vault/branch-00/branch-00/branch-03/note-00003.md`, + `vault/branch-00/branch-02/branch-01/note-00017.md`, + `vault/branch-01/branch-04/branch-05/note-00101.md`, + `vault/branch-04/branch-06/branch-03/note-00307.md`. +- `high-hit`: exactly the first 20 generated paths in lexical order whose file + index is divisible by 25, as embedded in the immutable fixture receipt. The + 10k result hashes are `9b3d7e484ca325f107d3363721ce8b99b038149f4b79a4f23c25e52c393bfe3f` + (shallow) and `0bcab7da3781ba39e4bdcbc5e04bb5f2bc293ad8dbc75d19f43ee4fca5c0da58` + (nested). Low-hit hashes are `6bc96a7e535089d0f1232a6aa21cf6cf2b255f9d1ddccfe2dea5b098ff0c98b5` + and `3426468c469ccf3a134e93a71f6dea6f3a21d3e622918668af97a098bee9b3fd`; + no-hit is `4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945`. + +The harness attributes accepted-file paths separately: every cell has exactly +one `lstat`, one `realpath`, and one `open` on each accepted file path. Other +containment-path counts remain separately visible. Between 2,500 and 10,000, +both fixed topologies have exact total-operation slopes of 2 `lstat`, 1 +`realpath`, and 1 `open` per added file; the nested fixed overhead is unchanged +at 2,930 `lstat` and 588 `realpath`, and shallow fixed overhead is unchanged at +20/6. Therefore accepted-file-path per-file counts differ by 0%, and a fixed-topology +doubling is bounded by 2.0x (below 2.1x). Enumeration occurs once before file +mapping and final observed-directory validation occurs once after ranking; no +directory scan is inside the per-file mapper. If the strict per-file comparison +is instead calculated by dividing request-level aggregate totals by file count, +shallow stays within 1%, while +nested `lstat` falls from 3.172 to 2.293 per file (27.7%) and nested `realpath` +from 1.2352 to 1.0588 (14.3%) as the same fixed 512-leaf topology overhead is +amortized. Those derived aggregate ratios exceed 10%, but they are not per-file +operations: the harness records fixed containment-path work separately from the +accepted-file-path work that the invariant is intended to constrain. + +## Real AIOS folder check + +The product CLI searched the configured local vault (`~/aios`) for +`indexing`, limited to five results. The returned output was inspected and +contained five substantive research records about source handling, retrieval, +competitive systems, and web discoverability. The pre-change and optimized +commands produced the same visible output SHA-256 +`8eecbb5f4b4c1faa8f8a2e679d7f8be6a932f7807743bc4b22e7d018b7f25b06`. + +After two warm-up runs, five measured fresh-process samples produced: + +| Path | Median | p95 | Stable output | +| --- | ---: | ---: | --- | +| Pre-change `9815261` | 476.99 ms | 553.04 ms | yes | +| Optimized working tree | 254.07 ms | 259.19 ms | yes | + +This is informational evidence, not a replacement for the frozen fixture. An +unscoped product search also printed `Project corpus omitted because no +--project selector was supplied.`, confirming that projects are not silently +included without a selector. + +## Gate verdicts + +- **R1 / AE1: PASS.** All six 10,000-file warmed p95 values are below 1,000 + ms. Worst observed is 818.54 ms (nested/high-entropy/high-hit), with 20 + measured samples and exact 20-hit order. +- **Parity / safety / request scope: PASS.** The differential suite covers + substring, inflection, punctuation and line-spanning terms, frontmatter + descriptions, title/path boosts, recency/ties, memory streams/daily/inbox, + plugin manifest predicate, project selector, external vault, omitted secret, + plugin, and project sources. A changed final generation rejects ranked + callback output; added, modified, and deleted files appear next request. +- **500 regression: PASS.** Worst safe warm p95 is 28.86 ms shallow and + 106.94 ms nested, versus baselines of 107.91–111.40 ms and 942.54–958.66 ms; + every query is below baseline plus the larger of 20% or 50 ms. +- **Operation scaling: PASS.** Accepted-file-path counts are exactly 1/1/1 and + their 2,500-to-10,000 per-file difference is 0%; total-operation slopes are + 2/1/1 per added file, fixed-topology doubling is no worse than 2.0x, and + there is no file-by-directory multiplication. Request-level directory work + is reported separately; dividing that fixed overhead by changing file counts + produces the transparent 27.7%/14.3% nested aggregate ratios above but does + not change the per-file operation invariant. +- **10k safe/full-unsafe target: PASS for shallow, EXCEPTION for nested.** + Shallow ratios are 1.45x, 1.40x, and 1.34x. Nested ratios are 1.52x, 1.48x, + and 1.57x. The no-hit and high-hit nested samples exceed the 1.5x target by + 0.02x and 0.07x while still passing R1. +- **Bytes-only raw-read: informational only.** It deliberately omits matching, + snippets, corpus statistics, ranking, and all containment and is not the U6 + relative authority. +- **U5 raw-read +150 ms preliminary gate: EXCEPTION in this matrix.** Safe + corpus-read minus raw-read warm p95 is +11.61 ms (500 shallow), +91.32 ms + (500 nested), +55.88 ms (2,500 shallow), +161.81 ms (2,500 nested), +243.81 + ms (10,000 shallow), and +348.92 ms (10,000 nested). This supplemental U6 + measurement does not satisfy that preliminary gate at 2,500 nested or either + 10,000 cell. The implementation and frozen protocol were not weakened or + moved to conceal the miss. + +The U5 preliminary comparison is a conservative proxy whose bytes-only control +omits the canonical matching/ranking work that U6 must perform. Execution +proceeded because the actual end-to-end U6 authority is stronger and passed: +all six safe 10k searches are below R1's one-second limit, the worst is below +the plan's 900 ms combined target, exact parity and safety pass, and the real +folder result improved. Removing per-file canonical or identity validation to +force the unlike bytes-only proxy green would violate R3; no such weakening was +accepted. Persistent indexing therefore remains deferred under KTD2. + +No dependency, build step, persistent index/cache, vector, embedding, graph, +daemon, or database was introduced. diff --git a/docs/benchmarks/2026-08-14-search-public-entrypoint.md b/docs/benchmarks/2026-08-14-search-public-entrypoint.md new file mode 100644 index 00000000..648afca6 --- /dev/null +++ b/docs/benchmarks/2026-08-14-search-public-entrypoint.md @@ -0,0 +1,63 @@ +# Public `searchAios` release receipt — 2026-08-14 + +This receipt closes the benchmark blind spot discovered before PR #80 merged. +The earlier harness measured the internal `searchMarkdownDir` primitive, while +the CLI and MCP call the request-wide `searchAios` entry point. The primitive +optimization was real, but repeated preflight and final containment validation +made the public path take roughly 10–12 seconds on 10,000 files. + +The v2 harness now calls the default all-scope `searchAios` operation. Every +sample must be complete, contain no omissions, return the exact controlled +results, and stay within a fixed operation allowance over a safe request-scoped +corpus-read control. Vacuous low-hit or high-hit samples are rejected. + +## Authority and protocol + +- Code authority: `5160f23` (public-path optimization: `c6bf99c`). +- Manifest: `benchmarks/search/manifest.json`. +- Manifest SHA-256: + `b6c38cb5920f91b0a84c66be8181f6c14f7a1c73360fa4f07993b32a7704d55a`. +- Full v2 reports: [`reports/`](./reports/), named + `2026-08-14-public---.report.json`. +- Machine: Apple M4, 10 cores, 16 GiB; macOS/Darwin arm64; Node 22.22.3. +- Each query used 3 cold samples, 3 warm-up samples, and 20 measured warm + samples with a fresh request-scoped reader for every safe sample. +- Search surface in every report: `entryPoint=searchAios`, + `requestedScope=all`, `completeness=complete`. + +## End-to-end public search + +Times are warm p95 milliseconds. Every cell returned 0, 4, and 20 exact hits +for the no-hit, low-hit, and high-hit queries respectively. + +| Fixture | No hit | Low hit | High hit | lstat | realpath | open | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| 500 / shallow / prose | 39.96 | 39.51 | 38.88 | 1,808 | 61 | 500 | +| 500 / nested / high-entropy | 192.88 | 195.38 | 192.26 | 10,762 | 1,768 | 500 | +| 2,500 / shallow / prose | 173.25 | 176.34 | 177.22 | 7,808 | 61 | 2,500 | +| 2,500 / nested / high-entropy | 347.99 | 357.03 | 348.90 | 16,968 | 1,807 | 2,500 | +| 10,000 / shallow / prose | 675.50 | 689.51 | 689.64 | 30,308 | 61 | 10,000 | +| 10,000 / nested / high-entropy | 937.58 | 954.16 | 954.99 | 39,468 | 1,807 | 10,000 | + +## Gate verdicts + +- **Public entry point: PASS.** Every production sample executed default + all-scope `searchAios`; none fell back to the directory primitive. +- **R1 latency: PASS.** All six 10,000-file warm p95 values are below 1,000 ms. + Worst is 954.99 ms on the adversarial nested/high-entropy/high-hit cell. +- **Completeness and exact parity: PASS.** All 18 measured query cells were + complete, reported zero omissions, and matched the immutable fixture receipt. +- **Operation gate: PASS.** Every report stayed within the safe preflight + corpus-read control plus the fixed allowance of 512 `lstat`, 256 `realpath`, + and 16 `open` calls. The 10,000-file public path is no longer allowed to + repeat a full containment walk for every file and phase. +- **Safety: PASS.** Focused containment, search, ranking, scaling, and benchmark + suites passed 162/162 before integration; the integrated full repository + suite passed 1,949 tests with zero failures (9 platform/fixture skips). +- **Persistent index: DEFERRED.** The real public search meets the release goal + without introducing a database, vector store, daemon, or second memory + authority. + +The [2026-08-13 final receipt](./2026-08-13-search-final.md) remains useful +historical evidence for the internal safe-search primitive. This receipt is the +release authority for the public search surface. diff --git a/docs/benchmarks/reports/2026-08-13-10000-nested-high-entropy.report.json b/docs/benchmarks/reports/2026-08-13-10000-nested-high-entropy.report.json new file mode 100644 index 00000000..3b7942e0 --- /dev/null +++ b/docs/benchmarks/reports/2026-08-13-10000-nested-high-entropy.report.json @@ -0,0 +1,489 @@ +{ + "schemaVersion": "dotaios-search-benchmark-result/v1", + "benchmarkId": "search-baseline-2026-08-13", + "manifestSha256": "b6c38cb5920f91b0a84c66be8181f6c14f7a1c73360fa4f07993b32a7704d55a", + "inventorySha256": "cc818211f579b4c54fcacfaa42ed195be052d156c5161c1ccdf2f23cdd1bf9a8", + "selection": { + "fileCount": 10000, + "layout": "nested", + "distribution": "high-entropy" + }, + "runtime": { + "node": "22.22.3", + "platform": "darwin", + "architecture": "arm64" + }, + "protocol": { + "coldSamples": 3, + "warmupSamples": 3, + "measuredSamples": 20, + "concurrency": 32, + "resultLimit": 20, + "rssPollIntervalMs": 5, + "coldDefinition": "fresh request-scoped reader before harness warm-up; operating-system file cache is uncontrolled", + "warmDefinition": "fresh request-scoped reader after warm-up in the same Node process", + "rawReadControl": { + "enabled": true, + "method": "open-handle-read-all", + "validation": "exact file count and byte total", + "concurrency": 32 + } + }, + "searches": [ + { + "id": "no-hit", + "cold": { + "samples": 3, + "medianMs": 816.743, + "p95Ms": 846.102167, + "peakRssBytes": 313638912, + "operations": { + "lstat": 28610, + "realpath": 11756, + "open": 10000 + }, + "readBudget": { + "bytes": 10319743, + "files": 10000, + "entries": 10584 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 10000, + "realpath": 10000, + "open": 10000 + }, + "containmentPaths": { + "lstat": 18610, + "realpath": 1756, + "open": 0 + } + } + }, + "warm": { + "samples": 20, + "medianMs": 805.272042, + "p95Ms": 820.6085, + "peakRssBytes": 470417408, + "operations": { + "lstat": 28610, + "realpath": 11756, + "open": 10000 + }, + "readBudget": { + "bytes": 10319743, + "files": 10000, + "entries": 10584 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 10000, + "realpath": 10000, + "open": 10000 + }, + "containmentPaths": { + "lstat": 18610, + "realpath": 1756, + "open": 0 + } + } + }, + "exactResults": [], + "outputSha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + { + "id": "low-hit", + "cold": { + "samples": 3, + "medianMs": 832.963459, + "p95Ms": 901.3715, + "peakRssBytes": 468434944, + "operations": { + "lstat": 28610, + "realpath": 11756, + "open": 10000 + }, + "readBudget": { + "bytes": 10319743, + "files": 10000, + "entries": 10584 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 10000, + "realpath": 10000, + "open": 10000 + }, + "containmentPaths": { + "lstat": 18610, + "realpath": 1756, + "open": 0 + } + } + }, + "warm": { + "samples": 20, + "medianMs": 813.539541, + "p95Ms": 844.553416, + "peakRssBytes": 382959616, + "operations": { + "lstat": 28610, + "realpath": 11756, + "open": 10000 + }, + "readBudget": { + "bytes": 10319743, + "files": 10000, + "entries": 10584 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 10000, + "realpath": 10000, + "open": 10000 + }, + "containmentPaths": { + "lstat": 18610, + "realpath": 1756, + "open": 0 + } + } + }, + "exactResults": [ + "vault/branch-00/branch-00/branch-03/note-00003.md", + "vault/branch-00/branch-02/branch-01/note-00017.md", + "vault/branch-01/branch-04/branch-05/note-00101.md", + "vault/branch-04/branch-06/branch-03/note-00307.md" + ], + "outputSha256": "3426468c469ccf3a134e93a71f6dea6f3a21d3e622918668af97a098bee9b3fd" + }, + { + "id": "high-hit", + "cold": { + "samples": 3, + "medianMs": 817.327375, + "p95Ms": 893.53475, + "peakRssBytes": 382959616, + "operations": { + "lstat": 28610, + "realpath": 11756, + "open": 10000 + }, + "readBudget": { + "bytes": 10319743, + "files": 10000, + "entries": 10584 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 10000, + "realpath": 10000, + "open": 10000 + }, + "containmentPaths": { + "lstat": 18610, + "realpath": 1756, + "open": 0 + } + } + }, + "warm": { + "samples": 20, + "medianMs": 817.515958, + "p95Ms": 835.113416, + "peakRssBytes": 471924736, + "operations": { + "lstat": 28610, + "realpath": 11756, + "open": 10000 + }, + "readBudget": { + "bytes": 10319743, + "files": 10000, + "entries": 10584 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 10000, + "realpath": 10000, + "open": 10000 + }, + "containmentPaths": { + "lstat": 18610, + "realpath": 1756, + "open": 0 + } + } + }, + "exactResults": [ + "vault/branch-00/branch-00/branch-00/note-00000.md", + "vault/branch-00/branch-00/branch-01/note-01025.md", + "vault/branch-00/branch-00/branch-02/note-02050.md", + "vault/branch-00/branch-00/branch-03/note-03075.md", + "vault/branch-00/branch-00/branch-04/note-04100.md", + "vault/branch-00/branch-00/branch-05/note-05125.md", + "vault/branch-00/branch-00/branch-06/note-06150.md", + "vault/branch-00/branch-00/branch-07/note-07175.md", + "vault/branch-00/branch-01/branch-00/note-08200.md", + "vault/branch-00/branch-01/branch-01/note-09225.md", + "vault/branch-00/branch-01/branch-05/note-00525.md", + "vault/branch-00/branch-01/branch-06/note-01550.md", + "vault/branch-00/branch-01/branch-07/note-02575.md", + "vault/branch-00/branch-02/branch-00/note-03600.md", + "vault/branch-00/branch-02/branch-01/note-04625.md", + "vault/branch-00/branch-02/branch-02/note-05650.md", + "vault/branch-00/branch-02/branch-03/note-06675.md", + "vault/branch-00/branch-02/branch-04/note-07700.md", + "vault/branch-00/branch-02/branch-05/note-08725.md", + "vault/branch-00/branch-02/branch-06/note-09750.md" + ], + "outputSha256": "0bcab7da3781ba39e4bdcbc5e04bb5f2bc293ad8dbc75d19f43ee4fca5c0da58" + } + ], + "rawSearchControl": [ + { + "safety": "unsafe-benchmark-only", + "id": "no-hit", + "cold": { + "samples": 3, + "medianMs": 484.797042, + "p95Ms": 490.183708, + "peakRssBytes": 428883968, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 10000 + }, + "readBudget": { + "files": 10000, + "bytes": 10319743, + "entries": 0 + } + }, + "warm": { + "samples": 20, + "medianMs": 483.581708, + "p95Ms": 493.236625, + "peakRssBytes": 529793024, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 10000 + }, + "readBudget": { + "files": 10000, + "bytes": 10319743, + "entries": 0 + } + }, + "exactResults": [], + "outputSha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + { + "safety": "unsafe-benchmark-only", + "id": "low-hit", + "cold": { + "samples": 3, + "medianMs": 508.63975, + "p95Ms": 509.690875, + "peakRssBytes": 429621248, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 10000 + }, + "readBudget": { + "files": 10000, + "bytes": 10319743, + "entries": 0 + } + }, + "warm": { + "samples": 20, + "medianMs": 494.160917, + "p95Ms": 506.841125, + "peakRssBytes": 486359040, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 10000 + }, + "readBudget": { + "files": 10000, + "bytes": 10319743, + "entries": 0 + } + }, + "exactResults": [ + "vault/branch-00/branch-00/branch-03/note-00003.md", + "vault/branch-00/branch-02/branch-01/note-00017.md", + "vault/branch-01/branch-04/branch-05/note-00101.md", + "vault/branch-04/branch-06/branch-03/note-00307.md" + ], + "outputSha256": "3426468c469ccf3a134e93a71f6dea6f3a21d3e622918668af97a098bee9b3fd" + }, + { + "safety": "unsafe-benchmark-only", + "id": "high-hit", + "cold": { + "samples": 3, + "medianMs": 501.667417, + "p95Ms": 501.803583, + "peakRssBytes": 429637632, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 10000 + }, + "readBudget": { + "files": 10000, + "bytes": 10319743, + "entries": 0 + } + }, + "warm": { + "samples": 20, + "medianMs": 500.330625, + "p95Ms": 523.916833, + "peakRssBytes": 471908352, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 10000 + }, + "readBudget": { + "files": 10000, + "bytes": 10319743, + "entries": 0 + } + }, + "exactResults": [ + "vault/branch-00/branch-00/branch-00/note-00000.md", + "vault/branch-00/branch-00/branch-01/note-01025.md", + "vault/branch-00/branch-00/branch-02/note-02050.md", + "vault/branch-00/branch-00/branch-03/note-03075.md", + "vault/branch-00/branch-00/branch-04/note-04100.md", + "vault/branch-00/branch-00/branch-05/note-05125.md", + "vault/branch-00/branch-00/branch-06/note-06150.md", + "vault/branch-00/branch-00/branch-07/note-07175.md", + "vault/branch-00/branch-01/branch-00/note-08200.md", + "vault/branch-00/branch-01/branch-01/note-09225.md", + "vault/branch-00/branch-01/branch-05/note-00525.md", + "vault/branch-00/branch-01/branch-06/note-01550.md", + "vault/branch-00/branch-01/branch-07/note-02575.md", + "vault/branch-00/branch-02/branch-00/note-03600.md", + "vault/branch-00/branch-02/branch-01/note-04625.md", + "vault/branch-00/branch-02/branch-02/note-05650.md", + "vault/branch-00/branch-02/branch-03/note-06675.md", + "vault/branch-00/branch-02/branch-04/note-07700.md", + "vault/branch-00/branch-02/branch-05/note-08725.md", + "vault/branch-00/branch-02/branch-06/note-09750.md" + ], + "outputSha256": "0bcab7da3781ba39e4bdcbc5e04bb5f2bc293ad8dbc75d19f43ee4fca5c0da58" + } + ], + "rawReadControl": { + "id": "raw-read-control", + "cold": { + "samples": 3, + "medianMs": 147.041958, + "p95Ms": 156.745708, + "peakRssBytes": 385351680, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 10000 + }, + "readBudget": { + "files": 10000, + "bytes": 10319743, + "entries": 0 + } + }, + "warm": { + "samples": 20, + "medianMs": 147.718792, + "p95Ms": 149.600333, + "peakRssBytes": 357662720, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 10000 + }, + "readBudget": { + "files": 10000, + "bytes": 10319743, + "entries": 0 + } + }, + "exactResults": { + "fileCount": 10000, + "totalBytes": 10319743 + }, + "outputSha256": "f0d599e033e431e53d934958618fb5f6d86b584ec3dd361252e0677895078d4c" + }, + "safeCorpusReadControl": { + "id": "safe-corpus-read-control", + "cold": { + "samples": 3, + "medianMs": 546.651833, + "p95Ms": 577.19725, + "peakRssBytes": 358924288, + "operations": { + "lstat": 28610, + "realpath": 11756, + "open": 10000 + }, + "readBudget": { + "bytes": 10319743, + "files": 10000, + "entries": 10584 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 10000, + "realpath": 10000, + "open": 10000 + }, + "containmentPaths": { + "lstat": 18610, + "realpath": 1756, + "open": 0 + } + } + }, + "warm": { + "samples": 20, + "medianMs": 542.086209, + "p95Ms": 551.255125, + "peakRssBytes": 441909248, + "operations": { + "lstat": 28610, + "realpath": 11756, + "open": 10000 + }, + "readBudget": { + "bytes": 10319743, + "files": 10000, + "entries": 10584 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 10000, + "realpath": 10000, + "open": 10000 + }, + "containmentPaths": { + "lstat": 18610, + "realpath": 1756, + "open": 0 + } + } + }, + "exactResults": { + "fileCount": 10000, + "totalBytes": 10319743 + }, + "outputSha256": "f0d599e033e431e53d934958618fb5f6d86b584ec3dd361252e0677895078d4c" + } +} diff --git a/docs/benchmarks/reports/2026-08-13-10000-shallow-prose.report.json b/docs/benchmarks/reports/2026-08-13-10000-shallow-prose.report.json new file mode 100644 index 00000000..87cece50 --- /dev/null +++ b/docs/benchmarks/reports/2026-08-13-10000-shallow-prose.report.json @@ -0,0 +1,489 @@ +{ + "schemaVersion": "dotaios-search-benchmark-result/v1", + "benchmarkId": "search-baseline-2026-08-13", + "manifestSha256": "b6c38cb5920f91b0a84c66be8181f6c14f7a1c73360fa4f07993b32a7704d55a", + "inventorySha256": "50a3e8026807256715a8f7ea5ffcb8d35c8a074d04c08badbf4e27c6119008c6", + "selection": { + "fileCount": 10000, + "layout": "shallow", + "distribution": "prose" + }, + "runtime": { + "node": "22.22.3", + "platform": "darwin", + "architecture": "arm64" + }, + "protocol": { + "coldSamples": 3, + "warmupSamples": 3, + "measuredSamples": 20, + "concurrency": 32, + "resultLimit": 20, + "rssPollIntervalMs": 5, + "coldDefinition": "fresh request-scoped reader before harness warm-up; operating-system file cache is uncontrolled", + "warmDefinition": "fresh request-scoped reader after warm-up in the same Node process", + "rawReadControl": { + "enabled": true, + "method": "open-handle-read-all", + "validation": "exact file count and byte total", + "concurrency": 32 + } + }, + "searches": [ + { + "id": "no-hit", + "cold": { + "samples": 3, + "medianMs": 536.216792, + "p95Ms": 560.905208, + "peakRssBytes": 185843712, + "operations": { + "lstat": 20032, + "realpath": 10010, + "open": 10000 + }, + "readBudget": { + "bytes": 10886041, + "files": 10000, + "entries": 10002 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 10000, + "realpath": 10000, + "open": 10000 + }, + "containmentPaths": { + "lstat": 10032, + "realpath": 10, + "open": 0 + } + } + }, + "warm": { + "samples": 20, + "medianMs": 533.649417, + "p95Ms": 551.289792, + "peakRssBytes": 205766656, + "operations": { + "lstat": 20032, + "realpath": 10010, + "open": 10000 + }, + "readBudget": { + "bytes": 10886041, + "files": 10000, + "entries": 10002 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 10000, + "realpath": 10000, + "open": 10000 + }, + "containmentPaths": { + "lstat": 10032, + "realpath": 10, + "open": 0 + } + } + }, + "exactResults": [], + "outputSha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + { + "id": "low-hit", + "cold": { + "samples": 3, + "medianMs": 544.007333, + "p95Ms": 545.647166, + "peakRssBytes": 204554240, + "operations": { + "lstat": 20032, + "realpath": 10010, + "open": 10000 + }, + "readBudget": { + "bytes": 10886041, + "files": 10000, + "entries": 10002 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 10000, + "realpath": 10000, + "open": 10000 + }, + "containmentPaths": { + "lstat": 10032, + "realpath": 10, + "open": 0 + } + } + }, + "warm": { + "samples": 20, + "medianMs": 544.503667, + "p95Ms": 553.324125, + "peakRssBytes": 212533248, + "operations": { + "lstat": 20032, + "realpath": 10010, + "open": 10000 + }, + "readBudget": { + "bytes": 10886041, + "files": 10000, + "entries": 10002 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 10000, + "realpath": 10000, + "open": 10000 + }, + "containmentPaths": { + "lstat": 10032, + "realpath": 10, + "open": 0 + } + } + }, + "exactResults": [ + "vault/bucket-01/note-00003.md", + "vault/bucket-01/note-00017.md", + "vault/bucket-01/note-00101.md", + "vault/bucket-01/note-00307.md" + ], + "outputSha256": "6bc96a7e535089d0f1232a6aa21cf6cf2b255f9d1ddccfe2dea5b098ff0c98b5" + }, + { + "id": "high-hit", + "cold": { + "samples": 3, + "medianMs": 544.931625, + "p95Ms": 612.680708, + "peakRssBytes": 213368832, + "operations": { + "lstat": 20032, + "realpath": 10010, + "open": 10000 + }, + "readBudget": { + "bytes": 10886041, + "files": 10000, + "entries": 10002 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 10000, + "realpath": 10000, + "open": 10000 + }, + "containmentPaths": { + "lstat": 10032, + "realpath": 10, + "open": 0 + } + } + }, + "warm": { + "samples": 20, + "medianMs": 546.438958, + "p95Ms": 556.884417, + "peakRssBytes": 213467136, + "operations": { + "lstat": 20032, + "realpath": 10010, + "open": 10000 + }, + "readBudget": { + "bytes": 10886041, + "files": 10000, + "entries": 10002 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 10000, + "realpath": 10000, + "open": 10000 + }, + "containmentPaths": { + "lstat": 10032, + "realpath": 10, + "open": 0 + } + } + }, + "exactResults": [ + "vault/bucket-00/note-00000.md", + "vault/bucket-00/note-00050.md", + "vault/bucket-00/note-00100.md", + "vault/bucket-00/note-00150.md", + "vault/bucket-00/note-00200.md", + "vault/bucket-00/note-00250.md", + "vault/bucket-00/note-00300.md", + "vault/bucket-00/note-00350.md", + "vault/bucket-00/note-00400.md", + "vault/bucket-00/note-00450.md", + "vault/bucket-00/note-00500.md", + "vault/bucket-00/note-00550.md", + "vault/bucket-00/note-00600.md", + "vault/bucket-00/note-00650.md", + "vault/bucket-00/note-00700.md", + "vault/bucket-00/note-00750.md", + "vault/bucket-00/note-00800.md", + "vault/bucket-00/note-00850.md", + "vault/bucket-00/note-00900.md", + "vault/bucket-00/note-00950.md" + ], + "outputSha256": "9b3d7e484ca325f107d3363721ce8b99b038149f4b79a4f23c25e52c393bfe3f" + } + ], + "rawSearchControl": [ + { + "safety": "unsafe-benchmark-only", + "id": "no-hit", + "cold": { + "samples": 3, + "medianMs": 392.462, + "p95Ms": 394.908625, + "peakRssBytes": 214073344, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 10000 + }, + "readBudget": { + "files": 10000, + "bytes": 10886041, + "entries": 0 + } + }, + "warm": { + "samples": 20, + "medianMs": 385.338292, + "p95Ms": 391.062, + "peakRssBytes": 215449600, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 10000 + }, + "readBudget": { + "files": 10000, + "bytes": 10886041, + "entries": 0 + } + }, + "exactResults": [], + "outputSha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + { + "safety": "unsafe-benchmark-only", + "id": "low-hit", + "cold": { + "samples": 3, + "medianMs": 400.529208, + "p95Ms": 413.090583, + "peakRssBytes": 215449600, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 10000 + }, + "readBudget": { + "files": 10000, + "bytes": 10886041, + "entries": 0 + } + }, + "warm": { + "samples": 20, + "medianMs": 398.853791, + "p95Ms": 404.834125, + "peakRssBytes": 215449600, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 10000 + }, + "readBudget": { + "files": 10000, + "bytes": 10886041, + "entries": 0 + } + }, + "exactResults": [ + "vault/bucket-01/note-00003.md", + "vault/bucket-01/note-00017.md", + "vault/bucket-01/note-00101.md", + "vault/bucket-01/note-00307.md" + ], + "outputSha256": "6bc96a7e535089d0f1232a6aa21cf6cf2b255f9d1ddccfe2dea5b098ff0c98b5" + }, + { + "safety": "unsafe-benchmark-only", + "id": "high-hit", + "cold": { + "samples": 3, + "medianMs": 399.308167, + "p95Ms": 402.528667, + "peakRssBytes": 214253568, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 10000 + }, + "readBudget": { + "files": 10000, + "bytes": 10886041, + "entries": 0 + } + }, + "warm": { + "samples": 20, + "medianMs": 400.629833, + "p95Ms": 416.796208, + "peakRssBytes": 215449600, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 10000 + }, + "readBudget": { + "files": 10000, + "bytes": 10886041, + "entries": 0 + } + }, + "exactResults": [ + "vault/bucket-00/note-00000.md", + "vault/bucket-00/note-00050.md", + "vault/bucket-00/note-00100.md", + "vault/bucket-00/note-00150.md", + "vault/bucket-00/note-00200.md", + "vault/bucket-00/note-00250.md", + "vault/bucket-00/note-00300.md", + "vault/bucket-00/note-00350.md", + "vault/bucket-00/note-00400.md", + "vault/bucket-00/note-00450.md", + "vault/bucket-00/note-00500.md", + "vault/bucket-00/note-00550.md", + "vault/bucket-00/note-00600.md", + "vault/bucket-00/note-00650.md", + "vault/bucket-00/note-00700.md", + "vault/bucket-00/note-00750.md", + "vault/bucket-00/note-00800.md", + "vault/bucket-00/note-00850.md", + "vault/bucket-00/note-00900.md", + "vault/bucket-00/note-00950.md" + ], + "outputSha256": "9b3d7e484ca325f107d3363721ce8b99b038149f4b79a4f23c25e52c393bfe3f" + } + ], + "rawReadControl": { + "id": "raw-read-control", + "cold": { + "samples": 3, + "medianMs": 142.552, + "p95Ms": 143.99475, + "peakRssBytes": 212041728, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 10000 + }, + "readBudget": { + "files": 10000, + "bytes": 10886041, + "entries": 0 + } + }, + "warm": { + "samples": 20, + "medianMs": 141.703458, + "p95Ms": 145.447792, + "peakRssBytes": 212090880, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 10000 + }, + "readBudget": { + "files": 10000, + "bytes": 10886041, + "entries": 0 + } + }, + "exactResults": { + "fileCount": 10000, + "totalBytes": 10886041 + }, + "outputSha256": "a50af3418a32f28d3a9f37ff53b576c7244e6c68d0dbd13c2c14f31d0580cac2" + }, + "safeCorpusReadControl": { + "id": "safe-corpus-read-control", + "cold": { + "samples": 3, + "medianMs": 366.766792, + "p95Ms": 368.082042, + "peakRssBytes": 213368832, + "operations": { + "lstat": 20032, + "realpath": 10010, + "open": 10000 + }, + "readBudget": { + "bytes": 10886041, + "files": 10000, + "entries": 10002 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 10000, + "realpath": 10000, + "open": 10000 + }, + "containmentPaths": { + "lstat": 10032, + "realpath": 10, + "open": 0 + } + } + }, + "warm": { + "samples": 20, + "medianMs": 365.223916, + "p95Ms": 406.203833, + "peakRssBytes": 213385216, + "operations": { + "lstat": 20032, + "realpath": 10010, + "open": 10000 + }, + "readBudget": { + "bytes": 10886041, + "files": 10000, + "entries": 10002 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 10000, + "realpath": 10000, + "open": 10000 + }, + "containmentPaths": { + "lstat": 10032, + "realpath": 10, + "open": 0 + } + } + }, + "exactResults": { + "fileCount": 10000, + "totalBytes": 10886041 + }, + "outputSha256": "a50af3418a32f28d3a9f37ff53b576c7244e6c68d0dbd13c2c14f31d0580cac2" + } +} diff --git a/docs/benchmarks/reports/2026-08-13-2500-nested-high-entropy.report.json b/docs/benchmarks/reports/2026-08-13-2500-nested-high-entropy.report.json new file mode 100644 index 00000000..1b8d53f8 --- /dev/null +++ b/docs/benchmarks/reports/2026-08-13-2500-nested-high-entropy.report.json @@ -0,0 +1,489 @@ +{ + "schemaVersion": "dotaios-search-benchmark-result/v1", + "benchmarkId": "search-baseline-2026-08-13", + "manifestSha256": "b6c38cb5920f91b0a84c66be8181f6c14f7a1c73360fa4f07993b32a7704d55a", + "inventorySha256": "42178a1a3d649a5223662730f7373ce1505289feefbb88dd2afad3918194e537", + "selection": { + "fileCount": 2500, + "layout": "nested", + "distribution": "high-entropy" + }, + "runtime": { + "node": "22.22.3", + "platform": "darwin", + "architecture": "arm64" + }, + "protocol": { + "coldSamples": 3, + "warmupSamples": 3, + "measuredSamples": 20, + "concurrency": 32, + "resultLimit": 20, + "rssPollIntervalMs": 5, + "coldDefinition": "fresh request-scoped reader before harness warm-up; operating-system file cache is uncontrolled", + "warmDefinition": "fresh request-scoped reader after warm-up in the same Node process", + "rawReadControl": { + "enabled": true, + "method": "open-handle-read-all", + "validation": "exact file count and byte total", + "concurrency": 32 + } + }, + "searches": [ + { + "id": "no-hit", + "cold": { + "samples": 3, + "medianMs": 328.409083, + "p95Ms": 338.787416, + "peakRssBytes": 170377216, + "operations": { + "lstat": 13610, + "realpath": 4256, + "open": 2500 + }, + "readBudget": { + "bytes": 2582784, + "files": 2500, + "entries": 3084 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 2500, + "realpath": 2500, + "open": 2500 + }, + "containmentPaths": { + "lstat": 11110, + "realpath": 1756, + "open": 0 + } + } + }, + "warm": { + "samples": 20, + "medianMs": 303.404958, + "p95Ms": 310.747084, + "peakRssBytes": 264552448, + "operations": { + "lstat": 13610, + "realpath": 4256, + "open": 2500 + }, + "readBudget": { + "bytes": 2582784, + "files": 2500, + "entries": 3084 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 2500, + "realpath": 2500, + "open": 2500 + }, + "containmentPaths": { + "lstat": 11110, + "realpath": 1756, + "open": 0 + } + } + }, + "exactResults": [], + "outputSha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + { + "id": "low-hit", + "cold": { + "samples": 3, + "medianMs": 306.026708, + "p95Ms": 310.965625, + "peakRssBytes": 226394112, + "operations": { + "lstat": 13610, + "realpath": 4256, + "open": 2500 + }, + "readBudget": { + "bytes": 2582784, + "files": 2500, + "entries": 3084 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 2500, + "realpath": 2500, + "open": 2500 + }, + "containmentPaths": { + "lstat": 11110, + "realpath": 1756, + "open": 0 + } + } + }, + "warm": { + "samples": 20, + "medianMs": 306.555875, + "p95Ms": 337.061875, + "peakRssBytes": 229408768, + "operations": { + "lstat": 13610, + "realpath": 4256, + "open": 2500 + }, + "readBudget": { + "bytes": 2582784, + "files": 2500, + "entries": 3084 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 2500, + "realpath": 2500, + "open": 2500 + }, + "containmentPaths": { + "lstat": 11110, + "realpath": 1756, + "open": 0 + } + } + }, + "exactResults": [ + "vault/branch-00/branch-00/branch-03/note-00003.md", + "vault/branch-00/branch-02/branch-01/note-00017.md", + "vault/branch-01/branch-04/branch-05/note-00101.md", + "vault/branch-04/branch-06/branch-03/note-00307.md" + ], + "outputSha256": "3426468c469ccf3a134e93a71f6dea6f3a21d3e622918668af97a098bee9b3fd" + }, + { + "id": "high-hit", + "cold": { + "samples": 3, + "medianMs": 310.973208, + "p95Ms": 323.862625, + "peakRssBytes": 229425152, + "operations": { + "lstat": 13610, + "realpath": 4256, + "open": 2500 + }, + "readBudget": { + "bytes": 2582784, + "files": 2500, + "entries": 3084 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 2500, + "realpath": 2500, + "open": 2500 + }, + "containmentPaths": { + "lstat": 11110, + "realpath": 1756, + "open": 0 + } + } + }, + "warm": { + "samples": 20, + "medianMs": 307.031541, + "p95Ms": 313.105291, + "peakRssBytes": 230359040, + "operations": { + "lstat": 13610, + "realpath": 4256, + "open": 2500 + }, + "readBudget": { + "bytes": 2582784, + "files": 2500, + "entries": 3084 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 2500, + "realpath": 2500, + "open": 2500 + }, + "containmentPaths": { + "lstat": 11110, + "realpath": 1756, + "open": 0 + } + } + }, + "exactResults": [ + "vault/branch-00/branch-00/branch-00/note-00000.md", + "vault/branch-00/branch-00/branch-01/note-01025.md", + "vault/branch-00/branch-00/branch-02/note-02050.md", + "vault/branch-00/branch-01/branch-05/note-00525.md", + "vault/branch-00/branch-01/branch-06/note-01550.md", + "vault/branch-00/branch-03/branch-01/note-00025.md", + "vault/branch-00/branch-03/branch-02/note-01050.md", + "vault/branch-00/branch-03/branch-03/note-02075.md", + "vault/branch-00/branch-04/branch-06/note-00550.md", + "vault/branch-00/branch-04/branch-07/note-01575.md", + "vault/branch-00/branch-06/branch-02/note-00050.md", + "vault/branch-00/branch-06/branch-03/note-01075.md", + "vault/branch-00/branch-06/branch-04/note-02100.md", + "vault/branch-00/branch-07/branch-07/note-00575.md", + "vault/branch-01/branch-00/branch-00/note-01600.md", + "vault/branch-01/branch-01/branch-03/note-00075.md", + "vault/branch-01/branch-01/branch-04/note-01100.md", + "vault/branch-01/branch-01/branch-05/note-02125.md", + "vault/branch-01/branch-03/branch-00/note-00600.md", + "vault/branch-01/branch-03/branch-01/note-01625.md" + ], + "outputSha256": "cd69f09fe24115d75bb9e0b9d77835b6e944712ea4ac7e87e6275ea09cf4baa4" + } + ], + "rawSearchControl": [ + { + "safety": "unsafe-benchmark-only", + "id": "no-hit", + "cold": { + "samples": 3, + "medianMs": 107.731625, + "p95Ms": 109.204917, + "peakRssBytes": 229801984, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 2500 + }, + "readBudget": { + "files": 2500, + "bytes": 2582784, + "entries": 0 + } + }, + "warm": { + "samples": 20, + "medianMs": 105.944292, + "p95Ms": 109.645833, + "peakRssBytes": 230539264, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 2500 + }, + "readBudget": { + "files": 2500, + "bytes": 2582784, + "entries": 0 + } + }, + "exactResults": [], + "outputSha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + { + "safety": "unsafe-benchmark-only", + "id": "low-hit", + "cold": { + "samples": 3, + "medianMs": 108.130125, + "p95Ms": 109.976083, + "peakRssBytes": 228687872, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 2500 + }, + "readBudget": { + "files": 2500, + "bytes": 2582784, + "entries": 0 + } + }, + "warm": { + "samples": 20, + "medianMs": 109.858125, + "p95Ms": 111.841167, + "peakRssBytes": 228687872, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 2500 + }, + "readBudget": { + "files": 2500, + "bytes": 2582784, + "entries": 0 + } + }, + "exactResults": [ + "vault/branch-00/branch-00/branch-03/note-00003.md", + "vault/branch-00/branch-02/branch-01/note-00017.md", + "vault/branch-01/branch-04/branch-05/note-00101.md", + "vault/branch-04/branch-06/branch-03/note-00307.md" + ], + "outputSha256": "3426468c469ccf3a134e93a71f6dea6f3a21d3e622918668af97a098bee9b3fd" + }, + { + "safety": "unsafe-benchmark-only", + "id": "high-hit", + "cold": { + "samples": 3, + "medianMs": 109.48875, + "p95Ms": 110.862167, + "peakRssBytes": 227278848, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 2500 + }, + "readBudget": { + "files": 2500, + "bytes": 2582784, + "entries": 0 + } + }, + "warm": { + "samples": 20, + "medianMs": 110.760709, + "p95Ms": 123.48, + "peakRssBytes": 228704256, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 2500 + }, + "readBudget": { + "files": 2500, + "bytes": 2582784, + "entries": 0 + } + }, + "exactResults": [ + "vault/branch-00/branch-00/branch-00/note-00000.md", + "vault/branch-00/branch-00/branch-01/note-01025.md", + "vault/branch-00/branch-00/branch-02/note-02050.md", + "vault/branch-00/branch-01/branch-05/note-00525.md", + "vault/branch-00/branch-01/branch-06/note-01550.md", + "vault/branch-00/branch-03/branch-01/note-00025.md", + "vault/branch-00/branch-03/branch-02/note-01050.md", + "vault/branch-00/branch-03/branch-03/note-02075.md", + "vault/branch-00/branch-04/branch-06/note-00550.md", + "vault/branch-00/branch-04/branch-07/note-01575.md", + "vault/branch-00/branch-06/branch-02/note-00050.md", + "vault/branch-00/branch-06/branch-03/note-01075.md", + "vault/branch-00/branch-06/branch-04/note-02100.md", + "vault/branch-00/branch-07/branch-07/note-00575.md", + "vault/branch-01/branch-00/branch-00/note-01600.md", + "vault/branch-01/branch-01/branch-03/note-00075.md", + "vault/branch-01/branch-01/branch-04/note-01100.md", + "vault/branch-01/branch-01/branch-05/note-02125.md", + "vault/branch-01/branch-03/branch-00/note-00600.md", + "vault/branch-01/branch-03/branch-01/note-01625.md" + ], + "outputSha256": "cd69f09fe24115d75bb9e0b9d77835b6e944712ea4ac7e87e6275ea09cf4baa4" + } + ], + "rawReadControl": { + "id": "raw-read-control", + "cold": { + "samples": 3, + "medianMs": 37.21725, + "p95Ms": 39.799333, + "peakRssBytes": 216580096, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 2500 + }, + "readBudget": { + "files": 2500, + "bytes": 2582784, + "entries": 0 + } + }, + "warm": { + "samples": 20, + "medianMs": 36.036167, + "p95Ms": 38.002292, + "peakRssBytes": 216645632, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 2500 + }, + "readBudget": { + "files": 2500, + "bytes": 2582784, + "entries": 0 + } + }, + "exactResults": { + "fileCount": 2500, + "totalBytes": 2582784 + }, + "outputSha256": "e0592c7a140a929f31c46afb50699cb72c70caa987d5992b1043b40ce8f2f65a" + }, + "safeCorpusReadControl": { + "id": "safe-corpus-read-control", + "cold": { + "samples": 3, + "medianMs": 249.20625, + "p95Ms": 255.601833, + "peakRssBytes": 216793088, + "operations": { + "lstat": 13610, + "realpath": 4256, + "open": 2500 + }, + "readBudget": { + "bytes": 2582784, + "files": 2500, + "entries": 3084 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 2500, + "realpath": 2500, + "open": 2500 + }, + "containmentPaths": { + "lstat": 11110, + "realpath": 1756, + "open": 0 + } + } + }, + "warm": { + "samples": 20, + "medianMs": 247.350292, + "p95Ms": 254.323208, + "peakRssBytes": 206012416, + "operations": { + "lstat": 13610, + "realpath": 4256, + "open": 2500 + }, + "readBudget": { + "bytes": 2582784, + "files": 2500, + "entries": 3084 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 2500, + "realpath": 2500, + "open": 2500 + }, + "containmentPaths": { + "lstat": 11110, + "realpath": 1756, + "open": 0 + } + } + }, + "exactResults": { + "fileCount": 2500, + "totalBytes": 2582784 + }, + "outputSha256": "e0592c7a140a929f31c46afb50699cb72c70caa987d5992b1043b40ce8f2f65a" + } +} diff --git a/docs/benchmarks/reports/2026-08-13-2500-shallow-prose.report.json b/docs/benchmarks/reports/2026-08-13-2500-shallow-prose.report.json new file mode 100644 index 00000000..864487dc --- /dev/null +++ b/docs/benchmarks/reports/2026-08-13-2500-shallow-prose.report.json @@ -0,0 +1,489 @@ +{ + "schemaVersion": "dotaios-search-benchmark-result/v1", + "benchmarkId": "search-baseline-2026-08-13", + "manifestSha256": "b6c38cb5920f91b0a84c66be8181f6c14f7a1c73360fa4f07993b32a7704d55a", + "inventorySha256": "229da6c47d147660684099dacb22af3362a3467b2e607a6823dd7d8bc582a910", + "selection": { + "fileCount": 2500, + "layout": "shallow", + "distribution": "prose" + }, + "runtime": { + "node": "22.22.3", + "platform": "darwin", + "architecture": "arm64" + }, + "protocol": { + "coldSamples": 3, + "warmupSamples": 3, + "measuredSamples": 20, + "concurrency": 32, + "resultLimit": 20, + "rssPollIntervalMs": 5, + "coldDefinition": "fresh request-scoped reader before harness warm-up; operating-system file cache is uncontrolled", + "warmDefinition": "fresh request-scoped reader after warm-up in the same Node process", + "rawReadControl": { + "enabled": true, + "method": "open-handle-read-all", + "validation": "exact file count and byte total", + "concurrency": 32 + } + }, + "searches": [ + { + "id": "no-hit", + "cold": { + "samples": 3, + "medianMs": 139.805958, + "p95Ms": 159.154125, + "peakRssBytes": 118063104, + "operations": { + "lstat": 5032, + "realpath": 2510, + "open": 2500 + }, + "readBudget": { + "bytes": 2731459, + "files": 2500, + "entries": 2502 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 2500, + "realpath": 2500, + "open": 2500 + }, + "containmentPaths": { + "lstat": 2532, + "realpath": 10, + "open": 0 + } + } + }, + "warm": { + "samples": 20, + "medianMs": 133.152875, + "p95Ms": 135.459292, + "peakRssBytes": 144769024, + "operations": { + "lstat": 5032, + "realpath": 2510, + "open": 2500 + }, + "readBudget": { + "bytes": 2731459, + "files": 2500, + "entries": 2502 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 2500, + "realpath": 2500, + "open": 2500 + }, + "containmentPaths": { + "lstat": 2532, + "realpath": 10, + "open": 0 + } + } + }, + "exactResults": [], + "outputSha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + { + "id": "low-hit", + "cold": { + "samples": 3, + "medianMs": 145.43675, + "p95Ms": 154.888125, + "peakRssBytes": 144900096, + "operations": { + "lstat": 5032, + "realpath": 2510, + "open": 2500 + }, + "readBudget": { + "bytes": 2731459, + "files": 2500, + "entries": 2502 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 2500, + "realpath": 2500, + "open": 2500 + }, + "containmentPaths": { + "lstat": 2532, + "realpath": 10, + "open": 0 + } + } + }, + "warm": { + "samples": 20, + "medianMs": 135.888833, + "p95Ms": 138.629208, + "peakRssBytes": 144949248, + "operations": { + "lstat": 5032, + "realpath": 2510, + "open": 2500 + }, + "readBudget": { + "bytes": 2731459, + "files": 2500, + "entries": 2502 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 2500, + "realpath": 2500, + "open": 2500 + }, + "containmentPaths": { + "lstat": 2532, + "realpath": 10, + "open": 0 + } + } + }, + "exactResults": [ + "vault/bucket-01/note-00003.md", + "vault/bucket-01/note-00017.md", + "vault/bucket-01/note-00101.md", + "vault/bucket-01/note-00307.md" + ], + "outputSha256": "6bc96a7e535089d0f1232a6aa21cf6cf2b255f9d1ddccfe2dea5b098ff0c98b5" + }, + { + "id": "high-hit", + "cold": { + "samples": 3, + "medianMs": 136.340333, + "p95Ms": 138.190708, + "peakRssBytes": 144965632, + "operations": { + "lstat": 5032, + "realpath": 2510, + "open": 2500 + }, + "readBudget": { + "bytes": 2731459, + "files": 2500, + "entries": 2502 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 2500, + "realpath": 2500, + "open": 2500 + }, + "containmentPaths": { + "lstat": 2532, + "realpath": 10, + "open": 0 + } + } + }, + "warm": { + "samples": 20, + "medianMs": 135.227833, + "p95Ms": 139.356417, + "peakRssBytes": 145178624, + "operations": { + "lstat": 5032, + "realpath": 2510, + "open": 2500 + }, + "readBudget": { + "bytes": 2731459, + "files": 2500, + "entries": 2502 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 2500, + "realpath": 2500, + "open": 2500 + }, + "containmentPaths": { + "lstat": 2532, + "realpath": 10, + "open": 0 + } + } + }, + "exactResults": [ + "vault/bucket-00/note-00000.md", + "vault/bucket-00/note-00050.md", + "vault/bucket-00/note-00100.md", + "vault/bucket-00/note-00150.md", + "vault/bucket-00/note-00200.md", + "vault/bucket-00/note-00250.md", + "vault/bucket-00/note-00300.md", + "vault/bucket-00/note-00350.md", + "vault/bucket-00/note-00400.md", + "vault/bucket-00/note-00450.md", + "vault/bucket-00/note-00500.md", + "vault/bucket-00/note-00550.md", + "vault/bucket-00/note-00600.md", + "vault/bucket-00/note-00650.md", + "vault/bucket-00/note-00700.md", + "vault/bucket-00/note-00750.md", + "vault/bucket-00/note-00800.md", + "vault/bucket-00/note-00850.md", + "vault/bucket-00/note-00900.md", + "vault/bucket-00/note-00950.md" + ], + "outputSha256": "9b3d7e484ca325f107d3363721ce8b99b038149f4b79a4f23c25e52c393bfe3f" + } + ], + "rawSearchControl": [ + { + "safety": "unsafe-benchmark-only", + "id": "no-hit", + "cold": { + "samples": 3, + "medianMs": 97.316, + "p95Ms": 98.385916, + "peakRssBytes": 145555456, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 2500 + }, + "readBudget": { + "files": 2500, + "bytes": 2731459, + "entries": 0 + } + }, + "warm": { + "samples": 20, + "medianMs": 95.879792, + "p95Ms": 101.634334, + "peakRssBytes": 145735680, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 2500 + }, + "readBudget": { + "files": 2500, + "bytes": 2731459, + "entries": 0 + } + }, + "exactResults": [], + "outputSha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + { + "safety": "unsafe-benchmark-only", + "id": "low-hit", + "cold": { + "samples": 3, + "medianMs": 99.074, + "p95Ms": 99.187375, + "peakRssBytes": 145735680, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 2500 + }, + "readBudget": { + "files": 2500, + "bytes": 2731459, + "entries": 0 + } + }, + "warm": { + "samples": 20, + "medianMs": 98.937542, + "p95Ms": 105.103792, + "peakRssBytes": 145735680, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 2500 + }, + "readBudget": { + "files": 2500, + "bytes": 2731459, + "entries": 0 + } + }, + "exactResults": [ + "vault/bucket-01/note-00003.md", + "vault/bucket-01/note-00017.md", + "vault/bucket-01/note-00101.md", + "vault/bucket-01/note-00307.md" + ], + "outputSha256": "6bc96a7e535089d0f1232a6aa21cf6cf2b255f9d1ddccfe2dea5b098ff0c98b5" + }, + { + "safety": "unsafe-benchmark-only", + "id": "high-hit", + "cold": { + "samples": 3, + "medianMs": 100.391667, + "p95Ms": 101.549083, + "peakRssBytes": 145768448, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 2500 + }, + "readBudget": { + "files": 2500, + "bytes": 2731459, + "entries": 0 + } + }, + "warm": { + "samples": 20, + "medianMs": 99.205625, + "p95Ms": 102.94625, + "peakRssBytes": 145784832, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 2500 + }, + "readBudget": { + "files": 2500, + "bytes": 2731459, + "entries": 0 + } + }, + "exactResults": [ + "vault/bucket-00/note-00000.md", + "vault/bucket-00/note-00050.md", + "vault/bucket-00/note-00100.md", + "vault/bucket-00/note-00150.md", + "vault/bucket-00/note-00200.md", + "vault/bucket-00/note-00250.md", + "vault/bucket-00/note-00300.md", + "vault/bucket-00/note-00350.md", + "vault/bucket-00/note-00400.md", + "vault/bucket-00/note-00450.md", + "vault/bucket-00/note-00500.md", + "vault/bucket-00/note-00550.md", + "vault/bucket-00/note-00600.md", + "vault/bucket-00/note-00650.md", + "vault/bucket-00/note-00700.md", + "vault/bucket-00/note-00750.md", + "vault/bucket-00/note-00800.md", + "vault/bucket-00/note-00850.md", + "vault/bucket-00/note-00900.md", + "vault/bucket-00/note-00950.md" + ], + "outputSha256": "9b3d7e484ca325f107d3363721ce8b99b038149f4b79a4f23c25e52c393bfe3f" + } + ], + "rawReadControl": { + "id": "raw-read-control", + "cold": { + "samples": 3, + "medianMs": 35.961875, + "p95Ms": 36.390084, + "peakRssBytes": 148176896, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 2500 + }, + "readBudget": { + "files": 2500, + "bytes": 2731459, + "entries": 0 + } + }, + "warm": { + "samples": 20, + "medianMs": 35.301167, + "p95Ms": 37.68, + "peakRssBytes": 148258816, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 2500 + }, + "readBudget": { + "files": 2500, + "bytes": 2731459, + "entries": 0 + } + }, + "exactResults": { + "fileCount": 2500, + "totalBytes": 2731459 + }, + "outputSha256": "20cf6592fa8dfbeffac3b056d6e8189b44e7df62cc29b654280e12a1268287fe" + }, + "safeCorpusReadControl": { + "id": "safe-corpus-read-control", + "cold": { + "samples": 3, + "medianMs": 90.512916, + "p95Ms": 96.20825, + "peakRssBytes": 148602880, + "operations": { + "lstat": 5032, + "realpath": 2510, + "open": 2500 + }, + "readBudget": { + "bytes": 2731459, + "files": 2500, + "entries": 2502 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 2500, + "realpath": 2500, + "open": 2500 + }, + "containmentPaths": { + "lstat": 2532, + "realpath": 10, + "open": 0 + } + } + }, + "warm": { + "samples": 20, + "medianMs": 89.927208, + "p95Ms": 93.237333, + "peakRssBytes": 148979712, + "operations": { + "lstat": 5032, + "realpath": 2510, + "open": 2500 + }, + "readBudget": { + "bytes": 2731459, + "files": 2500, + "entries": 2502 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 2500, + "realpath": 2500, + "open": 2500 + }, + "containmentPaths": { + "lstat": 2532, + "realpath": 10, + "open": 0 + } + } + }, + "exactResults": { + "fileCount": 2500, + "totalBytes": 2731459 + }, + "outputSha256": "20cf6592fa8dfbeffac3b056d6e8189b44e7df62cc29b654280e12a1268287fe" + } +} diff --git a/docs/benchmarks/reports/2026-08-13-500-nested-high-entropy.report.json b/docs/benchmarks/reports/2026-08-13-500-nested-high-entropy.report.json new file mode 100644 index 00000000..0e795a6b --- /dev/null +++ b/docs/benchmarks/reports/2026-08-13-500-nested-high-entropy.report.json @@ -0,0 +1,489 @@ +{ + "schemaVersion": "dotaios-search-benchmark-result/v1", + "benchmarkId": "search-baseline-2026-08-13", + "manifestSha256": "b6c38cb5920f91b0a84c66be8181f6c14f7a1c73360fa4f07993b32a7704d55a", + "inventorySha256": "16d7f2dc55c48230a2a65d71e93453245cd4bfd6b562701cd5c5cc95c3a5fe19", + "selection": { + "fileCount": 500, + "layout": "nested", + "distribution": "high-entropy" + }, + "runtime": { + "node": "22.22.3", + "platform": "darwin", + "architecture": "arm64" + }, + "protocol": { + "coldSamples": 3, + "warmupSamples": 3, + "measuredSamples": 20, + "concurrency": 32, + "resultLimit": 20, + "rssPollIntervalMs": 5, + "coldDefinition": "fresh request-scoped reader before harness warm-up; operating-system file cache is uncontrolled", + "warmDefinition": "fresh request-scoped reader after warm-up in the same Node process", + "rawReadControl": { + "enabled": true, + "method": "open-handle-read-all", + "validation": "exact file count and byte total", + "concurrency": 32 + } + }, + "searches": [ + { + "id": "no-hit", + "cold": { + "samples": 3, + "medianMs": 189.142, + "p95Ms": 209.629834, + "peakRssBytes": 116817920, + "operations": { + "lstat": 9417, + "realpath": 2217, + "open": 500 + }, + "readBudget": { + "bytes": 519370, + "files": 500, + "entries": 1071 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 500, + "realpath": 500, + "open": 500 + }, + "containmentPaths": { + "lstat": 8917, + "realpath": 1717, + "open": 0 + } + } + }, + "warm": { + "samples": 20, + "medianMs": 177.982042, + "p95Ms": 181.799583, + "peakRssBytes": 145702912, + "operations": { + "lstat": 9417, + "realpath": 2217, + "open": 500 + }, + "readBudget": { + "bytes": 519370, + "files": 500, + "entries": 1071 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 500, + "realpath": 500, + "open": 500 + }, + "containmentPaths": { + "lstat": 8917, + "realpath": 1717, + "open": 0 + } + } + }, + "exactResults": [], + "outputSha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + { + "id": "low-hit", + "cold": { + "samples": 3, + "medianMs": 181.272541, + "p95Ms": 181.717958, + "peakRssBytes": 145981440, + "operations": { + "lstat": 9417, + "realpath": 2217, + "open": 500 + }, + "readBudget": { + "bytes": 519370, + "files": 500, + "entries": 1071 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 500, + "realpath": 500, + "open": 500 + }, + "containmentPaths": { + "lstat": 8917, + "realpath": 1717, + "open": 0 + } + } + }, + "warm": { + "samples": 20, + "medianMs": 178.694541, + "p95Ms": 181.222375, + "peakRssBytes": 147718144, + "operations": { + "lstat": 9417, + "realpath": 2217, + "open": 500 + }, + "readBudget": { + "bytes": 519370, + "files": 500, + "entries": 1071 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 500, + "realpath": 500, + "open": 500 + }, + "containmentPaths": { + "lstat": 8917, + "realpath": 1717, + "open": 0 + } + } + }, + "exactResults": [ + "vault/branch-00/branch-00/branch-03/note-00003.md", + "vault/branch-00/branch-02/branch-01/note-00017.md", + "vault/branch-01/branch-04/branch-05/note-00101.md", + "vault/branch-04/branch-06/branch-03/note-00307.md" + ], + "outputSha256": "3426468c469ccf3a134e93a71f6dea6f3a21d3e622918668af97a098bee9b3fd" + }, + { + "id": "high-hit", + "cold": { + "samples": 3, + "medianMs": 178.020208, + "p95Ms": 179.697417, + "peakRssBytes": 147718144, + "operations": { + "lstat": 9417, + "realpath": 2217, + "open": 500 + }, + "readBudget": { + "bytes": 519370, + "files": 500, + "entries": 1071 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 500, + "realpath": 500, + "open": 500 + }, + "containmentPaths": { + "lstat": 8917, + "realpath": 1717, + "open": 0 + } + } + }, + "warm": { + "samples": 20, + "medianMs": 179.376875, + "p95Ms": 181.105583, + "peakRssBytes": 148340736, + "operations": { + "lstat": 9417, + "realpath": 2217, + "open": 500 + }, + "readBudget": { + "bytes": 519370, + "files": 500, + "entries": 1071 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 500, + "realpath": 500, + "open": 500 + }, + "containmentPaths": { + "lstat": 8917, + "realpath": 1717, + "open": 0 + } + } + }, + "exactResults": [ + "vault/branch-00/branch-00/branch-00/note-00000.md", + "vault/branch-00/branch-03/branch-01/note-00025.md", + "vault/branch-00/branch-06/branch-02/note-00050.md", + "vault/branch-01/branch-01/branch-03/note-00075.md", + "vault/branch-01/branch-04/branch-04/note-00100.md", + "vault/branch-01/branch-07/branch-05/note-00125.md", + "vault/branch-02/branch-02/branch-06/note-00150.md", + "vault/branch-02/branch-05/branch-07/note-00175.md", + "vault/branch-03/branch-01/branch-00/note-00200.md", + "vault/branch-03/branch-04/branch-01/note-00225.md", + "vault/branch-03/branch-07/branch-02/note-00250.md", + "vault/branch-04/branch-02/branch-03/note-00275.md", + "vault/branch-04/branch-05/branch-04/note-00300.md", + "vault/branch-05/branch-00/branch-05/note-00325.md", + "vault/branch-05/branch-03/branch-06/note-00350.md", + "vault/branch-05/branch-06/branch-07/note-00375.md", + "vault/branch-06/branch-02/branch-00/note-00400.md", + "vault/branch-06/branch-05/branch-01/note-00425.md", + "vault/branch-07/branch-00/branch-02/note-00450.md", + "vault/branch-07/branch-03/branch-03/note-00475.md" + ], + "outputSha256": "7f89b11c4f05e3bdb46a359a0cc8f9df6f1f0ba7b5c91b591213096cac7a1c15" + } + ], + "rawSearchControl": [ + { + "safety": "unsafe-benchmark-only", + "id": "no-hit", + "cold": { + "samples": 3, + "medianMs": 19.695542, + "p95Ms": 20.315375, + "peakRssBytes": 146391040, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 500 + }, + "readBudget": { + "files": 500, + "bytes": 519370, + "entries": 0 + } + }, + "warm": { + "samples": 20, + "medianMs": 19.369916, + "p95Ms": 20.64975, + "peakRssBytes": 155664384, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 500 + }, + "readBudget": { + "files": 500, + "bytes": 519370, + "entries": 0 + } + }, + "exactResults": [], + "outputSha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + { + "safety": "unsafe-benchmark-only", + "id": "low-hit", + "cold": { + "samples": 3, + "medianMs": 19.421292, + "p95Ms": 21.291625, + "peakRssBytes": 151748608, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 500 + }, + "readBudget": { + "files": 500, + "bytes": 519370, + "entries": 0 + } + }, + "warm": { + "samples": 20, + "medianMs": 20.489833, + "p95Ms": 24.041458, + "peakRssBytes": 156762112, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 500 + }, + "readBudget": { + "files": 500, + "bytes": 519370, + "entries": 0 + } + }, + "exactResults": [ + "vault/branch-00/branch-00/branch-03/note-00003.md", + "vault/branch-00/branch-02/branch-01/note-00017.md", + "vault/branch-01/branch-04/branch-05/note-00101.md", + "vault/branch-04/branch-06/branch-03/note-00307.md" + ], + "outputSha256": "3426468c469ccf3a134e93a71f6dea6f3a21d3e622918668af97a098bee9b3fd" + }, + { + "safety": "unsafe-benchmark-only", + "id": "high-hit", + "cold": { + "samples": 3, + "medianMs": 20.446417, + "p95Ms": 20.558042, + "peakRssBytes": 158629888, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 500 + }, + "readBudget": { + "files": 500, + "bytes": 519370, + "entries": 0 + } + }, + "warm": { + "samples": 20, + "medianMs": 20.202708, + "p95Ms": 21.238958, + "peakRssBytes": 160530432, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 500 + }, + "readBudget": { + "files": 500, + "bytes": 519370, + "entries": 0 + } + }, + "exactResults": [ + "vault/branch-00/branch-00/branch-00/note-00000.md", + "vault/branch-00/branch-03/branch-01/note-00025.md", + "vault/branch-00/branch-06/branch-02/note-00050.md", + "vault/branch-01/branch-01/branch-03/note-00075.md", + "vault/branch-01/branch-04/branch-04/note-00100.md", + "vault/branch-01/branch-07/branch-05/note-00125.md", + "vault/branch-02/branch-02/branch-06/note-00150.md", + "vault/branch-02/branch-05/branch-07/note-00175.md", + "vault/branch-03/branch-01/branch-00/note-00200.md", + "vault/branch-03/branch-04/branch-01/note-00225.md", + "vault/branch-03/branch-07/branch-02/note-00250.md", + "vault/branch-04/branch-02/branch-03/note-00275.md", + "vault/branch-04/branch-05/branch-04/note-00300.md", + "vault/branch-05/branch-00/branch-05/note-00325.md", + "vault/branch-05/branch-03/branch-06/note-00350.md", + "vault/branch-05/branch-06/branch-07/note-00375.md", + "vault/branch-06/branch-02/branch-00/note-00400.md", + "vault/branch-06/branch-05/branch-01/note-00425.md", + "vault/branch-07/branch-00/branch-02/note-00450.md", + "vault/branch-07/branch-03/branch-03/note-00475.md" + ], + "outputSha256": "7f89b11c4f05e3bdb46a359a0cc8f9df6f1f0ba7b5c91b591213096cac7a1c15" + } + ], + "rawReadControl": { + "id": "raw-read-control", + "cold": { + "samples": 3, + "medianMs": 7.475875, + "p95Ms": 8.391292, + "peakRssBytes": 148144128, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 500 + }, + "readBudget": { + "files": 500, + "bytes": 519370, + "entries": 0 + } + }, + "warm": { + "samples": 20, + "medianMs": 7.304167, + "p95Ms": 7.906292, + "peakRssBytes": 149094400, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 500 + }, + "readBudget": { + "files": 500, + "bytes": 519370, + "entries": 0 + } + }, + "exactResults": { + "fileCount": 500, + "totalBytes": 519370 + }, + "outputSha256": "77e3eb3a648893817a0489c4f1bb50a90602b142dc285e9c9b856f710b40dfc6" + }, + "safeCorpusReadControl": { + "id": "safe-corpus-read-control", + "cold": { + "samples": 3, + "medianMs": 173.672208, + "p95Ms": 175.981333, + "peakRssBytes": 149127168, + "operations": { + "lstat": 9417, + "realpath": 2217, + "open": 500 + }, + "readBudget": { + "bytes": 519370, + "files": 500, + "entries": 1071 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 500, + "realpath": 500, + "open": 500 + }, + "containmentPaths": { + "lstat": 8917, + "realpath": 1717, + "open": 0 + } + } + }, + "warm": { + "samples": 20, + "medianMs": 169.536458, + "p95Ms": 173.305583, + "peakRssBytes": 146587648, + "operations": { + "lstat": 9417, + "realpath": 2217, + "open": 500 + }, + "readBudget": { + "bytes": 519370, + "files": 500, + "entries": 1071 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 500, + "realpath": 500, + "open": 500 + }, + "containmentPaths": { + "lstat": 8917, + "realpath": 1717, + "open": 0 + } + } + }, + "exactResults": { + "fileCount": 500, + "totalBytes": 519370 + }, + "outputSha256": "77e3eb3a648893817a0489c4f1bb50a90602b142dc285e9c9b856f710b40dfc6" + } +} diff --git a/docs/benchmarks/reports/2026-08-13-500-shallow-prose.report.json b/docs/benchmarks/reports/2026-08-13-500-shallow-prose.report.json new file mode 100644 index 00000000..2eecb1b7 --- /dev/null +++ b/docs/benchmarks/reports/2026-08-13-500-shallow-prose.report.json @@ -0,0 +1,489 @@ +{ + "schemaVersion": "dotaios-search-benchmark-result/v1", + "benchmarkId": "search-baseline-2026-08-13", + "manifestSha256": "b6c38cb5920f91b0a84c66be8181f6c14f7a1c73360fa4f07993b32a7704d55a", + "inventorySha256": "db6ab0454118d7cd1e2c54c3519db2018fddcba75e124105f13f1345dc525306", + "selection": { + "fileCount": 500, + "layout": "shallow", + "distribution": "prose" + }, + "runtime": { + "node": "22.22.3", + "platform": "darwin", + "architecture": "arm64" + }, + "protocol": { + "coldSamples": 3, + "warmupSamples": 3, + "measuredSamples": 20, + "concurrency": 32, + "resultLimit": 20, + "rssPollIntervalMs": 5, + "coldDefinition": "fresh request-scoped reader before harness warm-up; operating-system file cache is uncontrolled", + "warmDefinition": "fresh request-scoped reader after warm-up in the same Node process", + "rawReadControl": { + "enabled": true, + "method": "open-handle-read-all", + "validation": "exact file count and byte total", + "concurrency": 32 + } + }, + "searches": [ + { + "id": "no-hit", + "cold": { + "samples": 3, + "medianMs": 41.55975, + "p95Ms": 45.727542, + "peakRssBytes": 82378752, + "operations": { + "lstat": 1032, + "realpath": 510, + "open": 500 + }, + "readBudget": { + "bytes": 543978, + "files": 500, + "entries": 502 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 500, + "realpath": 500, + "open": 500 + }, + "containmentPaths": { + "lstat": 532, + "realpath": 10, + "open": 0 + } + } + }, + "warm": { + "samples": 20, + "medianMs": 27.861084, + "p95Ms": 33.004708, + "peakRssBytes": 117702656, + "operations": { + "lstat": 1032, + "realpath": 510, + "open": 500 + }, + "readBudget": { + "bytes": 543978, + "files": 500, + "entries": 502 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 500, + "realpath": 500, + "open": 500 + }, + "containmentPaths": { + "lstat": 532, + "realpath": 10, + "open": 0 + } + } + }, + "exactResults": [], + "outputSha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + { + "id": "low-hit", + "cold": { + "samples": 3, + "medianMs": 29.559917, + "p95Ms": 30.090417, + "peakRssBytes": 117915648, + "operations": { + "lstat": 1032, + "realpath": 510, + "open": 500 + }, + "readBudget": { + "bytes": 543978, + "files": 500, + "entries": 502 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 500, + "realpath": 500, + "open": 500 + }, + "containmentPaths": { + "lstat": 532, + "realpath": 10, + "open": 0 + } + } + }, + "warm": { + "samples": 20, + "medianMs": 27.242, + "p95Ms": 32.689542, + "peakRssBytes": 123682816, + "operations": { + "lstat": 1032, + "realpath": 510, + "open": 500 + }, + "readBudget": { + "bytes": 543978, + "files": 500, + "entries": 502 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 500, + "realpath": 500, + "open": 500 + }, + "containmentPaths": { + "lstat": 532, + "realpath": 10, + "open": 0 + } + } + }, + "exactResults": [ + "vault/bucket-01/note-00003.md", + "vault/bucket-01/note-00017.md", + "vault/bucket-01/note-00101.md", + "vault/bucket-01/note-00307.md" + ], + "outputSha256": "6bc96a7e535089d0f1232a6aa21cf6cf2b255f9d1ddccfe2dea5b098ff0c98b5" + }, + { + "id": "high-hit", + "cold": { + "samples": 3, + "medianMs": 28.539375, + "p95Ms": 29.099709, + "peakRssBytes": 123699200, + "operations": { + "lstat": 1032, + "realpath": 510, + "open": 500 + }, + "readBudget": { + "bytes": 543978, + "files": 500, + "entries": 502 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 500, + "realpath": 500, + "open": 500 + }, + "containmentPaths": { + "lstat": 532, + "realpath": 10, + "open": 0 + } + } + }, + "warm": { + "samples": 20, + "medianMs": 27.905167, + "p95Ms": 29.8005, + "peakRssBytes": 144064512, + "operations": { + "lstat": 1032, + "realpath": 510, + "open": 500 + }, + "readBudget": { + "bytes": 543978, + "files": 500, + "entries": 502 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 500, + "realpath": 500, + "open": 500 + }, + "containmentPaths": { + "lstat": 532, + "realpath": 10, + "open": 0 + } + } + }, + "exactResults": [ + "vault/bucket-00/note-00000.md", + "vault/bucket-00/note-00050.md", + "vault/bucket-00/note-00100.md", + "vault/bucket-00/note-00150.md", + "vault/bucket-00/note-00200.md", + "vault/bucket-00/note-00250.md", + "vault/bucket-00/note-00300.md", + "vault/bucket-00/note-00350.md", + "vault/bucket-00/note-00400.md", + "vault/bucket-00/note-00450.md", + "vault/bucket-01/note-00025.md", + "vault/bucket-01/note-00075.md", + "vault/bucket-01/note-00125.md", + "vault/bucket-01/note-00175.md", + "vault/bucket-01/note-00225.md", + "vault/bucket-01/note-00275.md", + "vault/bucket-01/note-00325.md", + "vault/bucket-01/note-00375.md", + "vault/bucket-01/note-00425.md", + "vault/bucket-01/note-00475.md" + ], + "outputSha256": "0c51d10ccea2f49c756db6a66d988559b5db90ccb4843048ebc80b01444a5fca" + } + ], + "rawSearchControl": [ + { + "safety": "unsafe-benchmark-only", + "id": "no-hit", + "cold": { + "samples": 3, + "medianMs": 19.646166, + "p95Ms": 19.99575, + "peakRssBytes": 146374656, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 500 + }, + "readBudget": { + "files": 500, + "bytes": 543978, + "entries": 0 + } + }, + "warm": { + "samples": 20, + "medianMs": 18.608333, + "p95Ms": 19.932583, + "peakRssBytes": 154861568, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 500 + }, + "readBudget": { + "files": 500, + "bytes": 543978, + "entries": 0 + } + }, + "exactResults": [], + "outputSha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + { + "safety": "unsafe-benchmark-only", + "id": "low-hit", + "cold": { + "samples": 3, + "medianMs": 18.857459, + "p95Ms": 19.675417, + "peakRssBytes": 154943488, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 500 + }, + "readBudget": { + "files": 500, + "bytes": 543978, + "entries": 0 + } + }, + "warm": { + "samples": 20, + "medianMs": 19.500625, + "p95Ms": 20.110708, + "peakRssBytes": 155189248, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 500 + }, + "readBudget": { + "files": 500, + "bytes": 543978, + "entries": 0 + } + }, + "exactResults": [ + "vault/bucket-01/note-00003.md", + "vault/bucket-01/note-00017.md", + "vault/bucket-01/note-00101.md", + "vault/bucket-01/note-00307.md" + ], + "outputSha256": "6bc96a7e535089d0f1232a6aa21cf6cf2b255f9d1ddccfe2dea5b098ff0c98b5" + }, + { + "safety": "unsafe-benchmark-only", + "id": "high-hit", + "cold": { + "samples": 3, + "medianMs": 19.561958, + "p95Ms": 19.694208, + "peakRssBytes": 155385856, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 500 + }, + "readBudget": { + "files": 500, + "bytes": 543978, + "entries": 0 + } + }, + "warm": { + "samples": 20, + "medianMs": 19.790416, + "p95Ms": 20.481792, + "peakRssBytes": 155320320, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 500 + }, + "readBudget": { + "files": 500, + "bytes": 543978, + "entries": 0 + } + }, + "exactResults": [ + "vault/bucket-00/note-00000.md", + "vault/bucket-00/note-00050.md", + "vault/bucket-00/note-00100.md", + "vault/bucket-00/note-00150.md", + "vault/bucket-00/note-00200.md", + "vault/bucket-00/note-00250.md", + "vault/bucket-00/note-00300.md", + "vault/bucket-00/note-00350.md", + "vault/bucket-00/note-00400.md", + "vault/bucket-00/note-00450.md", + "vault/bucket-01/note-00025.md", + "vault/bucket-01/note-00075.md", + "vault/bucket-01/note-00125.md", + "vault/bucket-01/note-00175.md", + "vault/bucket-01/note-00225.md", + "vault/bucket-01/note-00275.md", + "vault/bucket-01/note-00325.md", + "vault/bucket-01/note-00375.md", + "vault/bucket-01/note-00425.md", + "vault/bucket-01/note-00475.md" + ], + "outputSha256": "0c51d10ccea2f49c756db6a66d988559b5db90ccb4843048ebc80b01444a5fca" + } + ], + "rawReadControl": { + "id": "raw-read-control", + "cold": { + "samples": 3, + "medianMs": 7.413458, + "p95Ms": 7.416625, + "peakRssBytes": 157777920, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 500 + }, + "readBudget": { + "files": 500, + "bytes": 543978, + "entries": 0 + } + }, + "warm": { + "samples": 20, + "medianMs": 6.996583, + "p95Ms": 8.004792, + "peakRssBytes": 158007296, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 500 + }, + "readBudget": { + "files": 500, + "bytes": 543978, + "entries": 0 + } + }, + "exactResults": { + "fileCount": 500, + "totalBytes": 543978 + }, + "outputSha256": "0a60b34e49b0085aac50f5362f1cb3b84efd580fdf3e5b53b05934b8f57a962a" + }, + "safeCorpusReadControl": { + "id": "safe-corpus-read-control", + "cold": { + "samples": 3, + "medianMs": 20.05875, + "p95Ms": 21.098667, + "peakRssBytes": 158433280, + "operations": { + "lstat": 1032, + "realpath": 510, + "open": 500 + }, + "readBudget": { + "bytes": 543978, + "files": 500, + "entries": 502 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 500, + "realpath": 500, + "open": 500 + }, + "containmentPaths": { + "lstat": 532, + "realpath": 10, + "open": 0 + } + } + }, + "warm": { + "samples": 20, + "medianMs": 18.600583, + "p95Ms": 21.432917, + "peakRssBytes": 158564352, + "operations": { + "lstat": 1032, + "realpath": 510, + "open": 500 + }, + "readBudget": { + "bytes": 543978, + "files": 500, + "entries": 502 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 500, + "realpath": 500, + "open": 500 + }, + "containmentPaths": { + "lstat": 532, + "realpath": 10, + "open": 0 + } + } + }, + "exactResults": { + "fileCount": 500, + "totalBytes": 543978 + }, + "outputSha256": "0a60b34e49b0085aac50f5362f1cb3b84efd580fdf3e5b53b05934b8f57a962a" + } +} diff --git a/docs/benchmarks/reports/2026-08-14-public-10000-nested-high-entropy.report.json b/docs/benchmarks/reports/2026-08-14-public-10000-nested-high-entropy.report.json new file mode 100644 index 00000000..815b4b0e --- /dev/null +++ b/docs/benchmarks/reports/2026-08-14-public-10000-nested-high-entropy.report.json @@ -0,0 +1,551 @@ +{ + "schemaVersion": "dotaios-search-benchmark-result/v2", + "benchmarkId": "search-baseline-2026-08-13", + "manifestSha256": "b6c38cb5920f91b0a84c66be8181f6c14f7a1c73360fa4f07993b32a7704d55a", + "inventorySha256": "cc818211f579b4c54fcacfaa42ed195be052d156c5161c1ccdf2f23cdd1bf9a8", + "selection": { + "fileCount": 10000, + "layout": "nested", + "distribution": "high-entropy" + }, + "runtime": { + "node": "22.22.3", + "platform": "darwin", + "architecture": "arm64" + }, + "protocol": { + "coldSamples": 3, + "warmupSamples": 3, + "measuredSamples": 20, + "concurrency": 32, + "resultLimit": 20, + "rssPollIntervalMs": 5, + "coldDefinition": "fresh request-scoped reader before harness warm-up; operating-system file cache is uncontrolled", + "warmDefinition": "fresh request-scoped reader after warm-up in the same Node process", + "rawReadControl": { + "enabled": true, + "method": "open-handle-read-all", + "validation": "exact file count and byte total", + "concurrency": 32 + } + }, + "searches": [ + { + "id": "no-hit", + "cold": { + "samples": 3, + "medianMs": 932.341083, + "p95Ms": 979.751416, + "peakRssBytes": 541016064, + "operations": { + "lstat": 39468, + "realpath": 1807, + "open": 10000 + }, + "readBudget": { + "bytes": 10319743, + "files": 10000, + "entries": 10584 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 20000, + "realpath": 0, + "open": 10000 + }, + "containmentPaths": { + "lstat": 19468, + "realpath": 1807, + "open": 0 + } + } + }, + "warm": { + "samples": 20, + "medianMs": 926.978042, + "p95Ms": 937.5805, + "peakRssBytes": 494551040, + "operations": { + "lstat": 39468, + "realpath": 1807, + "open": 10000 + }, + "readBudget": { + "bytes": 10319743, + "files": 10000, + "entries": 10584 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 20000, + "realpath": 0, + "open": 10000 + }, + "containmentPaths": { + "lstat": 19468, + "realpath": 1807, + "open": 0 + } + } + }, + "surface": { + "entryPoint": "searchAios", + "requestedScope": "all", + "completeness": "complete", + "omissions": [], + "returnedScopes": [ + "sessions", + "context", + "memory", + "vault", + "decisions", + "skills", + "references", + "plugins" + ] + }, + "exactResults": [], + "outputSha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + { + "id": "low-hit", + "cold": { + "samples": 3, + "medianMs": 930.358875, + "p95Ms": 953.129167, + "peakRssBytes": 584826880, + "operations": { + "lstat": 39468, + "realpath": 1807, + "open": 10000 + }, + "readBudget": { + "bytes": 10319743, + "files": 10000, + "entries": 10584 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 20000, + "realpath": 0, + "open": 10000 + }, + "containmentPaths": { + "lstat": 19468, + "realpath": 1807, + "open": 0 + } + } + }, + "warm": { + "samples": 20, + "medianMs": 941.613875, + "p95Ms": 954.158167, + "peakRssBytes": 594886656, + "operations": { + "lstat": 39468, + "realpath": 1807, + "open": 10000 + }, + "readBudget": { + "bytes": 10319743, + "files": 10000, + "entries": 10584 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 20000, + "realpath": 0, + "open": 10000 + }, + "containmentPaths": { + "lstat": 19468, + "realpath": 1807, + "open": 0 + } + } + }, + "surface": { + "entryPoint": "searchAios", + "requestedScope": "all", + "completeness": "complete", + "omissions": [], + "returnedScopes": [ + "sessions", + "context", + "memory", + "vault", + "decisions", + "skills", + "references", + "plugins" + ] + }, + "exactResults": [ + "vault/branch-00/branch-00/branch-03/note-00003.md", + "vault/branch-00/branch-02/branch-01/note-00017.md", + "vault/branch-01/branch-04/branch-05/note-00101.md", + "vault/branch-04/branch-06/branch-03/note-00307.md" + ], + "outputSha256": "3426468c469ccf3a134e93a71f6dea6f3a21d3e622918668af97a098bee9b3fd" + }, + { + "id": "high-hit", + "cold": { + "samples": 3, + "medianMs": 936.134458, + "p95Ms": 947.206041, + "peakRssBytes": 608419840, + "operations": { + "lstat": 39468, + "realpath": 1807, + "open": 10000 + }, + "readBudget": { + "bytes": 10319743, + "files": 10000, + "entries": 10584 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 20000, + "realpath": 0, + "open": 10000 + }, + "containmentPaths": { + "lstat": 19468, + "realpath": 1807, + "open": 0 + } + } + }, + "warm": { + "samples": 20, + "medianMs": 941.180292, + "p95Ms": 954.98525, + "peakRssBytes": 614268928, + "operations": { + "lstat": 39468, + "realpath": 1807, + "open": 10000 + }, + "readBudget": { + "bytes": 10319743, + "files": 10000, + "entries": 10584 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 20000, + "realpath": 0, + "open": 10000 + }, + "containmentPaths": { + "lstat": 19468, + "realpath": 1807, + "open": 0 + } + } + }, + "surface": { + "entryPoint": "searchAios", + "requestedScope": "all", + "completeness": "complete", + "omissions": [], + "returnedScopes": [ + "sessions", + "context", + "memory", + "vault", + "decisions", + "skills", + "references", + "plugins" + ] + }, + "exactResults": [ + "vault/branch-00/branch-00/branch-00/note-00000.md", + "vault/branch-00/branch-00/branch-01/note-01025.md", + "vault/branch-00/branch-00/branch-02/note-02050.md", + "vault/branch-00/branch-00/branch-03/note-03075.md", + "vault/branch-00/branch-00/branch-04/note-04100.md", + "vault/branch-00/branch-00/branch-05/note-05125.md", + "vault/branch-00/branch-00/branch-06/note-06150.md", + "vault/branch-00/branch-00/branch-07/note-07175.md", + "vault/branch-00/branch-01/branch-00/note-08200.md", + "vault/branch-00/branch-01/branch-01/note-09225.md", + "vault/branch-00/branch-01/branch-05/note-00525.md", + "vault/branch-00/branch-01/branch-06/note-01550.md", + "vault/branch-00/branch-01/branch-07/note-02575.md", + "vault/branch-00/branch-02/branch-00/note-03600.md", + "vault/branch-00/branch-02/branch-01/note-04625.md", + "vault/branch-00/branch-02/branch-02/note-05650.md", + "vault/branch-00/branch-02/branch-03/note-06675.md", + "vault/branch-00/branch-02/branch-04/note-07700.md", + "vault/branch-00/branch-02/branch-05/note-08725.md", + "vault/branch-00/branch-02/branch-06/note-09750.md" + ], + "outputSha256": "0bcab7da3781ba39e4bdcbc5e04bb5f2bc293ad8dbc75d19f43ee4fca5c0da58" + } + ], + "searchSurface": { + "entryPoint": "searchAios", + "requestedScope": "all", + "completeness": "complete" + }, + "operationGate": { + "comparison": "safe-corpus-read-control-plus-fixed-directory-allowance", + "allowance": { + "lstat": 512, + "realpath": 256, + "open": 16 + }, + "passed": true + }, + "rawSearchControl": [ + { + "safety": "unsafe-benchmark-only", + "id": "no-hit", + "cold": { + "samples": 3, + "medianMs": 438.585291, + "p95Ms": 458.887208, + "peakRssBytes": 563609600, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 10000 + }, + "readBudget": { + "files": 10000, + "bytes": 10319743, + "entries": 0 + } + }, + "warm": { + "samples": 20, + "medianMs": 432.335542, + "p95Ms": 441.400708, + "peakRssBytes": 609812480, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 10000 + }, + "readBudget": { + "files": 10000, + "bytes": 10319743, + "entries": 0 + } + }, + "exactResults": [], + "outputSha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + { + "safety": "unsafe-benchmark-only", + "id": "low-hit", + "cold": { + "samples": 3, + "medianMs": 449.243667, + "p95Ms": 487.028833, + "peakRssBytes": 507101184, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 10000 + }, + "readBudget": { + "files": 10000, + "bytes": 10319743, + "entries": 0 + } + }, + "warm": { + "samples": 20, + "medianMs": 445.957375, + "p95Ms": 449.731334, + "peakRssBytes": 649805824, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 10000 + }, + "readBudget": { + "files": 10000, + "bytes": 10319743, + "entries": 0 + } + }, + "exactResults": [ + "vault/branch-00/branch-00/branch-03/note-00003.md", + "vault/branch-00/branch-02/branch-01/note-00017.md", + "vault/branch-01/branch-04/branch-05/note-00101.md", + "vault/branch-04/branch-06/branch-03/note-00307.md" + ], + "outputSha256": "3426468c469ccf3a134e93a71f6dea6f3a21d3e622918668af97a098bee9b3fd" + }, + { + "safety": "unsafe-benchmark-only", + "id": "high-hit", + "cold": { + "samples": 3, + "medianMs": 445.212458, + "p95Ms": 454.356708, + "peakRssBytes": 535576576, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 10000 + }, + "readBudget": { + "files": 10000, + "bytes": 10319743, + "entries": 0 + } + }, + "warm": { + "samples": 20, + "medianMs": 451.265625, + "p95Ms": 462.859917, + "peakRssBytes": 651657216, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 10000 + }, + "readBudget": { + "files": 10000, + "bytes": 10319743, + "entries": 0 + } + }, + "exactResults": [ + "vault/branch-00/branch-00/branch-00/note-00000.md", + "vault/branch-00/branch-00/branch-01/note-01025.md", + "vault/branch-00/branch-00/branch-02/note-02050.md", + "vault/branch-00/branch-00/branch-03/note-03075.md", + "vault/branch-00/branch-00/branch-04/note-04100.md", + "vault/branch-00/branch-00/branch-05/note-05125.md", + "vault/branch-00/branch-00/branch-06/note-06150.md", + "vault/branch-00/branch-00/branch-07/note-07175.md", + "vault/branch-00/branch-01/branch-00/note-08200.md", + "vault/branch-00/branch-01/branch-01/note-09225.md", + "vault/branch-00/branch-01/branch-05/note-00525.md", + "vault/branch-00/branch-01/branch-06/note-01550.md", + "vault/branch-00/branch-01/branch-07/note-02575.md", + "vault/branch-00/branch-02/branch-00/note-03600.md", + "vault/branch-00/branch-02/branch-01/note-04625.md", + "vault/branch-00/branch-02/branch-02/note-05650.md", + "vault/branch-00/branch-02/branch-03/note-06675.md", + "vault/branch-00/branch-02/branch-04/note-07700.md", + "vault/branch-00/branch-02/branch-05/note-08725.md", + "vault/branch-00/branch-02/branch-06/note-09750.md" + ], + "outputSha256": "0bcab7da3781ba39e4bdcbc5e04bb5f2bc293ad8dbc75d19f43ee4fca5c0da58" + } + ], + "rawReadControl": { + "id": "raw-read-control", + "cold": { + "samples": 3, + "medianMs": 138.236708, + "p95Ms": 142.722792, + "peakRssBytes": 493813760, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 10000 + }, + "readBudget": { + "files": 10000, + "bytes": 10319743, + "entries": 0 + } + }, + "warm": { + "samples": 20, + "medianMs": 137.689791, + "p95Ms": 139.5425, + "peakRssBytes": 493895680, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 10000 + }, + "readBudget": { + "files": 10000, + "bytes": 10319743, + "entries": 0 + } + }, + "exactResults": { + "fileCount": 10000, + "totalBytes": 10319743 + }, + "outputSha256": "f0d599e033e431e53d934958618fb5f6d86b584ec3dd361252e0677895078d4c" + }, + "safeCorpusReadControl": { + "id": "safe-corpus-read-control", + "cold": { + "samples": 3, + "medianMs": 697.215584, + "p95Ms": 702.11625, + "peakRssBytes": 500465664, + "operations": { + "lstat": 39195, + "realpath": 1756, + "open": 10000 + }, + "readBudget": { + "bytes": 10319743, + "files": 10000, + "entries": 10584 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 20000, + "realpath": 0, + "open": 10000 + }, + "containmentPaths": { + "lstat": 19195, + "realpath": 1756, + "open": 0 + } + } + }, + "warm": { + "samples": 20, + "medianMs": 692.901041, + "p95Ms": 720.881875, + "peakRssBytes": 459653120, + "operations": { + "lstat": 39195, + "realpath": 1756, + "open": 10000 + }, + "readBudget": { + "bytes": 10319743, + "files": 10000, + "entries": 10584 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 20000, + "realpath": 0, + "open": 10000 + }, + "containmentPaths": { + "lstat": 19195, + "realpath": 1756, + "open": 0 + } + } + }, + "exactResults": { + "fileCount": 10000, + "totalBytes": 10319743 + }, + "outputSha256": "f0d599e033e431e53d934958618fb5f6d86b584ec3dd361252e0677895078d4c" + } +} diff --git a/docs/benchmarks/reports/2026-08-14-public-10000-shallow-prose.report.json b/docs/benchmarks/reports/2026-08-14-public-10000-shallow-prose.report.json new file mode 100644 index 00000000..044e5527 --- /dev/null +++ b/docs/benchmarks/reports/2026-08-14-public-10000-shallow-prose.report.json @@ -0,0 +1,551 @@ +{ + "schemaVersion": "dotaios-search-benchmark-result/v2", + "benchmarkId": "search-baseline-2026-08-13", + "manifestSha256": "b6c38cb5920f91b0a84c66be8181f6c14f7a1c73360fa4f07993b32a7704d55a", + "inventorySha256": "50a3e8026807256715a8f7ea5ffcb8d35c8a074d04c08badbf4e27c6119008c6", + "selection": { + "fileCount": 10000, + "layout": "shallow", + "distribution": "prose" + }, + "runtime": { + "node": "22.22.3", + "platform": "darwin", + "architecture": "arm64" + }, + "protocol": { + "coldSamples": 3, + "warmupSamples": 3, + "measuredSamples": 20, + "concurrency": 32, + "resultLimit": 20, + "rssPollIntervalMs": 5, + "coldDefinition": "fresh request-scoped reader before harness warm-up; operating-system file cache is uncontrolled", + "warmDefinition": "fresh request-scoped reader after warm-up in the same Node process", + "rawReadControl": { + "enabled": true, + "method": "open-handle-read-all", + "validation": "exact file count and byte total", + "concurrency": 32 + } + }, + "searches": [ + { + "id": "no-hit", + "cold": { + "samples": 3, + "medianMs": 674.226791, + "p95Ms": 708.283292, + "peakRssBytes": 233488384, + "operations": { + "lstat": 30308, + "realpath": 61, + "open": 10000 + }, + "readBudget": { + "bytes": 10886041, + "files": 10000, + "entries": 10002 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 20000, + "realpath": 0, + "open": 10000 + }, + "containmentPaths": { + "lstat": 10308, + "realpath": 61, + "open": 0 + } + } + }, + "warm": { + "samples": 20, + "medianMs": 666.226416, + "p95Ms": 675.498959, + "peakRssBytes": 313868288, + "operations": { + "lstat": 30308, + "realpath": 61, + "open": 10000 + }, + "readBudget": { + "bytes": 10886041, + "files": 10000, + "entries": 10002 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 20000, + "realpath": 0, + "open": 10000 + }, + "containmentPaths": { + "lstat": 10308, + "realpath": 61, + "open": 0 + } + } + }, + "surface": { + "entryPoint": "searchAios", + "requestedScope": "all", + "completeness": "complete", + "omissions": [], + "returnedScopes": [ + "sessions", + "context", + "memory", + "vault", + "decisions", + "skills", + "references", + "plugins" + ] + }, + "exactResults": [], + "outputSha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + { + "id": "low-hit", + "cold": { + "samples": 3, + "medianMs": 679.0175, + "p95Ms": 687.715875, + "peakRssBytes": 311918592, + "operations": { + "lstat": 30308, + "realpath": 61, + "open": 10000 + }, + "readBudget": { + "bytes": 10886041, + "files": 10000, + "entries": 10002 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 20000, + "realpath": 0, + "open": 10000 + }, + "containmentPaths": { + "lstat": 10308, + "realpath": 61, + "open": 0 + } + } + }, + "warm": { + "samples": 20, + "medianMs": 679.186208, + "p95Ms": 689.509458, + "peakRssBytes": 314654720, + "operations": { + "lstat": 30308, + "realpath": 61, + "open": 10000 + }, + "readBudget": { + "bytes": 10886041, + "files": 10000, + "entries": 10002 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 20000, + "realpath": 0, + "open": 10000 + }, + "containmentPaths": { + "lstat": 10308, + "realpath": 61, + "open": 0 + } + } + }, + "surface": { + "entryPoint": "searchAios", + "requestedScope": "all", + "completeness": "complete", + "omissions": [], + "returnedScopes": [ + "sessions", + "context", + "memory", + "vault", + "decisions", + "skills", + "references", + "plugins" + ] + }, + "exactResults": [ + "vault/bucket-01/note-00003.md", + "vault/bucket-01/note-00017.md", + "vault/bucket-01/note-00101.md", + "vault/bucket-01/note-00307.md" + ], + "outputSha256": "6bc96a7e535089d0f1232a6aa21cf6cf2b255f9d1ddccfe2dea5b098ff0c98b5" + }, + { + "id": "high-hit", + "cold": { + "samples": 3, + "medianMs": 678.39, + "p95Ms": 683.1565, + "peakRssBytes": 314146816, + "operations": { + "lstat": 30308, + "realpath": 61, + "open": 10000 + }, + "readBudget": { + "bytes": 10886041, + "files": 10000, + "entries": 10002 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 20000, + "realpath": 0, + "open": 10000 + }, + "containmentPaths": { + "lstat": 10308, + "realpath": 61, + "open": 0 + } + } + }, + "warm": { + "samples": 20, + "medianMs": 678.746958, + "p95Ms": 689.644042, + "peakRssBytes": 336805888, + "operations": { + "lstat": 30308, + "realpath": 61, + "open": 10000 + }, + "readBudget": { + "bytes": 10886041, + "files": 10000, + "entries": 10002 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 20000, + "realpath": 0, + "open": 10000 + }, + "containmentPaths": { + "lstat": 10308, + "realpath": 61, + "open": 0 + } + } + }, + "surface": { + "entryPoint": "searchAios", + "requestedScope": "all", + "completeness": "complete", + "omissions": [], + "returnedScopes": [ + "sessions", + "context", + "memory", + "vault", + "decisions", + "skills", + "references", + "plugins" + ] + }, + "exactResults": [ + "vault/bucket-00/note-00000.md", + "vault/bucket-00/note-00050.md", + "vault/bucket-00/note-00100.md", + "vault/bucket-00/note-00150.md", + "vault/bucket-00/note-00200.md", + "vault/bucket-00/note-00250.md", + "vault/bucket-00/note-00300.md", + "vault/bucket-00/note-00350.md", + "vault/bucket-00/note-00400.md", + "vault/bucket-00/note-00450.md", + "vault/bucket-00/note-00500.md", + "vault/bucket-00/note-00550.md", + "vault/bucket-00/note-00600.md", + "vault/bucket-00/note-00650.md", + "vault/bucket-00/note-00700.md", + "vault/bucket-00/note-00750.md", + "vault/bucket-00/note-00800.md", + "vault/bucket-00/note-00850.md", + "vault/bucket-00/note-00900.md", + "vault/bucket-00/note-00950.md" + ], + "outputSha256": "9b3d7e484ca325f107d3363721ce8b99b038149f4b79a4f23c25e52c393bfe3f" + } + ], + "searchSurface": { + "entryPoint": "searchAios", + "requestedScope": "all", + "completeness": "complete" + }, + "operationGate": { + "comparison": "safe-corpus-read-control-plus-fixed-directory-allowance", + "allowance": { + "lstat": 512, + "realpath": 256, + "open": 16 + }, + "passed": true + }, + "rawSearchControl": [ + { + "safety": "unsafe-benchmark-only", + "id": "no-hit", + "cold": { + "samples": 3, + "medianMs": 368.323, + "p95Ms": 372.413125, + "peakRssBytes": 334807040, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 10000 + }, + "readBudget": { + "files": 10000, + "bytes": 10886041, + "entries": 0 + } + }, + "warm": { + "samples": 20, + "medianMs": 365.848458, + "p95Ms": 369.360375, + "peakRssBytes": 331792384, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 10000 + }, + "readBudget": { + "files": 10000, + "bytes": 10886041, + "entries": 0 + } + }, + "exactResults": [], + "outputSha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + { + "safety": "unsafe-benchmark-only", + "id": "low-hit", + "cold": { + "samples": 3, + "medianMs": 378.68975, + "p95Ms": 381.905083, + "peakRssBytes": 330596352, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 10000 + }, + "readBudget": { + "files": 10000, + "bytes": 10886041, + "entries": 0 + } + }, + "warm": { + "samples": 20, + "medianMs": 379.330375, + "p95Ms": 392.949875, + "peakRssBytes": 330612736, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 10000 + }, + "readBudget": { + "files": 10000, + "bytes": 10886041, + "entries": 0 + } + }, + "exactResults": [ + "vault/bucket-01/note-00003.md", + "vault/bucket-01/note-00017.md", + "vault/bucket-01/note-00101.md", + "vault/bucket-01/note-00307.md" + ], + "outputSha256": "6bc96a7e535089d0f1232a6aa21cf6cf2b255f9d1ddccfe2dea5b098ff0c98b5" + }, + { + "safety": "unsafe-benchmark-only", + "id": "high-hit", + "cold": { + "samples": 3, + "medianMs": 378.327041, + "p95Ms": 380.364292, + "peakRssBytes": 330612736, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 10000 + }, + "readBudget": { + "files": 10000, + "bytes": 10886041, + "entries": 0 + } + }, + "warm": { + "samples": 20, + "medianMs": 378.1865, + "p95Ms": 394.093917, + "peakRssBytes": 331808768, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 10000 + }, + "readBudget": { + "files": 10000, + "bytes": 10886041, + "entries": 0 + } + }, + "exactResults": [ + "vault/bucket-00/note-00000.md", + "vault/bucket-00/note-00050.md", + "vault/bucket-00/note-00100.md", + "vault/bucket-00/note-00150.md", + "vault/bucket-00/note-00200.md", + "vault/bucket-00/note-00250.md", + "vault/bucket-00/note-00300.md", + "vault/bucket-00/note-00350.md", + "vault/bucket-00/note-00400.md", + "vault/bucket-00/note-00450.md", + "vault/bucket-00/note-00500.md", + "vault/bucket-00/note-00550.md", + "vault/bucket-00/note-00600.md", + "vault/bucket-00/note-00650.md", + "vault/bucket-00/note-00700.md", + "vault/bucket-00/note-00750.md", + "vault/bucket-00/note-00800.md", + "vault/bucket-00/note-00850.md", + "vault/bucket-00/note-00900.md", + "vault/bucket-00/note-00950.md" + ], + "outputSha256": "9b3d7e484ca325f107d3363721ce8b99b038149f4b79a4f23c25e52c393bfe3f" + } + ], + "rawReadControl": { + "id": "raw-read-control", + "cold": { + "samples": 3, + "medianMs": 134.151292, + "p95Ms": 135.742208, + "peakRssBytes": 332070912, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 10000 + }, + "readBudget": { + "files": 10000, + "bytes": 10886041, + "entries": 0 + } + }, + "warm": { + "samples": 20, + "medianMs": 133.114292, + "p95Ms": 135.479125, + "peakRssBytes": 332087296, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 10000 + }, + "readBudget": { + "files": 10000, + "bytes": 10886041, + "entries": 0 + } + }, + "exactResults": { + "fileCount": 10000, + "totalBytes": 10886041 + }, + "outputSha256": "a50af3418a32f28d3a9f37ff53b576c7244e6c68d0dbd13c2c14f31d0580cac2" + }, + "safeCorpusReadControl": { + "id": "safe-corpus-read-control", + "cold": { + "samples": 3, + "medianMs": 497.184791, + "p95Ms": 516.633667, + "peakRssBytes": 333692928, + "operations": { + "lstat": 30035, + "realpath": 10, + "open": 10000 + }, + "readBudget": { + "bytes": 10886041, + "files": 10000, + "entries": 10002 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 20000, + "realpath": 0, + "open": 10000 + }, + "containmentPaths": { + "lstat": 10035, + "realpath": 10, + "open": 0 + } + } + }, + "warm": { + "samples": 20, + "medianMs": 497.559167, + "p95Ms": 503.842875, + "peakRssBytes": 339001344, + "operations": { + "lstat": 30035, + "realpath": 10, + "open": 10000 + }, + "readBudget": { + "bytes": 10886041, + "files": 10000, + "entries": 10002 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 20000, + "realpath": 0, + "open": 10000 + }, + "containmentPaths": { + "lstat": 10035, + "realpath": 10, + "open": 0 + } + } + }, + "exactResults": { + "fileCount": 10000, + "totalBytes": 10886041 + }, + "outputSha256": "a50af3418a32f28d3a9f37ff53b576c7244e6c68d0dbd13c2c14f31d0580cac2" + } +} diff --git a/docs/benchmarks/reports/2026-08-14-public-2500-nested-high-entropy.report.json b/docs/benchmarks/reports/2026-08-14-public-2500-nested-high-entropy.report.json new file mode 100644 index 00000000..66ebed36 --- /dev/null +++ b/docs/benchmarks/reports/2026-08-14-public-2500-nested-high-entropy.report.json @@ -0,0 +1,551 @@ +{ + "schemaVersion": "dotaios-search-benchmark-result/v2", + "benchmarkId": "search-baseline-2026-08-13", + "manifestSha256": "b6c38cb5920f91b0a84c66be8181f6c14f7a1c73360fa4f07993b32a7704d55a", + "inventorySha256": "42178a1a3d649a5223662730f7373ce1505289feefbb88dd2afad3918194e537", + "selection": { + "fileCount": 2500, + "layout": "nested", + "distribution": "high-entropy" + }, + "runtime": { + "node": "22.22.3", + "platform": "darwin", + "architecture": "arm64" + }, + "protocol": { + "coldSamples": 3, + "warmupSamples": 3, + "measuredSamples": 20, + "concurrency": 32, + "resultLimit": 20, + "rssPollIntervalMs": 5, + "coldDefinition": "fresh request-scoped reader before harness warm-up; operating-system file cache is uncontrolled", + "warmDefinition": "fresh request-scoped reader after warm-up in the same Node process", + "rawReadControl": { + "enabled": true, + "method": "open-handle-read-all", + "validation": "exact file count and byte total", + "concurrency": 32 + } + }, + "searches": [ + { + "id": "no-hit", + "cold": { + "samples": 3, + "medianMs": 358.031416, + "p95Ms": 385.568166, + "peakRssBytes": 193609728, + "operations": { + "lstat": 16968, + "realpath": 1807, + "open": 2500 + }, + "readBudget": { + "bytes": 2582784, + "files": 2500, + "entries": 3084 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 5000, + "realpath": 0, + "open": 2500 + }, + "containmentPaths": { + "lstat": 11968, + "realpath": 1807, + "open": 0 + } + } + }, + "warm": { + "samples": 20, + "medianMs": 342.319125, + "p95Ms": 347.990708, + "peakRssBytes": 206585856, + "operations": { + "lstat": 16968, + "realpath": 1807, + "open": 2500 + }, + "readBudget": { + "bytes": 2582784, + "files": 2500, + "entries": 3084 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 5000, + "realpath": 0, + "open": 2500 + }, + "containmentPaths": { + "lstat": 11968, + "realpath": 1807, + "open": 0 + } + } + }, + "surface": { + "entryPoint": "searchAios", + "requestedScope": "all", + "completeness": "complete", + "omissions": [], + "returnedScopes": [ + "sessions", + "context", + "memory", + "vault", + "decisions", + "skills", + "references", + "plugins" + ] + }, + "exactResults": [], + "outputSha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + { + "id": "low-hit", + "cold": { + "samples": 3, + "medianMs": 346.570667, + "p95Ms": 349.699792, + "peakRssBytes": 201637888, + "operations": { + "lstat": 16968, + "realpath": 1807, + "open": 2500 + }, + "readBudget": { + "bytes": 2582784, + "files": 2500, + "entries": 3084 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 5000, + "realpath": 0, + "open": 2500 + }, + "containmentPaths": { + "lstat": 11968, + "realpath": 1807, + "open": 0 + } + } + }, + "warm": { + "samples": 20, + "medianMs": 345.249292, + "p95Ms": 357.027166, + "peakRssBytes": 207863808, + "operations": { + "lstat": 16968, + "realpath": 1807, + "open": 2500 + }, + "readBudget": { + "bytes": 2582784, + "files": 2500, + "entries": 3084 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 5000, + "realpath": 0, + "open": 2500 + }, + "containmentPaths": { + "lstat": 11968, + "realpath": 1807, + "open": 0 + } + } + }, + "surface": { + "entryPoint": "searchAios", + "requestedScope": "all", + "completeness": "complete", + "omissions": [], + "returnedScopes": [ + "sessions", + "context", + "memory", + "vault", + "decisions", + "skills", + "references", + "plugins" + ] + }, + "exactResults": [ + "vault/branch-00/branch-00/branch-03/note-00003.md", + "vault/branch-00/branch-02/branch-01/note-00017.md", + "vault/branch-01/branch-04/branch-05/note-00101.md", + "vault/branch-04/branch-06/branch-03/note-00307.md" + ], + "outputSha256": "3426468c469ccf3a134e93a71f6dea6f3a21d3e622918668af97a098bee9b3fd" + }, + { + "id": "high-hit", + "cold": { + "samples": 3, + "medianMs": 345.223042, + "p95Ms": 346.288, + "peakRssBytes": 207880192, + "operations": { + "lstat": 16968, + "realpath": 1807, + "open": 2500 + }, + "readBudget": { + "bytes": 2582784, + "files": 2500, + "entries": 3084 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 5000, + "realpath": 0, + "open": 2500 + }, + "containmentPaths": { + "lstat": 11968, + "realpath": 1807, + "open": 0 + } + } + }, + "warm": { + "samples": 20, + "medianMs": 346.198417, + "p95Ms": 348.900417, + "peakRssBytes": 207880192, + "operations": { + "lstat": 16968, + "realpath": 1807, + "open": 2500 + }, + "readBudget": { + "bytes": 2582784, + "files": 2500, + "entries": 3084 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 5000, + "realpath": 0, + "open": 2500 + }, + "containmentPaths": { + "lstat": 11968, + "realpath": 1807, + "open": 0 + } + } + }, + "surface": { + "entryPoint": "searchAios", + "requestedScope": "all", + "completeness": "complete", + "omissions": [], + "returnedScopes": [ + "sessions", + "context", + "memory", + "vault", + "decisions", + "skills", + "references", + "plugins" + ] + }, + "exactResults": [ + "vault/branch-00/branch-00/branch-00/note-00000.md", + "vault/branch-00/branch-00/branch-01/note-01025.md", + "vault/branch-00/branch-00/branch-02/note-02050.md", + "vault/branch-00/branch-01/branch-05/note-00525.md", + "vault/branch-00/branch-01/branch-06/note-01550.md", + "vault/branch-00/branch-03/branch-01/note-00025.md", + "vault/branch-00/branch-03/branch-02/note-01050.md", + "vault/branch-00/branch-03/branch-03/note-02075.md", + "vault/branch-00/branch-04/branch-06/note-00550.md", + "vault/branch-00/branch-04/branch-07/note-01575.md", + "vault/branch-00/branch-06/branch-02/note-00050.md", + "vault/branch-00/branch-06/branch-03/note-01075.md", + "vault/branch-00/branch-06/branch-04/note-02100.md", + "vault/branch-00/branch-07/branch-07/note-00575.md", + "vault/branch-01/branch-00/branch-00/note-01600.md", + "vault/branch-01/branch-01/branch-03/note-00075.md", + "vault/branch-01/branch-01/branch-04/note-01100.md", + "vault/branch-01/branch-01/branch-05/note-02125.md", + "vault/branch-01/branch-03/branch-00/note-00600.md", + "vault/branch-01/branch-03/branch-01/note-01625.md" + ], + "outputSha256": "cd69f09fe24115d75bb9e0b9d77835b6e944712ea4ac7e87e6275ea09cf4baa4" + } + ], + "searchSurface": { + "entryPoint": "searchAios", + "requestedScope": "all", + "completeness": "complete" + }, + "operationGate": { + "comparison": "safe-corpus-read-control-plus-fixed-directory-allowance", + "allowance": { + "lstat": 512, + "realpath": 256, + "open": 16 + }, + "passed": true + }, + "rawSearchControl": [ + { + "safety": "unsafe-benchmark-only", + "id": "no-hit", + "cold": { + "samples": 3, + "medianMs": 97.71675, + "p95Ms": 99.847333, + "peakRssBytes": 220151808, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 2500 + }, + "readBudget": { + "files": 2500, + "bytes": 2582784, + "entries": 0 + } + }, + "warm": { + "samples": 20, + "medianMs": 98.747459, + "p95Ms": 110.613792, + "peakRssBytes": 209747968, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 2500 + }, + "readBudget": { + "files": 2500, + "bytes": 2582784, + "entries": 0 + } + }, + "exactResults": [], + "outputSha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + { + "safety": "unsafe-benchmark-only", + "id": "low-hit", + "cold": { + "samples": 3, + "medianMs": 100.562042, + "p95Ms": 104.410709, + "peakRssBytes": 209747968, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 2500 + }, + "readBudget": { + "files": 2500, + "bytes": 2582784, + "entries": 0 + } + }, + "warm": { + "samples": 20, + "medianMs": 101.261375, + "p95Ms": 103.941958, + "peakRssBytes": 207912960, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 2500 + }, + "readBudget": { + "files": 2500, + "bytes": 2582784, + "entries": 0 + } + }, + "exactResults": [ + "vault/branch-00/branch-00/branch-03/note-00003.md", + "vault/branch-00/branch-02/branch-01/note-00017.md", + "vault/branch-01/branch-04/branch-05/note-00101.md", + "vault/branch-04/branch-06/branch-03/note-00307.md" + ], + "outputSha256": "3426468c469ccf3a134e93a71f6dea6f3a21d3e622918668af97a098bee9b3fd" + }, + { + "safety": "unsafe-benchmark-only", + "id": "high-hit", + "cold": { + "samples": 3, + "medianMs": 104.191416, + "p95Ms": 104.651375, + "peakRssBytes": 207912960, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 2500 + }, + "readBudget": { + "files": 2500, + "bytes": 2582784, + "entries": 0 + } + }, + "warm": { + "samples": 20, + "medianMs": 101.374917, + "p95Ms": 104.824542, + "peakRssBytes": 207912960, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 2500 + }, + "readBudget": { + "files": 2500, + "bytes": 2582784, + "entries": 0 + } + }, + "exactResults": [ + "vault/branch-00/branch-00/branch-00/note-00000.md", + "vault/branch-00/branch-00/branch-01/note-01025.md", + "vault/branch-00/branch-00/branch-02/note-02050.md", + "vault/branch-00/branch-01/branch-05/note-00525.md", + "vault/branch-00/branch-01/branch-06/note-01550.md", + "vault/branch-00/branch-03/branch-01/note-00025.md", + "vault/branch-00/branch-03/branch-02/note-01050.md", + "vault/branch-00/branch-03/branch-03/note-02075.md", + "vault/branch-00/branch-04/branch-06/note-00550.md", + "vault/branch-00/branch-04/branch-07/note-01575.md", + "vault/branch-00/branch-06/branch-02/note-00050.md", + "vault/branch-00/branch-06/branch-03/note-01075.md", + "vault/branch-00/branch-06/branch-04/note-02100.md", + "vault/branch-00/branch-07/branch-07/note-00575.md", + "vault/branch-01/branch-00/branch-00/note-01600.md", + "vault/branch-01/branch-01/branch-03/note-00075.md", + "vault/branch-01/branch-01/branch-04/note-01100.md", + "vault/branch-01/branch-01/branch-05/note-02125.md", + "vault/branch-01/branch-03/branch-00/note-00600.md", + "vault/branch-01/branch-03/branch-01/note-01625.md" + ], + "outputSha256": "cd69f09fe24115d75bb9e0b9d77835b6e944712ea4ac7e87e6275ea09cf4baa4" + } + ], + "rawReadControl": { + "id": "raw-read-control", + "cold": { + "samples": 3, + "medianMs": 34.9845, + "p95Ms": 36.772167, + "peakRssBytes": 195543040, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 2500 + }, + "readBudget": { + "files": 2500, + "bytes": 2582784, + "entries": 0 + } + }, + "warm": { + "samples": 20, + "medianMs": 34.213541, + "p95Ms": 35.409959, + "peakRssBytes": 195592192, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 2500 + }, + "readBudget": { + "files": 2500, + "bytes": 2582784, + "entries": 0 + } + }, + "exactResults": { + "fileCount": 2500, + "totalBytes": 2582784 + }, + "outputSha256": "e0592c7a140a929f31c46afb50699cb72c70caa987d5992b1043b40ce8f2f65a" + }, + "safeCorpusReadControl": { + "id": "safe-corpus-read-control", + "cold": { + "samples": 3, + "medianMs": 293.948917, + "p95Ms": 302.140125, + "peakRssBytes": 195624960, + "operations": { + "lstat": 16695, + "realpath": 1756, + "open": 2500 + }, + "readBudget": { + "bytes": 2582784, + "files": 2500, + "entries": 3084 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 5000, + "realpath": 0, + "open": 2500 + }, + "containmentPaths": { + "lstat": 11695, + "realpath": 1756, + "open": 0 + } + } + }, + "warm": { + "samples": 20, + "medianMs": 288.678875, + "p95Ms": 295.151375, + "peakRssBytes": 194805760, + "operations": { + "lstat": 16695, + "realpath": 1756, + "open": 2500 + }, + "readBudget": { + "bytes": 2582784, + "files": 2500, + "entries": 3084 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 5000, + "realpath": 0, + "open": 2500 + }, + "containmentPaths": { + "lstat": 11695, + "realpath": 1756, + "open": 0 + } + } + }, + "exactResults": { + "fileCount": 2500, + "totalBytes": 2582784 + }, + "outputSha256": "e0592c7a140a929f31c46afb50699cb72c70caa987d5992b1043b40ce8f2f65a" + } +} diff --git a/docs/benchmarks/reports/2026-08-14-public-2500-shallow-prose.report.json b/docs/benchmarks/reports/2026-08-14-public-2500-shallow-prose.report.json new file mode 100644 index 00000000..20e67c18 --- /dev/null +++ b/docs/benchmarks/reports/2026-08-14-public-2500-shallow-prose.report.json @@ -0,0 +1,551 @@ +{ + "schemaVersion": "dotaios-search-benchmark-result/v2", + "benchmarkId": "search-baseline-2026-08-13", + "manifestSha256": "b6c38cb5920f91b0a84c66be8181f6c14f7a1c73360fa4f07993b32a7704d55a", + "inventorySha256": "229da6c47d147660684099dacb22af3362a3467b2e607a6823dd7d8bc582a910", + "selection": { + "fileCount": 2500, + "layout": "shallow", + "distribution": "prose" + }, + "runtime": { + "node": "22.22.3", + "platform": "darwin", + "architecture": "arm64" + }, + "protocol": { + "coldSamples": 3, + "warmupSamples": 3, + "measuredSamples": 20, + "concurrency": 32, + "resultLimit": 20, + "rssPollIntervalMs": 5, + "coldDefinition": "fresh request-scoped reader before harness warm-up; operating-system file cache is uncontrolled", + "warmDefinition": "fresh request-scoped reader after warm-up in the same Node process", + "rawReadControl": { + "enabled": true, + "method": "open-handle-read-all", + "validation": "exact file count and byte total", + "concurrency": 32 + } + }, + "searches": [ + { + "id": "no-hit", + "cold": { + "samples": 3, + "medianMs": 180.427833, + "p95Ms": 197.422042, + "peakRssBytes": 132415488, + "operations": { + "lstat": 7808, + "realpath": 61, + "open": 2500 + }, + "readBudget": { + "bytes": 2731459, + "files": 2500, + "entries": 2502 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 5000, + "realpath": 0, + "open": 2500 + }, + "containmentPaths": { + "lstat": 2808, + "realpath": 61, + "open": 0 + } + } + }, + "warm": { + "samples": 20, + "medianMs": 167.632833, + "p95Ms": 173.252208, + "peakRssBytes": 168296448, + "operations": { + "lstat": 7808, + "realpath": 61, + "open": 2500 + }, + "readBudget": { + "bytes": 2731459, + "files": 2500, + "entries": 2502 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 5000, + "realpath": 0, + "open": 2500 + }, + "containmentPaths": { + "lstat": 2808, + "realpath": 61, + "open": 0 + } + } + }, + "surface": { + "entryPoint": "searchAios", + "requestedScope": "all", + "completeness": "complete", + "omissions": [], + "returnedScopes": [ + "sessions", + "context", + "memory", + "vault", + "decisions", + "skills", + "references", + "plugins" + ] + }, + "exactResults": [], + "outputSha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + { + "id": "low-hit", + "cold": { + "samples": 3, + "medianMs": 170.919708, + "p95Ms": 174.008625, + "peakRssBytes": 168378368, + "operations": { + "lstat": 7808, + "realpath": 61, + "open": 2500 + }, + "readBudget": { + "bytes": 2731459, + "files": 2500, + "entries": 2502 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 5000, + "realpath": 0, + "open": 2500 + }, + "containmentPaths": { + "lstat": 2808, + "realpath": 61, + "open": 0 + } + } + }, + "warm": { + "samples": 20, + "medianMs": 170.256625, + "p95Ms": 176.336459, + "peakRssBytes": 183549952, + "operations": { + "lstat": 7808, + "realpath": 61, + "open": 2500 + }, + "readBudget": { + "bytes": 2731459, + "files": 2500, + "entries": 2502 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 5000, + "realpath": 0, + "open": 2500 + }, + "containmentPaths": { + "lstat": 2808, + "realpath": 61, + "open": 0 + } + } + }, + "surface": { + "entryPoint": "searchAios", + "requestedScope": "all", + "completeness": "complete", + "omissions": [], + "returnedScopes": [ + "sessions", + "context", + "memory", + "vault", + "decisions", + "skills", + "references", + "plugins" + ] + }, + "exactResults": [ + "vault/bucket-01/note-00003.md", + "vault/bucket-01/note-00017.md", + "vault/bucket-01/note-00101.md", + "vault/bucket-01/note-00307.md" + ], + "outputSha256": "6bc96a7e535089d0f1232a6aa21cf6cf2b255f9d1ddccfe2dea5b098ff0c98b5" + }, + { + "id": "high-hit", + "cold": { + "samples": 3, + "medianMs": 169.711584, + "p95Ms": 172.344458, + "peakRssBytes": 183697408, + "operations": { + "lstat": 7808, + "realpath": 61, + "open": 2500 + }, + "readBudget": { + "bytes": 2731459, + "files": 2500, + "entries": 2502 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 5000, + "realpath": 0, + "open": 2500 + }, + "containmentPaths": { + "lstat": 2808, + "realpath": 61, + "open": 0 + } + } + }, + "warm": { + "samples": 20, + "medianMs": 170.518916, + "p95Ms": 177.217417, + "peakRssBytes": 184647680, + "operations": { + "lstat": 7808, + "realpath": 61, + "open": 2500 + }, + "readBudget": { + "bytes": 2731459, + "files": 2500, + "entries": 2502 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 5000, + "realpath": 0, + "open": 2500 + }, + "containmentPaths": { + "lstat": 2808, + "realpath": 61, + "open": 0 + } + } + }, + "surface": { + "entryPoint": "searchAios", + "requestedScope": "all", + "completeness": "complete", + "omissions": [], + "returnedScopes": [ + "sessions", + "context", + "memory", + "vault", + "decisions", + "skills", + "references", + "plugins" + ] + }, + "exactResults": [ + "vault/bucket-00/note-00000.md", + "vault/bucket-00/note-00050.md", + "vault/bucket-00/note-00100.md", + "vault/bucket-00/note-00150.md", + "vault/bucket-00/note-00200.md", + "vault/bucket-00/note-00250.md", + "vault/bucket-00/note-00300.md", + "vault/bucket-00/note-00350.md", + "vault/bucket-00/note-00400.md", + "vault/bucket-00/note-00450.md", + "vault/bucket-00/note-00500.md", + "vault/bucket-00/note-00550.md", + "vault/bucket-00/note-00600.md", + "vault/bucket-00/note-00650.md", + "vault/bucket-00/note-00700.md", + "vault/bucket-00/note-00750.md", + "vault/bucket-00/note-00800.md", + "vault/bucket-00/note-00850.md", + "vault/bucket-00/note-00900.md", + "vault/bucket-00/note-00950.md" + ], + "outputSha256": "9b3d7e484ca325f107d3363721ce8b99b038149f4b79a4f23c25e52c393bfe3f" + } + ], + "searchSurface": { + "entryPoint": "searchAios", + "requestedScope": "all", + "completeness": "complete" + }, + "operationGate": { + "comparison": "safe-corpus-read-control-plus-fixed-directory-allowance", + "allowance": { + "lstat": 512, + "realpath": 256, + "open": 16 + }, + "passed": true + }, + "rawSearchControl": [ + { + "safety": "unsafe-benchmark-only", + "id": "no-hit", + "cold": { + "samples": 3, + "medianMs": 91.786084, + "p95Ms": 93.3035, + "peakRssBytes": 184942592, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 2500 + }, + "readBudget": { + "files": 2500, + "bytes": 2731459, + "entries": 0 + } + }, + "warm": { + "samples": 20, + "medianMs": 90.796709, + "p95Ms": 94.046292, + "peakRssBytes": 185073664, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 2500 + }, + "readBudget": { + "files": 2500, + "bytes": 2731459, + "entries": 0 + } + }, + "exactResults": [], + "outputSha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + { + "safety": "unsafe-benchmark-only", + "id": "low-hit", + "cold": { + "samples": 3, + "medianMs": 93.508625, + "p95Ms": 93.603708, + "peakRssBytes": 185073664, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 2500 + }, + "readBudget": { + "files": 2500, + "bytes": 2731459, + "entries": 0 + } + }, + "warm": { + "samples": 20, + "medianMs": 93.798041, + "p95Ms": 95.80025, + "peakRssBytes": 185106432, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 2500 + }, + "readBudget": { + "files": 2500, + "bytes": 2731459, + "entries": 0 + } + }, + "exactResults": [ + "vault/bucket-01/note-00003.md", + "vault/bucket-01/note-00017.md", + "vault/bucket-01/note-00101.md", + "vault/bucket-01/note-00307.md" + ], + "outputSha256": "6bc96a7e535089d0f1232a6aa21cf6cf2b255f9d1ddccfe2dea5b098ff0c98b5" + }, + { + "safety": "unsafe-benchmark-only", + "id": "high-hit", + "cold": { + "samples": 3, + "medianMs": 94.659209, + "p95Ms": 95.07925, + "peakRssBytes": 185106432, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 2500 + }, + "readBudget": { + "files": 2500, + "bytes": 2731459, + "entries": 0 + } + }, + "warm": { + "samples": 20, + "medianMs": 93.884042, + "p95Ms": 96.434916, + "peakRssBytes": 185106432, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 2500 + }, + "readBudget": { + "files": 2500, + "bytes": 2731459, + "entries": 0 + } + }, + "exactResults": [ + "vault/bucket-00/note-00000.md", + "vault/bucket-00/note-00050.md", + "vault/bucket-00/note-00100.md", + "vault/bucket-00/note-00150.md", + "vault/bucket-00/note-00200.md", + "vault/bucket-00/note-00250.md", + "vault/bucket-00/note-00300.md", + "vault/bucket-00/note-00350.md", + "vault/bucket-00/note-00400.md", + "vault/bucket-00/note-00450.md", + "vault/bucket-00/note-00500.md", + "vault/bucket-00/note-00550.md", + "vault/bucket-00/note-00600.md", + "vault/bucket-00/note-00650.md", + "vault/bucket-00/note-00700.md", + "vault/bucket-00/note-00750.md", + "vault/bucket-00/note-00800.md", + "vault/bucket-00/note-00850.md", + "vault/bucket-00/note-00900.md", + "vault/bucket-00/note-00950.md" + ], + "outputSha256": "9b3d7e484ca325f107d3363721ce8b99b038149f4b79a4f23c25e52c393bfe3f" + } + ], + "rawReadControl": { + "id": "raw-read-control", + "cold": { + "samples": 3, + "medianMs": 33.250834, + "p95Ms": 33.851834, + "peakRssBytes": 187039744, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 2500 + }, + "readBudget": { + "files": 2500, + "bytes": 2731459, + "entries": 0 + } + }, + "warm": { + "samples": 20, + "medianMs": 33.048834, + "p95Ms": 36.112791, + "peakRssBytes": 187219968, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 2500 + }, + "readBudget": { + "files": 2500, + "bytes": 2731459, + "entries": 0 + } + }, + "exactResults": { + "fileCount": 2500, + "totalBytes": 2731459 + }, + "outputSha256": "20cf6592fa8dfbeffac3b056d6e8189b44e7df62cc29b654280e12a1268287fe" + }, + "safeCorpusReadControl": { + "id": "safe-corpus-read-control", + "cold": { + "samples": 3, + "medianMs": 128.075916, + "p95Ms": 128.155333, + "peakRssBytes": 187842560, + "operations": { + "lstat": 7535, + "realpath": 10, + "open": 2500 + }, + "readBudget": { + "bytes": 2731459, + "files": 2500, + "entries": 2502 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 5000, + "realpath": 0, + "open": 2500 + }, + "containmentPaths": { + "lstat": 2535, + "realpath": 10, + "open": 0 + } + } + }, + "warm": { + "samples": 20, + "medianMs": 122.58475, + "p95Ms": 129.377708, + "peakRssBytes": 187875328, + "operations": { + "lstat": 7535, + "realpath": 10, + "open": 2500 + }, + "readBudget": { + "bytes": 2731459, + "files": 2500, + "entries": 2502 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 5000, + "realpath": 0, + "open": 2500 + }, + "containmentPaths": { + "lstat": 2535, + "realpath": 10, + "open": 0 + } + } + }, + "exactResults": { + "fileCount": 2500, + "totalBytes": 2731459 + }, + "outputSha256": "20cf6592fa8dfbeffac3b056d6e8189b44e7df62cc29b654280e12a1268287fe" + } +} diff --git a/docs/benchmarks/reports/2026-08-14-public-500-nested-high-entropy.report.json b/docs/benchmarks/reports/2026-08-14-public-500-nested-high-entropy.report.json new file mode 100644 index 00000000..7534c119 --- /dev/null +++ b/docs/benchmarks/reports/2026-08-14-public-500-nested-high-entropy.report.json @@ -0,0 +1,551 @@ +{ + "schemaVersion": "dotaios-search-benchmark-result/v2", + "benchmarkId": "search-baseline-2026-08-13", + "manifestSha256": "b6c38cb5920f91b0a84c66be8181f6c14f7a1c73360fa4f07993b32a7704d55a", + "inventorySha256": "16d7f2dc55c48230a2a65d71e93453245cd4bfd6b562701cd5c5cc95c3a5fe19", + "selection": { + "fileCount": 500, + "layout": "nested", + "distribution": "high-entropy" + }, + "runtime": { + "node": "22.22.3", + "platform": "darwin", + "architecture": "arm64" + }, + "protocol": { + "coldSamples": 3, + "warmupSamples": 3, + "measuredSamples": 20, + "concurrency": 32, + "resultLimit": 20, + "rssPollIntervalMs": 5, + "coldDefinition": "fresh request-scoped reader before harness warm-up; operating-system file cache is uncontrolled", + "warmDefinition": "fresh request-scoped reader after warm-up in the same Node process", + "rawReadControl": { + "enabled": true, + "method": "open-handle-read-all", + "validation": "exact file count and byte total", + "concurrency": 32 + } + }, + "searches": [ + { + "id": "no-hit", + "cold": { + "samples": 3, + "medianMs": 201.30925, + "p95Ms": 226.117875, + "peakRssBytes": 121225216, + "operations": { + "lstat": 10762, + "realpath": 1768, + "open": 500 + }, + "readBudget": { + "bytes": 519370, + "files": 500, + "entries": 1071 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 1000, + "realpath": 0, + "open": 500 + }, + "containmentPaths": { + "lstat": 9762, + "realpath": 1768, + "open": 0 + } + } + }, + "warm": { + "samples": 20, + "medianMs": 188.782958, + "p95Ms": 192.876916, + "peakRssBytes": 158679040, + "operations": { + "lstat": 10762, + "realpath": 1768, + "open": 500 + }, + "readBudget": { + "bytes": 519370, + "files": 500, + "entries": 1071 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 1000, + "realpath": 0, + "open": 500 + }, + "containmentPaths": { + "lstat": 9762, + "realpath": 1768, + "open": 0 + } + } + }, + "surface": { + "entryPoint": "searchAios", + "requestedScope": "all", + "completeness": "complete", + "omissions": [], + "returnedScopes": [ + "sessions", + "context", + "memory", + "vault", + "decisions", + "skills", + "references", + "plugins" + ] + }, + "exactResults": [], + "outputSha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + { + "id": "low-hit", + "cold": { + "samples": 3, + "medianMs": 190.127, + "p95Ms": 193.552791, + "peakRssBytes": 157483008, + "operations": { + "lstat": 10762, + "realpath": 1768, + "open": 500 + }, + "readBudget": { + "bytes": 519370, + "files": 500, + "entries": 1071 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 1000, + "realpath": 0, + "open": 500 + }, + "containmentPaths": { + "lstat": 9762, + "realpath": 1768, + "open": 0 + } + } + }, + "warm": { + "samples": 20, + "medianMs": 190.423459, + "p95Ms": 195.383084, + "peakRssBytes": 157663232, + "operations": { + "lstat": 10762, + "realpath": 1768, + "open": 500 + }, + "readBudget": { + "bytes": 519370, + "files": 500, + "entries": 1071 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 1000, + "realpath": 0, + "open": 500 + }, + "containmentPaths": { + "lstat": 9762, + "realpath": 1768, + "open": 0 + } + } + }, + "surface": { + "entryPoint": "searchAios", + "requestedScope": "all", + "completeness": "complete", + "omissions": [], + "returnedScopes": [ + "sessions", + "context", + "memory", + "vault", + "decisions", + "skills", + "references", + "plugins" + ] + }, + "exactResults": [ + "vault/branch-00/branch-00/branch-03/note-00003.md", + "vault/branch-00/branch-02/branch-01/note-00017.md", + "vault/branch-01/branch-04/branch-05/note-00101.md", + "vault/branch-04/branch-06/branch-03/note-00307.md" + ], + "outputSha256": "3426468c469ccf3a134e93a71f6dea6f3a21d3e622918668af97a098bee9b3fd" + }, + { + "id": "high-hit", + "cold": { + "samples": 3, + "medianMs": 188.825834, + "p95Ms": 192.656125, + "peakRssBytes": 157712384, + "operations": { + "lstat": 10762, + "realpath": 1768, + "open": 500 + }, + "readBudget": { + "bytes": 519370, + "files": 500, + "entries": 1071 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 1000, + "realpath": 0, + "open": 500 + }, + "containmentPaths": { + "lstat": 9762, + "realpath": 1768, + "open": 0 + } + } + }, + "warm": { + "samples": 20, + "medianMs": 189.731375, + "p95Ms": 192.26425, + "peakRssBytes": 161398784, + "operations": { + "lstat": 10762, + "realpath": 1768, + "open": 500 + }, + "readBudget": { + "bytes": 519370, + "files": 500, + "entries": 1071 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 1000, + "realpath": 0, + "open": 500 + }, + "containmentPaths": { + "lstat": 9762, + "realpath": 1768, + "open": 0 + } + } + }, + "surface": { + "entryPoint": "searchAios", + "requestedScope": "all", + "completeness": "complete", + "omissions": [], + "returnedScopes": [ + "sessions", + "context", + "memory", + "vault", + "decisions", + "skills", + "references", + "plugins" + ] + }, + "exactResults": [ + "vault/branch-00/branch-00/branch-00/note-00000.md", + "vault/branch-00/branch-03/branch-01/note-00025.md", + "vault/branch-00/branch-06/branch-02/note-00050.md", + "vault/branch-01/branch-01/branch-03/note-00075.md", + "vault/branch-01/branch-04/branch-04/note-00100.md", + "vault/branch-01/branch-07/branch-05/note-00125.md", + "vault/branch-02/branch-02/branch-06/note-00150.md", + "vault/branch-02/branch-05/branch-07/note-00175.md", + "vault/branch-03/branch-01/branch-00/note-00200.md", + "vault/branch-03/branch-04/branch-01/note-00225.md", + "vault/branch-03/branch-07/branch-02/note-00250.md", + "vault/branch-04/branch-02/branch-03/note-00275.md", + "vault/branch-04/branch-05/branch-04/note-00300.md", + "vault/branch-05/branch-00/branch-05/note-00325.md", + "vault/branch-05/branch-03/branch-06/note-00350.md", + "vault/branch-05/branch-06/branch-07/note-00375.md", + "vault/branch-06/branch-02/branch-00/note-00400.md", + "vault/branch-06/branch-05/branch-01/note-00425.md", + "vault/branch-07/branch-00/branch-02/note-00450.md", + "vault/branch-07/branch-03/branch-03/note-00475.md" + ], + "outputSha256": "7f89b11c4f05e3bdb46a359a0cc8f9df6f1f0ba7b5c91b591213096cac7a1c15" + } + ], + "searchSurface": { + "entryPoint": "searchAios", + "requestedScope": "all", + "completeness": "complete" + }, + "operationGate": { + "comparison": "safe-corpus-read-control-plus-fixed-directory-allowance", + "allowance": { + "lstat": 512, + "realpath": 256, + "open": 16 + }, + "passed": true + }, + "rawSearchControl": [ + { + "safety": "unsafe-benchmark-only", + "id": "no-hit", + "cold": { + "samples": 3, + "medianMs": 18.959125, + "p95Ms": 19.222833, + "peakRssBytes": 161808384, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 500 + }, + "readBudget": { + "files": 500, + "bytes": 519370, + "entries": 0 + } + }, + "warm": { + "samples": 20, + "medianMs": 18.353625, + "p95Ms": 19.611333, + "peakRssBytes": 166363136, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 500 + }, + "readBudget": { + "files": 500, + "bytes": 519370, + "entries": 0 + } + }, + "exactResults": [], + "outputSha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + { + "safety": "unsafe-benchmark-only", + "id": "low-hit", + "cold": { + "samples": 3, + "medianMs": 19.7365, + "p95Ms": 21.276875, + "peakRssBytes": 166445056, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 500 + }, + "readBudget": { + "files": 500, + "bytes": 519370, + "entries": 0 + } + }, + "warm": { + "samples": 20, + "medianMs": 19.326208, + "p95Ms": 20.195584, + "peakRssBytes": 166461440, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 500 + }, + "readBudget": { + "files": 500, + "bytes": 519370, + "entries": 0 + } + }, + "exactResults": [ + "vault/branch-00/branch-00/branch-03/note-00003.md", + "vault/branch-00/branch-02/branch-01/note-00017.md", + "vault/branch-01/branch-04/branch-05/note-00101.md", + "vault/branch-04/branch-06/branch-03/note-00307.md" + ], + "outputSha256": "3426468c469ccf3a134e93a71f6dea6f3a21d3e622918668af97a098bee9b3fd" + }, + { + "safety": "unsafe-benchmark-only", + "id": "high-hit", + "cold": { + "samples": 3, + "medianMs": 19.131083, + "p95Ms": 20.426583, + "peakRssBytes": 166461440, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 500 + }, + "readBudget": { + "files": 500, + "bytes": 519370, + "entries": 0 + } + }, + "warm": { + "samples": 20, + "medianMs": 19.424334, + "p95Ms": 20.958125, + "peakRssBytes": 167411712, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 500 + }, + "readBudget": { + "files": 500, + "bytes": 519370, + "entries": 0 + } + }, + "exactResults": [ + "vault/branch-00/branch-00/branch-00/note-00000.md", + "vault/branch-00/branch-03/branch-01/note-00025.md", + "vault/branch-00/branch-06/branch-02/note-00050.md", + "vault/branch-01/branch-01/branch-03/note-00075.md", + "vault/branch-01/branch-04/branch-04/note-00100.md", + "vault/branch-01/branch-07/branch-05/note-00125.md", + "vault/branch-02/branch-02/branch-06/note-00150.md", + "vault/branch-02/branch-05/branch-07/note-00175.md", + "vault/branch-03/branch-01/branch-00/note-00200.md", + "vault/branch-03/branch-04/branch-01/note-00225.md", + "vault/branch-03/branch-07/branch-02/note-00250.md", + "vault/branch-04/branch-02/branch-03/note-00275.md", + "vault/branch-04/branch-05/branch-04/note-00300.md", + "vault/branch-05/branch-00/branch-05/note-00325.md", + "vault/branch-05/branch-03/branch-06/note-00350.md", + "vault/branch-05/branch-06/branch-07/note-00375.md", + "vault/branch-06/branch-02/branch-00/note-00400.md", + "vault/branch-06/branch-05/branch-01/note-00425.md", + "vault/branch-07/branch-00/branch-02/note-00450.md", + "vault/branch-07/branch-03/branch-03/note-00475.md" + ], + "outputSha256": "7f89b11c4f05e3bdb46a359a0cc8f9df6f1f0ba7b5c91b591213096cac7a1c15" + } + ], + "rawReadControl": { + "id": "raw-read-control", + "cold": { + "samples": 3, + "medianMs": 7.479334, + "p95Ms": 8.031083, + "peakRssBytes": 160890880, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 500 + }, + "readBudget": { + "files": 500, + "bytes": 519370, + "entries": 0 + } + }, + "warm": { + "samples": 20, + "medianMs": 6.747958, + "p95Ms": 7.192625, + "peakRssBytes": 161415168, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 500 + }, + "readBudget": { + "files": 500, + "bytes": 519370, + "entries": 0 + } + }, + "exactResults": { + "fileCount": 500, + "totalBytes": 519370 + }, + "outputSha256": "77e3eb3a648893817a0489c4f1bb50a90602b142dc285e9c9b856f710b40dfc6" + }, + "safeCorpusReadControl": { + "id": "safe-corpus-read-control", + "cold": { + "samples": 3, + "medianMs": 177.642875, + "p95Ms": 179.252417, + "peakRssBytes": 161480704, + "operations": { + "lstat": 10489, + "realpath": 1717, + "open": 500 + }, + "readBudget": { + "bytes": 519370, + "files": 500, + "entries": 1071 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 1000, + "realpath": 0, + "open": 500 + }, + "containmentPaths": { + "lstat": 9489, + "realpath": 1717, + "open": 0 + } + } + }, + "warm": { + "samples": 20, + "medianMs": 176.116, + "p95Ms": 180.171417, + "peakRssBytes": 160677888, + "operations": { + "lstat": 10489, + "realpath": 1717, + "open": 500 + }, + "readBudget": { + "bytes": 519370, + "files": 500, + "entries": 1071 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 1000, + "realpath": 0, + "open": 500 + }, + "containmentPaths": { + "lstat": 9489, + "realpath": 1717, + "open": 0 + } + } + }, + "exactResults": { + "fileCount": 500, + "totalBytes": 519370 + }, + "outputSha256": "77e3eb3a648893817a0489c4f1bb50a90602b142dc285e9c9b856f710b40dfc6" + } +} diff --git a/docs/benchmarks/reports/2026-08-14-public-500-shallow-prose.report.json b/docs/benchmarks/reports/2026-08-14-public-500-shallow-prose.report.json new file mode 100644 index 00000000..b1dd7318 --- /dev/null +++ b/docs/benchmarks/reports/2026-08-14-public-500-shallow-prose.report.json @@ -0,0 +1,551 @@ +{ + "schemaVersion": "dotaios-search-benchmark-result/v2", + "benchmarkId": "search-baseline-2026-08-13", + "manifestSha256": "b6c38cb5920f91b0a84c66be8181f6c14f7a1c73360fa4f07993b32a7704d55a", + "inventorySha256": "db6ab0454118d7cd1e2c54c3519db2018fddcba75e124105f13f1345dc525306", + "selection": { + "fileCount": 500, + "layout": "shallow", + "distribution": "prose" + }, + "runtime": { + "node": "22.22.3", + "platform": "darwin", + "architecture": "arm64" + }, + "protocol": { + "coldSamples": 3, + "warmupSamples": 3, + "measuredSamples": 20, + "concurrency": 32, + "resultLimit": 20, + "rssPollIntervalMs": 5, + "coldDefinition": "fresh request-scoped reader before harness warm-up; operating-system file cache is uncontrolled", + "warmDefinition": "fresh request-scoped reader after warm-up in the same Node process", + "rawReadControl": { + "enabled": true, + "method": "open-handle-read-all", + "validation": "exact file count and byte total", + "concurrency": 32 + } + }, + "searches": [ + { + "id": "no-hit", + "cold": { + "samples": 3, + "medianMs": 41.954917, + "p95Ms": 56.607125, + "peakRssBytes": 103661568, + "operations": { + "lstat": 1808, + "realpath": 61, + "open": 500 + }, + "readBudget": { + "bytes": 543978, + "files": 500, + "entries": 502 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 1000, + "realpath": 0, + "open": 500 + }, + "containmentPaths": { + "lstat": 808, + "realpath": 61, + "open": 0 + } + } + }, + "warm": { + "samples": 20, + "medianMs": 37.580459, + "p95Ms": 39.961917, + "peakRssBytes": 140263424, + "operations": { + "lstat": 1808, + "realpath": 61, + "open": 500 + }, + "readBudget": { + "bytes": 543978, + "files": 500, + "entries": 502 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 1000, + "realpath": 0, + "open": 500 + }, + "containmentPaths": { + "lstat": 808, + "realpath": 61, + "open": 0 + } + } + }, + "surface": { + "entryPoint": "searchAios", + "requestedScope": "all", + "completeness": "complete", + "omissions": [], + "returnedScopes": [ + "sessions", + "context", + "memory", + "vault", + "decisions", + "skills", + "references", + "plugins" + ] + }, + "exactResults": [], + "outputSha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + { + "id": "low-hit", + "cold": { + "samples": 3, + "medianMs": 41.529708, + "p95Ms": 42.262333, + "peakRssBytes": 140361728, + "operations": { + "lstat": 1808, + "realpath": 61, + "open": 500 + }, + "readBudget": { + "bytes": 543978, + "files": 500, + "entries": 502 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 1000, + "realpath": 0, + "open": 500 + }, + "containmentPaths": { + "lstat": 808, + "realpath": 61, + "open": 0 + } + } + }, + "warm": { + "samples": 20, + "medianMs": 37.4265, + "p95Ms": 39.506541, + "peakRssBytes": 152764416, + "operations": { + "lstat": 1808, + "realpath": 61, + "open": 500 + }, + "readBudget": { + "bytes": 543978, + "files": 500, + "entries": 502 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 1000, + "realpath": 0, + "open": 500 + }, + "containmentPaths": { + "lstat": 808, + "realpath": 61, + "open": 0 + } + } + }, + "surface": { + "entryPoint": "searchAios", + "requestedScope": "all", + "completeness": "complete", + "omissions": [], + "returnedScopes": [ + "sessions", + "context", + "memory", + "vault", + "decisions", + "skills", + "references", + "plugins" + ] + }, + "exactResults": [ + "vault/bucket-01/note-00003.md", + "vault/bucket-01/note-00017.md", + "vault/bucket-01/note-00101.md", + "vault/bucket-01/note-00307.md" + ], + "outputSha256": "6bc96a7e535089d0f1232a6aa21cf6cf2b255f9d1ddccfe2dea5b098ff0c98b5" + }, + { + "id": "high-hit", + "cold": { + "samples": 3, + "medianMs": 36.339, + "p95Ms": 37.793458, + "peakRssBytes": 160481280, + "operations": { + "lstat": 1808, + "realpath": 61, + "open": 500 + }, + "readBudget": { + "bytes": 543978, + "files": 500, + "entries": 502 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 1000, + "realpath": 0, + "open": 500 + }, + "containmentPaths": { + "lstat": 808, + "realpath": 61, + "open": 0 + } + } + }, + "warm": { + "samples": 20, + "medianMs": 36.827375, + "p95Ms": 38.880917, + "peakRssBytes": 166150144, + "operations": { + "lstat": 1808, + "realpath": 61, + "open": 500 + }, + "readBudget": { + "bytes": 543978, + "files": 500, + "entries": 502 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 1000, + "realpath": 0, + "open": 500 + }, + "containmentPaths": { + "lstat": 808, + "realpath": 61, + "open": 0 + } + } + }, + "surface": { + "entryPoint": "searchAios", + "requestedScope": "all", + "completeness": "complete", + "omissions": [], + "returnedScopes": [ + "sessions", + "context", + "memory", + "vault", + "decisions", + "skills", + "references", + "plugins" + ] + }, + "exactResults": [ + "vault/bucket-00/note-00000.md", + "vault/bucket-00/note-00050.md", + "vault/bucket-00/note-00100.md", + "vault/bucket-00/note-00150.md", + "vault/bucket-00/note-00200.md", + "vault/bucket-00/note-00250.md", + "vault/bucket-00/note-00300.md", + "vault/bucket-00/note-00350.md", + "vault/bucket-00/note-00400.md", + "vault/bucket-00/note-00450.md", + "vault/bucket-01/note-00025.md", + "vault/bucket-01/note-00075.md", + "vault/bucket-01/note-00125.md", + "vault/bucket-01/note-00175.md", + "vault/bucket-01/note-00225.md", + "vault/bucket-01/note-00275.md", + "vault/bucket-01/note-00325.md", + "vault/bucket-01/note-00375.md", + "vault/bucket-01/note-00425.md", + "vault/bucket-01/note-00475.md" + ], + "outputSha256": "0c51d10ccea2f49c756db6a66d988559b5db90ccb4843048ebc80b01444a5fca" + } + ], + "searchSurface": { + "entryPoint": "searchAios", + "requestedScope": "all", + "completeness": "complete" + }, + "operationGate": { + "comparison": "safe-corpus-read-control-plus-fixed-directory-allowance", + "allowance": { + "lstat": 512, + "realpath": 256, + "open": 16 + }, + "passed": true + }, + "rawSearchControl": [ + { + "safety": "unsafe-benchmark-only", + "id": "no-hit", + "cold": { + "samples": 3, + "medianMs": 19.356458, + "p95Ms": 19.698542, + "peakRssBytes": 166543360, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 500 + }, + "readBudget": { + "files": 500, + "bytes": 543978, + "entries": 0 + } + }, + "warm": { + "samples": 20, + "medianMs": 18.100125, + "p95Ms": 19.241209, + "peakRssBytes": 166576128, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 500 + }, + "readBudget": { + "files": 500, + "bytes": 543978, + "entries": 0 + } + }, + "exactResults": [], + "outputSha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + { + "safety": "unsafe-benchmark-only", + "id": "low-hit", + "cold": { + "samples": 3, + "medianMs": 18.739209, + "p95Ms": 18.903583, + "peakRssBytes": 166248448, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 500 + }, + "readBudget": { + "files": 500, + "bytes": 543978, + "entries": 0 + } + }, + "warm": { + "samples": 20, + "medianMs": 18.529417, + "p95Ms": 19.344583, + "peakRssBytes": 166297600, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 500 + }, + "readBudget": { + "files": 500, + "bytes": 543978, + "entries": 0 + } + }, + "exactResults": [ + "vault/bucket-01/note-00003.md", + "vault/bucket-01/note-00017.md", + "vault/bucket-01/note-00101.md", + "vault/bucket-01/note-00307.md" + ], + "outputSha256": "6bc96a7e535089d0f1232a6aa21cf6cf2b255f9d1ddccfe2dea5b098ff0c98b5" + }, + { + "safety": "unsafe-benchmark-only", + "id": "high-hit", + "cold": { + "samples": 3, + "medianMs": 19.964667, + "p95Ms": 21.927, + "peakRssBytes": 166297600, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 500 + }, + "readBudget": { + "files": 500, + "bytes": 543978, + "entries": 0 + } + }, + "warm": { + "samples": 20, + "medianMs": 18.67025, + "p95Ms": 19.015125, + "peakRssBytes": 166313984, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 500 + }, + "readBudget": { + "files": 500, + "bytes": 543978, + "entries": 0 + } + }, + "exactResults": [ + "vault/bucket-00/note-00000.md", + "vault/bucket-00/note-00050.md", + "vault/bucket-00/note-00100.md", + "vault/bucket-00/note-00150.md", + "vault/bucket-00/note-00200.md", + "vault/bucket-00/note-00250.md", + "vault/bucket-00/note-00300.md", + "vault/bucket-00/note-00350.md", + "vault/bucket-00/note-00400.md", + "vault/bucket-00/note-00450.md", + "vault/bucket-01/note-00025.md", + "vault/bucket-01/note-00075.md", + "vault/bucket-01/note-00125.md", + "vault/bucket-01/note-00175.md", + "vault/bucket-01/note-00225.md", + "vault/bucket-01/note-00275.md", + "vault/bucket-01/note-00325.md", + "vault/bucket-01/note-00375.md", + "vault/bucket-01/note-00425.md", + "vault/bucket-01/note-00475.md" + ], + "outputSha256": "0c51d10ccea2f49c756db6a66d988559b5db90ccb4843048ebc80b01444a5fca" + } + ], + "rawReadControl": { + "id": "raw-read-control", + "cold": { + "samples": 3, + "medianMs": 7.032792, + "p95Ms": 7.05825, + "peakRssBytes": 168591360, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 500 + }, + "readBudget": { + "files": 500, + "bytes": 543978, + "entries": 0 + } + }, + "warm": { + "samples": 20, + "medianMs": 6.852375, + "p95Ms": 7.391292, + "peakRssBytes": 168755200, + "operations": { + "lstat": 0, + "realpath": 0, + "open": 500 + }, + "readBudget": { + "files": 500, + "bytes": 543978, + "entries": 0 + } + }, + "exactResults": { + "fileCount": 500, + "totalBytes": 543978 + }, + "outputSha256": "0a60b34e49b0085aac50f5362f1cb3b84efd580fdf3e5b53b05934b8f57a962a" + }, + "safeCorpusReadControl": { + "id": "safe-corpus-read-control", + "cold": { + "samples": 3, + "medianMs": 26.377541, + "p95Ms": 27.251791, + "peakRssBytes": 168869888, + "operations": { + "lstat": 1535, + "realpath": 10, + "open": 500 + }, + "readBudget": { + "bytes": 543978, + "files": 500, + "entries": 502 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 1000, + "realpath": 0, + "open": 500 + }, + "containmentPaths": { + "lstat": 535, + "realpath": 10, + "open": 0 + } + } + }, + "warm": { + "samples": 20, + "medianMs": 24.574542, + "p95Ms": 25.837458, + "peakRssBytes": 169000960, + "operations": { + "lstat": 1535, + "realpath": 10, + "open": 500 + }, + "readBudget": { + "bytes": 543978, + "files": 500, + "entries": 502 + }, + "operationBreakdown": { + "acceptedFilePaths": { + "lstat": 1000, + "realpath": 0, + "open": 500 + }, + "containmentPaths": { + "lstat": 535, + "realpath": 10, + "open": 0 + } + } + }, + "exactResults": { + "fileCount": 500, + "totalBytes": 543978 + }, + "outputSha256": "0a60b34e49b0085aac50f5362f1cb3b84efd580fdf3e5b53b05934b8f57a962a" + } +} diff --git a/docs/mcp.md b/docs/mcp.md index e81f3cfb..4d7de6c0 100644 --- a/docs/mcp.md +++ b/docs/mcp.md @@ -33,9 +33,9 @@ The adapter exposes exactly these three read-only tool names: - `read_working_context`: return the same bounded, project-filtered projection as `dotaios brief --compact`; accepts optional `project`, session `limit`, and character `budget` - `search_aios`: search bounded local results by `query`, with optional `scope`, - canonical project `project`, result `limit`, and character `budget`; project-only - scope requires the selector, while all-scope search without it omits projects - and reports that omission + canonical project `project`, result `limit`, and a complete-response character + `budget` from 3,530 to 32,000 (default 6,000); project-only scope requires the + selector, while all-scope search without it omits projects as selection metadata - `resolve_skill`: match an `intent` to installed workflows, with an optional result `limit` and character `budget` from 256 to 32,000 (default 6,000); the complete serialized response, including budget metadata, stays within it @@ -45,6 +45,42 @@ External project-source retrieval and finite consent remain CLI-only because they publish machine-local receipts and require the same user's explicit shell apply. Task text and MCP calls cannot approve a grant. +`search_aios` has a higher budget floor than the other two tools because its +smallest honest incomplete response must retain the full omission objects. The +3,530-character floor is mechanically derived from the closed field bounds, +the maximum 200-code-point project selector metadata, and all nine logical +scopes the current tool can select at once; a public MCP fixture exercises that +maximum set at the exact floor. A budget from 256 through 3,529 is therefore a +specific input error for `search_aios`, while `read_working_context` and +`resolve_skill` continue to accept 256. Callers that do not need a custom limit +should omit `budget` and use the 6,000-character default. + +This is an explicit compatibility correction: the full named omission schema +is retained instead of adding a second compact encoding whose recovery meaning +would depend on an out-of-band decoder. Clients that previously sent a +`search_aios` budget below 3,530 must raise it or omit it. A future incompatible +omission encoding requires a separately versioned contract rather than a silent +shape switch. + +`search_aios` returns `complete: true` only when every selected logical scope +was inspected. A skippable resource ceiling returns valid results from admitted +scopes, `complete: false`, and the same logical omission records used by core and +the CLI; it remains a successful tool call with `isError: false`. Each omission +contains only `scope`, a closed `reason`, bounded `observed` counts, +`inspection` (`not_searched` or `partially_enumerated`), and a path-free +`recovery` code/message. The five ceiling reasons are `file_too_large`, +`directory_entries_exceeded`, `aggregate_bytes_exceeded`, +`file_count_exceeded`, and `entry_count_exceeded`. At most 32 omissions plus one +defensive `omissions_truncated` remainder are returned by the shared collector; +the current MCP scope allowlist can produce at most nine omissions in one call. +Linked or non-regular +evidence, unsafe paths, unauthorized roots, invalid UTF-8 or configuration, +observed mutation, and unexpected I/O remain failed tool calls. + +The response `budget.truncated` flag describes only result transport truncation; +it never changes corpus `complete` or removes omission metadata. A complete +zero-hit search is `complete: true` with an empty `results` array. + For `read_working_context`, `budget` describes the canonical Markdown working-context projection, not operational compatibility metadata. The response keeps that Markdown unchanged in `markdown` and returns diff --git a/docs/plans/2026-08-13-001-feat-search-index-and-icp-alignment-plan.md b/docs/plans/2026-08-13-001-feat-search-index-and-icp-alignment-plan.md new file mode 100644 index 00000000..5250bd14 --- /dev/null +++ b/docs/plans/2026-08-13-001-feat-search-index-and-icp-alignment-plan.md @@ -0,0 +1,809 @@ +--- +artifact_contract: ce-unified-plan/v1 +artifact_readiness: implementation-ready +execution: code +product_contract_source: ce-plan-bootstrap +title: Search Scale, Resilience, and ICP Alignment - Plan +type: perf +date: 2026-08-13 +deepened: 2026-08-13 +--- + +# Search Scale, Resilience, and ICP Alignment - Plan + +> **Amended by:** +> [`2026-08-13-002-search-performance-gate-amendment.md`](./2026-08-13-002-search-performance-gate-amendment.md), +> which governs and supersedes the withdrawn U5/U6 bytes-only `raw-read` +> performance clauses. All other requirements in this plan remain in force. + +## Goal Capsule + +Make end-to-end search fast and predictably degradable as a personal corpus +grows, without weakening the evidence-containment boundary or changing results; +then remove first-run language that exposes implementation details to the +non-expert user described by the product brief. + +- **Authority:** repository safety rules and ADRs outrank this plan; this Product + Contract outranks implementation convenience; measured output parity outranks + latency alone. +- **Stop conditions:** stop and re-plan if the latency target requires removing + a containment, identity, race, UTF-8, or budget guarantee; if optimized safe + scanning misses the performance gate; or if result ordering differs from the + current implementation. +- **Execution profile:** four independently reviewable slices: safe search + performance, honest ceiling behavior, bounded archive lifecycle, and first-run + language. The performance slice does not deliver predictable degradation; + that behavior is complete only when U3 lands. U4 and the archive-runway + evidence may proceed in parallel while U8/U5/U6 remain the search critical + path. +- **Release boundary:** these slices are supporting hardening, not the + Foundation launch proof. Foundation release readiness remains blocked on the + task-aware continuity outcome and host receipt owned by + `docs/plans/2026-08-09-001-feat-foundation-continuity-plan.md`; completing this + plan cannot by itself authorize a launch claim. +- **Tail ownership:** `ce-work` owns test-first implementation, review, commits, + CI, and PR landing; this plan remains a decision artifact. + +--- + +## Product Contract + +### Summary + +The first performance slice should optimize the operation the profiler proved +expensive: repeated containment validation for every file in one search +request. A persistent index is not part of this implementation unless a later +measurement gate proves the optimized safe scan insufficient. Resource ceilings +must return bounded, honest omissions when the affected source can be skipped, +while integrity violations continue to fail the whole request. + +### Problem Frame + +2.0.3 fixed one search failure. It did not make the safe search path scale, and +it left both resource-ceiling and archive-growth recurrence paths. + +Four separate problems, often conflated: + +1. **The measured bottleneck is safe file access, not corpus statistics.** A + controlled 10,000-file search takes about 3.9-4.1 seconds and performs about + 67 `lstat` and 16 `realpath` calls per file. The same traversal, matching, + tokenization, and ranking with containment isolated takes about 0.5 seconds + and returns identical hits. `buildCorpusStats` is material within that raw + logic but only about 7% of current wall time. Scaling from 500 to 10,000 files + is approximately linear; the earlier superlinear claim compared unlike + corpora. +2. **Resource ceilings currently erase unrelated results.** A directory over + the entry ceiling or one oversized JSONL rejects a shared request and aborts + parallel scopes, even when another scope contains a valid hit. The command + can also exit successfully after returning an empty result, creating false + completeness. +3. **Archives still grow without a lifecycle.** `memory/events-archive.jsonl` + has no rotation. + `maintainMemory` compacts `events.jsonl` to 50 entries whenever it passes 100, + moving the rest into that archive forever, and `search.mjs:317` reads it under + `maxFileBytes`. Rotation alone postpones the next aggregate byte/entry limit, + so lifecycle and explicit omission reporting must be designed together. +4. **ICP drift.** `docs/foundation-program/product-brief.md` states the user is a + non-expert who "should not need to understand prompt engineering, context + windows, skill routing, Git, MCP, or retrieval infrastructure." Measured on a + clean sandbox, `setup --dry-run` names Antigravity, Kimi Code CLI, Hermes, + "managed bridge", "managed skill links", and prints absolute paths in its + first 20 lines. The first `brief` a new user's agent receives leaks raw YAML + frontmatter (`source: dotaios init`, `created_at:`, `kind: context`). + +### Actors + +- A1. A person using local agents who needs continuity without learning the + retrieval and host-projection internals. +- A2. A CLI caller who needs useful results plus actionable warnings. +- A3. An MCP host that must retain read-only canonical-data behavior and receive + the same result/omission semantics as the CLI. + +### Flows + +- F1. A caller searches an authorized corpus; DotAIOS safely enumerates and + reads eligible files, ranks exact current content, and returns the same order + as the existing implementation within the latency budget. +- F2. A skippable resource ceiling affects one source; DotAIOS returns results + from unaffected sources and a bounded omission. An integrity or authorization + failure still rejects the whole request. +- F3. A first-time user previews setup or receives a brief; the default output + describes outcomes in product language, while an explicit verbose mode keeps + the operator detail. + +### Requirements + +#### Search performance and parity + +- R1. End-to-end safe search over the controlled 10,000-file fixture returns + validated results with p95 wall latency under one second on the benchmark + machine, after warm-up and across at least 20 measured runs. The PR records + hardware, Node version, median, p95, peak RSS, and file-operation counts. +- R2. Search returns the same ordered results as the pre-optimization path for + substring, inflection, phrase, frontmatter, path/title, recency, scope, and tie + behavior. +- R3. The optimization may amortize request-redundant validation but may not + weaken authorization, lexical and canonical containment, no-follow file + opening, handle/path identity binding, ancestor/directory race detection, + UTF-8 validation, or byte/file/entry ceilings. +- R4. No runtime dependency, build step, vector, embedding, graph, daemon, or + database is introduced. + +#### Honest degradation and lifecycle + +- R5. Skippable resource ceilings return available results plus a bounded, + non-path-leaking omission envelope across core, CLI, and MCP. Observed unsafe + paths, symlinks, evidence mutation, unauthorized roots, and invalid + configuration remain request-fatal. An over-ceiling directory is explicitly + **uninspected** beyond its observation boundary; the result may never imply + that unvisited children were security-validated. Every omission carries a + closed reason code and a path-safe, reason-specific recovery action. +- R6. Budget allocation and omission order are deterministic and starvation + resistant across scopes. A bounded preflight gives every requested logical + scope a fair-share reservation before unused capacity is redistributed in + declared order; a fast parallel scope cannot consume another scope's safety + budget by winning a race. +- R7. Event and signal archives rotate crash-safely before the per-file ceiling, + remain searchable without loss or duplication, and report any later aggregate + ceiling rather than claiming complete results. + +#### First-run language + +- R8. Default setup and doctor output names only detected clients and user + actions, uses home-relative paths where a path helps, and hides internal + projection terms. The `brief --compact` and MCP `read_working_context` + projections strip raw frontmatter from rendered identity context. +- R9. Verbose/operator output retains the diagnostic information removed from + the default view. + +### Acceptance Examples + +- AE1. On the 10,000-file fixture, the optimized safe path finds the same four + controlled needles in the same order and meets R1; a timed failure or empty + result never counts as a performance pass. +- AE2. One scope contains a directory above its entry ceiling while another + contains the query. Search returns the valid hit, marks the skipped logical + source as uninspected without an absolute path, and gives a reason-specific + next action for obtaining a complete result with equivalent meaning through + CLI and MCP. Replacing the directory root with an unsafe symlink rejects the + request; an unsafe child beyond the ceiling remains part of the uninspected + omission and can never be represented as complete. +- AE3. A fresh default setup preview names the detected client, explains the + user-visible outcome, and gives the next action without naming absent hosts or + managed projection vocabulary. Doctor does the same for healthy, warning, and + blocking states. A fresh compact brief and MCP working-context response retain + visible identity content but contain no YAML metadata; setup/doctor verbose + output still exposes the supported operator view. +- AE4. Rotation at a shard boundary followed by interruption and retry leaves + every event searchable exactly once. A concurrent search returns either the + complete pre-rotation generation or the complete post-rotation generation; + an unsafe target or an archive line above the read ceiling fails before source + removal. + +### Success Criteria + +- Safe search meets R1 with exact output parity and a bounded file-operation + profile, not merely a faster unsuccessful code path. +- All interfaces distinguish complete, partially omitted/uninspected, and + rejected search. +- The first-run copy assertions pass without making verbose diagnostics poorer. + +### Scope Boundaries + +**In scope:** request-scoped containment amortization; search integration and +benchmarking; typed resource-ceiling omissions across core/CLI/MCP; archive +rotation and discovery; the first-run language pass. + +**Separate release-critical dependency:** the Foundation continuity plan owns +the end-to-end proof that a fresh agent receives the smallest relevant project +evidence with provenance and can continue without retelling. It may execute in +parallel, but it must be approved, implemented, and evidenced before Foundation +launch readiness can be claimed. + +**Deferred to follow-up work:** + +- A persistent index. Reconsider only if the optimized safe scan misses R1 on + representative prose, or a materially larger real corpus demonstrates a + repeated-query win that justifies persistent derived state. The candidate must + prove exact candidate-superset behavior, logical-corpus isolation, safe + invalidation, CLI/MCP write posture, parse p95 at most 150 ms, peak heap delta + at most 64 MiB, and serialized size at most 1.5x indexed source bytes with an + explicit hard ceiling. Object-per-token frequency-map JSON is rejected by the + current measurement: 17.79 MiB and about 1.38 seconds to parse on the fixture. +- Hybrid, vector, embedding, or graph retrieval. Revisit only after a repeated + representative lexical miss, not from generic retrieval benchmarks. +- Default inclusion of every project in unscoped search; that is a product + decision, not a performance correction. +- PR #62 CI diagnosis, the loose bridge-certification check, and the high-entropy + V8 `Map` ceiling. +- Managed-skill CLI hardening discovered in the Sonnet session. The reported + `npx` defect was a wrapper dropping the required `sha256:` prefix and ignoring + exit status; a clean exact-fingerprint `npx` flow succeeds. A separate small + change may validate the prefix earlier, improve the error/help text, and add a + packaged-`npx` acceptance test. + +**Not in scope:** derived state inside the canonical AIOS folder; weakening any +evidence-reader security guarantee; hosted storage or model-written fact +extraction. + +### External Architecture and Product Evidence + +Researched 2026-08-13. This is the part that changes the plan's confidence. + +Claude Code validates bounded, on-demand plain-text context: auto memory is +repository-scoped and machine-local, and only the first 200 lines or 25 KB of +`MEMORY.md` load at startup; excess is silently omitted. Claude and Codex both +recommend moving procedures out of always-on instruction files into skills. +Skill bodies load progressively, but metadata still consumes bounded recurring +context, so “skills are free” is not a defensible claim. + +OpenAI's current Codex guidance similarly bounds the combined `AGENTS.md` chain +at 32 KiB by default and exposes only a bounded initial skills catalog. Codex +also now has generated local memory, so DotAIOS should differentiate on +canonical user-owned knowledge, deterministic routing, inspectability, and +cross-host portability rather than claiming other agents have no memory. + +The current RAG paper supports lexical-first evaluation, not this index design: +at 601M tokens its enterprise exact-fact workload reports BM25 50.5, raw +file-system agent 30.7, and dense retrieval 29.9. It does not contain the old +plan's hybrid recall claim, and its roughly 10M-token crossover compares BM25 +with iterative raw-file exploration, not BM25 with dense retrieval. + +Basic Memory and GBrain already combine canonical Markdown, rebuildable local +indexes, MCP, and cross-agent use; GBrain also makes Git part of the explicit +workflow. OpenMemory and Supermemory market one memory across tools. The +defensible product wedge is therefore the lighter contract: no account, daemon, +database, mandatory model, or opaque extracted-fact store. + +--- + +## Planning Contract + +### Key Technical Decisions + +- KTD1. Optimize containment at an evidence-reader-owned transaction seam, not + inside the generic single-file primitive. The caller performs matching inside + the transaction; it cannot obtain a successful result until final root, + ancestor, and directory-generation validation completes. Each file retains + lexical checking, no-follow handle binding, identity validation, canonical + containment, and a comparison with the enumerated parent identity at handle + validation. This localizes the performance change without weakening unrelated + consumers or relying on the caller to remember a commit check. +- KTD2. Measure before indexing. The prior index proposal cached work after + every canonical file had already been opened and scanned, and its literal + JSON shape misses R1 on parse time alone. If U5/U6 miss R1 while satisfying + R2/R3, execution stops for a new candidate-retrieval design; it does not add a + cache opportunistically. +- KTD3. Classify failures by meaning. Resource exhaustion that can be isolated + becomes a typed bounded omission; integrity, authorization, configuration, + and observed-mutation failures remain request-fatal. This prevents false + completeness without turning security failures into warnings. +- KTD4. Use a two-phase fair-share reservation. A bounded metadata preflight + captures each requested logical scope's demand and generation independently. + Half of each request-wide file, byte, and entry ceiling is divided equally as + a protected minimum tranche across requested scopes; the other half and any + unused protected capacity are allocated in declared order. Scopes that still + cannot be completed release their capacity for one deterministic second pass. + A scope is searched only when its full preflight demand fits its final + reservation; otherwise it becomes one omission. Bounded concurrency remains + within an admitted scope. This prevents timing races and protects later scopes + from one large early scope without pretending partial-corpus ranking is + equivalent to complete ranking. +- KTD5. Keep the unsuffixed archive as the active append target and rotate full + content into immutable, zero-padded numbered shards under the existing memory + lock. Search reads numbered shards in ascending order followed by the active + archive. The transition supports legacy unsuffixed archives in place, uses the + existing pending batch as recovery authority, and never overwrites a preexisting + shard. Rotation prevents a single-file recurrence; R5/R7 handle eventual + aggregate limits honestly. +- KTD6. Keep the ICP change as a default-copy pass. No command loses operator + information; verbose output retains it. +- KTD7. Keep lexical retrieval and canonical Markdown. External evidence + validates progressive disclosure and lexical-first testing but does not prove + DotAIOS needs persistent indexing at this corpus size. Competitive positioning + rests on a database-free, daemon-free, model-optional ownership contract. +- KTD8. Preserve the core search return shape by adding frozen `omissions` + metadata beside the existing `scope` metadata on the iterable result. CLI + keeps results on stdout, sends the omission/recovery warning to stderr, and + sets exit code 2 for every partial/uninspected result; exit 0 means a complete + search even when no hits exist, and exit 1 remains fatal. MCP returns valid + results plus structured `complete: false` and the same logical omissions + without marking the tool call failed. The closed omission schema contains + logical scope, reason code, bounded observed counts, inspection state, and a + path-safe recovery code/message. It exposes at most 32 entries plus one + aggregate remainder and never a machine path. Search currently has no JSON CLI + mode; adding one is outside this plan, and any future machine-readable surface + must reuse this schema rather than infer completeness from exit text. +- KTD9. Make the operator surface explicit. `setup --verbose` and + `doctor --verbose` augment the concise default with the diagnostic detail + those commands currently expose and are documented in help. Compact working + context has no raw-metadata mode: `brief --compact` and MCP + `read_working_context` both suppress frontmatter, while `brief --json` changes + only the transport envelope. Unknown options continue to fail. + +### High-Level Technical Design + +The existing request flow remains authoritative for matching and ranking: + +1. Search resolves the same authorized logical scopes and inclusion predicates. +2. The evidence reader opens a transaction-owned corpus snapshot from safe + traversal, recording the authorized root, eligible regular files, and the + observed directory/ancestor generation. +3. Bulk reads reuse invariant validation while keeping per-file no-follow open, + handle/path identity, byte, UTF-8, and mutation checks. +4. Search performs the existing canonical-content match, snippet, corpus-stat, + and ranking logic unchanged. +5. After matching/ranking and before the transaction resolves, the evidence + reader revalidates root identity and the observed ancestor/directory + generation. Any mismatch rejects the whole request. +6. Skippable resource failures enter the bounded omission collector; fatal + failures bypass it. CLI and MCP render the same logical fields and recovery + meaning on their native transports. + +No persistent derived search state or MCP write path is added. U3 adds only +canonical archive shards through the existing memory-maintenance write boundary. + +### System-Wide Impact + +- **Core boundary:** `evidence-reader.mjs` gains the bulk request abstraction; + `contained-read.mjs` supplies reusable snapshot checks without relaxing its + generic contract; `search.mjs` adopts the bulk path and omission envelope. +- **Interface parity:** CLI and MCP must both expose complete/partial/rejected + outcomes. Machine paths stay out of omissions; logical scope and reason are + sufficient. +- **Concurrency:** scope scheduling follows KTD4. Search does not acquire the + writer lock; atomic archive publication plus the evidence-reader generation + transaction means it observes a stable generation or fails with source-changed + and can be retried, never a partly published shard. +- **Canonical data:** search remains read-only. Archive rotation is an existing + memory-maintenance write concern, not a side effect of search. +- **Security:** performance is proven with adversarial ancestor, directory, + enumerated-parent, file-replacement, symlink, hard-link, and evidence-change + tests, not inferred from a happy-path benchmark. This preserves the repo's + observation-boundary threat model; portable Node does not claim immunity to an + unobserved same-user swap-and-restore entirely between validation barriers. + +### Assumptions + +- The controlled 10,000-file fixture and the current machine are the reproducible + performance reference; the PR records their characteristics so later runs are + comparable. +- The handle-bound prototype's roughly 0.55-0.65 second result demonstrates that + R1 is plausible, not that its abbreviated checks are production-ready. +- R1 is an engineering responsiveness gate, not evidence of user retention or + reduced abandonment; the separate continuity proof owns the product-outcome + measurement. +- Default output is concise for the product user; verbose output is the supported + operator surface. + +### Risks and Mitigations + +- **Security regression disguised as speed:** require per-barrier race tests and + end-of-request generation validation before accepting any latency result. +- **Benchmark-only optimization:** cover shallow/nested, prose/high-entropy, and + no/low/high-hit fixtures; validate real returned evidence before timing. +- **Partial results mistaken for complete:** make omissions structured and + bounded in core, cap explicit entries at 32 plus one aggregate remainder, keep + CLI results/warnings on stable channels, and assert equivalent rendering, + recovery meaning, and completion state in CLI and MCP. +- **Archive rotation creates loss or reordering:** publish under the memory lock, + preserve stable chronology, and test interruption/concurrency with exact event + sets rather than counts alone. +- **Scope creep back into indexing:** KTD2 and the Goal Capsule stop condition + require a plan revision with new measurements before persistent state work. + +### Sequencing and Landing + +1. U4 may begin immediately and land first as the independent first-run polish + PR; its positive plain-language assertions must pass alongside the absence + checks. +2. U8 freezes the benchmark authority and pre-change receipt before production + performance code begins. U5 and U6 then form the performance PR. U5 must pass + its safety and preliminary 10,000-file gates before U6 integration. Do not + begin persistent-index work if the full gate fails; stop and revise this plan + with the new measurements. +3. U3 follows the stabilized traversal seam and forms the ceiling-resilience PR. + Until it lands, search remains fail-closed on resource ceilings; release notes + must not claim predictable degradation after the performance PR alone. +4. U7 is independent of U5/U6 and has its own archive-lifecycle PR. Record the + archive runway before scheduling it. The 2026-08-13 user-corpus snapshot + measured events at 199,021 bytes (725 lines, 4.7% of 4 MiB) and signals at + 66,177 bytes (203 lines, 1.6%). Git-visible growth since the 2026-07-27 archive + creation was about 30 event lines/day and 10 signal lines/day; at the current + average line sizes, the coarse per-file runways are about 480 and 1,240 days. + Re-measure at execution time, but this evidence does not justify placing U7 + ahead of U3 or the release-critical continuity proof. + +--- + +## Implementation Units + +### U8. Freeze benchmark authority and pre-change receipt + +**Goal:** Make the performance decision reproducible and falsifiable before the +optimization changes the target. + +**Requirements:** R1, R2, R4; F1; AE1 + +**Dependencies:** none + +**Files:** +- `benchmarks/search/manifest.json` +- `scripts/bench-search.mjs` +- `docs/benchmarks/2026-08-13-search-baseline.md` +- `tests/core/search_benchmark_manifest.test.mjs` + +**Approach:** +1. Check in a manifest that fixes the reference machine identifier and power + profile, Node version, fixture-generator version and seed, file counts, + shallow/nested layouts, file-size/token/frontmatter distributions, query and + expected-hit sets, warm-up/sample protocol, and raw-read control. +2. Generate fixtures outside the repository from that manifest; do not commit a + 10,000-file corpus. Hash the generated corpus inventory so a changed fixture + invalidates comparison with the recorded baseline. +3. Record the pre-change contained path, raw control, cold/warm median and p95, + peak RSS, exact ordered results, and `lstat`/`realpath`/`open` counts before U5 + production changes begin. + +**Test scenarios:** +- The same seed produces the same inventory hash, needles, and expected order on + Node 20 and 22. +- Changing any corpus/query/protocol field changes the manifest receipt and + cannot reuse the old baseline. +- The harness validates output before accepting a timing sample and exits nonzero + on an error, empty controlled result, or order mismatch. + +**Verification:** The checked-in manifest, deterministic-generator test, and +pre-change receipt exist and agree; U5 may not start until this gate is green. + +### U5. Request-scoped safe corpus snapshot + +**Goal:** Amortize redundant containment work across one corpus scan while +preserving every security property in R3. + +**Requirements:** R3, R4; F1; AE1 + +**Dependencies:** U8 + +**Files:** +- `packages/core/src/evidence-reader.mjs` +- `packages/core/src/contained-read.mjs` +- `tests/core/evidence-reader.test.mjs` +- `tests/core/search-safety.test.mjs` + +**Approach:** +1. Add an opaque transaction-owned snapshot/bulk-read capability at the evidence + reader boundary. Matching/ranking executes within its callback/lifetime, and + successful results cannot escape until final validation completes. Keep the + existing single-file contained read unchanged for other consumers. +2. Safely enumerate eligible regular files once, bind the snapshot to the + authorized root and observed directory/ancestor generation, and bound every + collection before expansion. +3. Retain per-file lexical containment, no-follow open, handle/path identity, + canonical containment, byte, UTF-8, and before/after mutation checks. At + handle validation, bind each file to the identity of the parent directory + recorded during enumeration; if portable Node 20 primitives cannot establish + that binding within R1, stop rather than weaken R3. +4. After the callback finishes matching/ranking, revalidate root identity and + every observed directory/ancestor generation before resolving the + transaction. A changed generation fails closed and returns no partial result. + +**Test scenarios:** +- Static shallow, nested, external-vault, and selector-scoped corpora return the + same bytes and paths as ordinary contained reads. +- Root, ancestor, enumerated parent, directory, or file replacement at every + observation barrier rejects the batch; synchronized swap/restore attempts + spanning a barrier fail, and symlinks/non-regular files remain ineligible. +- Invalid UTF-8 and each configured resource ceiling retain their current safe + classification until U3 deliberately adds skippable handling. +- Operation counters show at most four `lstat`, two `realpath`, and one `open` + per accepted file, plus request-level traversal/final validation proportional + to observed directories and root depth—never files multiplied by observed + directories. +- On the U8 fixture, bulk read plus final validation p95 is no more than raw-read + control p95 plus 150 ms, and that p95 plus the recorded unchanged + match/tokenize/rank p95 is below 900 ms. A miss stops U6 integration. + +**Verification:** The bulk reader passes the existing safety suite plus the new +race matrix, canonical output equality against individual reads, and the +preliminary performance gate. + +### U6. Integrate safe bulk search and enforce the performance gate + +**Goal:** Meet R1 without changing search semantics or persisting derived state. + +**Requirements:** R1-R4; F1; AE1 + +**Dependencies:** U5 + +**Files:** +- `packages/core/src/search.mjs` +- `docs/architecture.md` +- `tests/core/search-ranking.test.mjs` +- `tests/core/search_corpus_scale.test.mjs` +- `tests/core/search-safety.test.mjs` + +**Approach:** +1. Route `searchMarkdownDir` through U5 while preserving source predicates, + canonical-content matching, snippets, corpus-stat boundaries, ranking, and + stable merge order. Ranking runs inside the transaction and its result becomes + observable only after U5's final generation validation. +2. Establish a differential oracle for substring and inflection matches, phrases + across punctuation/lines, frontmatter descriptions, path/title boosts, + recency/ties, memory sub-corpora, plugins, projects, and external vaults. +3. Benchmark 500, 2,500, and 10,000 files across shallow/nested and representative + prose/high-entropy corpora, including no-hit, low-hit, and high-hit queries. + Validate results before recording duration. +4. Record cold and warm latency, median/p95, peak RSS, and file operations. Keep + latency out of brittle general CI while enforcing deterministic operation + counts and output parity in CI. +5. Document the request-scoped safety/performance seam and the deliberate absence + of persistent derived search state. + +**Test scenarios:** +- Every parity fixture produces exactly the existing ordered results and snippets. +- A timed empty/error path is rejected as a benchmark sample. +- The 10,000-file warm benchmark meets R1 and remains within 1.5x of the raw-read + control while retaining R3. +- The 500-file fixture p95 regresses by no more than the larger of 20% or 50 ms + from the U8 contained baseline. +- At fixed directory topology, operation totals scale no worse than 2.1x when + file count doubles, per-file operation counts at 2,500 and 10,000 files stay + within 10%, and no observed-directory scan occurs inside the per-file loop. +- Added, deleted, and modified files are visible on the next request because the + optimization is request-scoped, not a stale cross-request cache. + +**Verification:** R1's measurement record and the differential suite pass. If +R1 fails, stop at the Goal Capsule condition and revise the plan before adding +any index. + +### U3. Honest resource ceilings + +**Goal:** Make skippable corpus ceilings partial-but-explicit rather than +request-wide false failures. + +**Requirements:** R5, R6; F2; AE2 + +**Dependencies:** U5, U6 + +**Files:** +- `packages/core/src/evidence-reader.mjs` +- `packages/core/src/search.mjs` +- `packages/cli/src/commands/search.mjs` +- `packages/mcp/src/server.mjs` +- `docs/architecture.md` +- `docs/mcp.md` +- `tests/core/evidence-reader.test.mjs` +- `tests/core/search-safety.test.mjs` +- `tests/cli/search-safety.test.mjs` +- `tests/mcp/server.test.mjs` + +**Approach:** +1. Define the closed omission contract in KTD8, including reason-specific, + path-safe recovery for per-file size, directory entries, aggregate bytes, + file count, and entry count. Freeze the envelope before exposing it. +2. Preflight requested logical scopes and reserve budget according to KTD4. Omit + an unadmitted scope as a whole so its IDF/ranking never masquerades as a + complete corpus. Pin each omitted directory's observed identity at the skip + decision and include it in final transaction revalidation. +3. Serialize the same logical fields through core, CLI, and MCP. CLI leaves + valid results on stdout, writes the warning/recovery guidance to stderr, and + exits 2; MCP returns the results with `complete: false`. Both still fail on + unsafe observed evidence. +4. Document completion states, reason/recovery codes, CLI channels/exit status, + and MCP representation. + +**Test scenarios:** +- Oversized JSONL, directory-entry, per-file, aggregate-byte, file-count, and + entry-count limits produce the intended bounded omission while unaffected + scopes still return controlled hits. +- Unsafe symlink/path, changed evidence, unauthorized root, and invalid config + remain request-fatal through core, CLI, and MCP. +- Concurrent scope timing does not change which source is searched or omitted. +- Omissions never contain an absolute home path and never allow a partial result + to be represented as complete; each reason gives one safe route to a complete + retry, and more than 32 omissions collapse into one counted remainder. + +**Verification:** Cross-interface golden fixtures prove identical result and +omission semantics, while the adversarial safety suite proves fatal boundaries +did not become warnings. + +### U7. Bounded archive lifecycle + +**Goal:** Prevent event and signal archives from crossing the per-file search +ceiling without coupling canonical-write safety to the search-performance gate. + +**Requirements:** R7; F2; AE4 + +**Dependencies:** none; schedule after the runway receipt in Sequencing and +Landing, and integrate with U3 omission semantics when both are present + +**Files:** +- `packages/core/src/memory.mjs` +- `packages/core/src/search.mjs` +- `packages/core/src/owned-state.mjs` +- `docs/architecture.md` +- `docs/advanced-memory.md` +- `tests/core/memory.test.mjs` +- `tests/core/search-safety.test.mjs` + +**Approach:** +1. Keep the unsuffixed archive as the active append target and rotate at 2 MiB + on JSONL line boundaries into immutable zero-padded numbered shards. A single + valid line above 2 MiB but within the 4 MiB read ceiling occupies its own + shard; a line above 4 MiB fails maintenance before committing source removal. +2. Under the existing memory lock, verify the active archive and next shard are + owned regular files with safe link counts; create with exclusive 0600 mode, + fsync file and directory, and atomically publish without overwriting an + existing shard. Reuse owned-state publication patterns rather than raw + pathname writes. +3. Preserve the pending batch as recovery authority across interruption. Support + legacy unsuffixed archives, discover numbered shards in numeric order followed + by the active file, and deduplicate exact events across retry boundaries. +4. Search takes one archive-generation snapshot and revalidates it before result + commit. Concurrent rotation yields the old complete generation, the new + complete generation, or a fatal source-changed retry—never a mixture. Once U3 + exists, later aggregate exhaustion becomes its bounded omission. + +**Test scenarios:** +- Normal, boundary-size, concurrent, and every injected interruption point lose + and duplicate zero events; re-running recovery is idempotent. +- Preexisting targets, symlinks, hard links, ownership/mode violations, and + active-file replacement fail before publication or source deletion. +- Live plus rotated archives remain searchable in stable chronology, including a + legacy archive upgraded in place and a search racing rotation. + +**Verification:** Lifecycle fixtures compare exact event identities and order, +not counts alone; security tests prove link-safe publication and the generation +contract, and the recorded archive runway accompanies the PR. + +### U4. ICP language pass + +**Goal:** Make first-run output read as intended for the person the brief +describes, with the full operator view still available. + +**Requirements:** R8, R9; F3; AE3 + +**Dependencies:** none + +**Files:** +- `packages/cli/src/commands/setup.mjs` +- `packages/cli/src/commands/doctor.mjs` +- `packages/cli/src/commands/brief.mjs` +- `packages/core/src/working-context.mjs` +- `tests/cli/first_run_language.test.mjs` +- `tests/core/working-context.test.mjs` +- `tests/mcp/server.test.mjs` + +**Approach:** +1. Default setup and doctor output name only detected clients by product name, + state the user outcome, give one safe next action for healthy/warning/blocking + states, and use home-relative paths where useful. Preserve non-color status + markers so meaning does not depend on color. +2. Add and document `--verbose` for setup and doctor; it augments the concise + default with the current diagnostic detail. Reject unknown options. Do not add + a raw-metadata mode to compact brief or MCP working context. +3. Strip YAML frontmatter from context files before rendering identity into a + brief; source files remain byte-unchanged, and CLI/MCP use the same core + rendered content. + +**Test scenarios:** +- A clean sandbox names no absent host and contains none of the internal managed + projection vocabulary enumerated by AE3. Detected-client, no-client, warning, + and blocking fixtures each state status, outcome, and one safe next action + without asserting exact prose. +- Default paths are home-relative where exposed; verbose output retains the full + setup/doctor diagnostic view, help names the option, and unknown options fail. +- A fresh compact brief contains no frontmatter keys, while a context file with + no frontmatter renders unchanged. The MCP `read_working_context` fixture + preserves the same visible identity/priorities content without YAML, and source + hashes remain unchanged. + +**Verification:** Positive semantic and absence-focused assertions protect the +user outcome and vocabulary without turning exact copy into a brittle API; CLI +and MCP frontmatter fixtures prove shared rendering. + +--- + +## Verification Contract + +- **Repository gates:** `npm run syntax-check`, `npm test`, `npm run smoke`, and + `npm run check` pass locally; CI passes on Node 20 and 22. Packaging-affecting + changes additionally pass `npm run pack:check`. +- **Test-first gate:** each behavioral or safety regression is observed failing + against the pre-change code before its implementation passes. +- **Parity gate:** U6's differential matrix proves exact ordered result/snippet + equality; U3's core/CLI/MCP fixtures prove identical complete/partial/rejected + semantics and recovery meaning on stable CLI channels. +- **Safety gate:** the existing containment suite plus the U5/U3 race and + ceiling matrices remain green. No resource omission can suppress an integrity + failure. +- **Performance gate:** after warm-up, at least 20 validated samples on the + 10,000-file fixture meet R1. The PR records cold and warm results for 500, + 2,500, and 10,000 files, representative prose and adversarial high-entropy + content, no/low/high-hit queries, median, p95, peak RSS, and operation counts. + The 500-file regression bound and U5 preliminary budget also pass. At fixed + topology, operation scaling meets U6's 2.1x/10% invariants and never performs + per-file multiplication by observed directories. +- **Real-folder gate:** compare current and optimized searches against the actual + AIOS folder, inspect returned evidence before accepting timing, record median + and p95 as informational evidence, and record the fact that unscoped search + does not include projects without a selector. The `ce-work` implementer owns + producing and attaching this receipt to the performance PR. +- **Lifecycle gate:** U7 records current size, observed growth window, and + estimated per-file runway, then proves exact event identity/order across + rotation, recovery, concurrency, and link-safety fixtures. +- **Documentation gate:** the implementation unit that changes a contract owns + the matching documentation edit and testable examples; documentation cannot + be deferred to an unowned tail task. +- **Behavioral skill gate:** no additional skill evaluation is required; this + work changes product code, not skill triggering or instructions. + +## Definition of Done + +- R1-R9 and AE1-AE4 hold with CI green on Node 20 and 22. Foundation launch + readiness is still prohibited until the separate continuity plan's approved + end-to-end outcome and host receipt are complete. +- U8 is done when the immutable benchmark manifest, deterministic fixture + receipt, and pre-change baseline are checked in before U5 production edits. +- U5 is done when the bulk evidence path is output-equivalent to individual safe + reads, passes every static/adversarial snapshot test, and meets its preliminary + U8-relative budget. +- U6 is done when exact search parity, R1, the 500-file regression bound, and the + explicit operation-scaling invariants pass without persistent state; a miss + invokes the stop condition instead of silently expanding scope. +- U3 is done when all three interfaces distinguish complete, partial, and fatal + outcomes, expose the bounded reason/recovery contract, preserve stable CLI + channels and exit status, and do not starve later scopes behind timing or one + large early scope. +- U7 is done when archive rollover/search proves zero loss and duplication across + interrupted and concurrent maintenance, link-safe publication, and the + old/new-generation search contract. +- U4 is done when default output meets positive outcome/next-action and absence + assertions, setup/doctor verbose mode preserves operator diagnostics, and CLI + plus MCP working context omit raw YAML without changing source bytes. +- `docs/architecture.md`, `docs/advanced-memory.md`, and user-facing search/MCP + documentation describe the request-scoped performance seam, omission + semantics, archive lifecycle, and supported operator-output boundary. +- No abandoned index experiment, compatibility branch, temporary benchmark + fixture, debug instrumentation, or dead-end code remains in the final diffs. +- The four implementation slices are independently reviewable and land through + green PRs with their measurement/safety evidence attached. + +--- + +## Appendix + +### Sources & Research + +**Primary vendor and standards sources** + +- Claude Code memory — +- Claude Code skills — +- Codex `AGENTS.md` — +- Codex skills — +- Codex generated memories — + +- OpenAI harness engineering — +- Node filesystem API — +- Node SQLite API and version floor — + +**Research and prior art** + +- *BM25 Wins at Scale* — +- Basic Memory — +- GBrain — +- OpenMemory — +- Supermemory — +- Letta context hierarchy — + + +**Repository evidence** + +- `packages/core/src/search.mjs` +- `packages/core/src/evidence-reader.mjs` +- `packages/core/src/contained-read.mjs` +- `packages/core/src/memory.mjs` +- `docs/adr/0003-keep-canonical-memory-separate-from-derived-views.md` +- `docs/foundation-program/product-brief.md` +- `docs/foundation-program/evidence-ledger.md` diff --git a/docs/plans/2026-08-13-002-search-performance-gate-amendment.md b/docs/plans/2026-08-13-002-search-performance-gate-amendment.md new file mode 100644 index 00000000..122251f2 --- /dev/null +++ b/docs/plans/2026-08-13-002-search-performance-gate-amendment.md @@ -0,0 +1,85 @@ +# Search performance gate amendment — 2026-08-13 + +**Status:** Accepted implementation amendment; final measurements remain subject +to the verification contract below. + +**Amends:** +[`2026-08-13-001-feat-search-index-and-icp-alignment-plan.md`](./2026-08-13-001-feat-search-index-and-icp-alignment-plan.md) + +## Decision + +The original plan remains authoritative except for these two clauses: + +1. U5's test scenario requiring safe bulk read plus final validation p95 to be + no more than bytes-only `raw-read` p95 plus 150 ms. +2. U6's test scenario requiring 10,000-file safe canonical search to remain + within 1.5x of the bytes-only `raw-read` control. + +Those clauses are superseded by the gates below. No safety property, output +parity requirement, resource ceiling, operation-scaling invariant, fixture, or +sample-validation rule is relaxed. + +## Why the comparator changes + +The frozen `raw-read` control opens and reads fixture bytes. It deliberately +omits recursive traversal, lexical and canonical containment, no-follow and +handle/path identity binding, UTF-8 validation, root/directory/ancestor +observation, final generation revalidation, matching, snippets, corpus +statistics, ranking, and result validation. These are mandatory parts of R2 and +R3, not removable overhead. + +The measured safe-corpus delta therefore describes the cost of required work, +not a regression that can be eliminated without changing the contract. Making +the bytes-only proxy authoritative would reward weakening containment or +skipping canonical search behavior. + +The fixed-protocol full unsafe canonical-search control is the appropriate +relative diagnostic. It retains the canonical matcher, snippets, full-document +corpus statistics, ranking, result validation, fixture inventory, queries, and +sample protocol while omitting containment only. It does not become a release +limit: the end-to-end R1 threshold remains the primary latency gate. + +## Amended performance contract + +A final benchmark run is accepted only when all of the following hold: + +- **End-to-end latency:** all no-hit, low-hit, and high-hit safe-search cells on + both 10,000-file frozen fixtures have warm p95 below 1,000 ms across at least + 20 validated measured samples after warm-up. +- **Exact parity:** every accepted safe sample has the exact ordered result hash + recorded by the frozen U8 authority and matches the fixed-protocol full + unsafe canonical-search control. +- **Safety:** all R3 containment, handle binding, mutation, final-generation, + and fail-closed tests pass. Performance work may amortize repeated validation + but may not remove it. +- **Small-corpus regression:** every 500-file safe-search cell remains within + the larger of 20% or 50 ms of its frozen contained baseline. +- **Scaling:** the original U6 operation-count and no-directory-scan-inside-the- + mapper invariants remain in force across 500, 2,500, and 10,000 files. +- **Relative diagnostic:** the receipt reports safe/full-unsafe canonical-search + p95 ratios for every cell. Exceptions require explanation but cannot override + a failed R1, parity, safety, regression, or scaling gate. +- **Bytes-only diagnostic:** `raw-read` remains in the report as an explicitly + informational lower bound. Its delta and ratio are recorded but are not + release gates. +- **Runtime coverage:** the differential, race, budget, archive, CLI, and MCP + suites pass on supported Node 20 and Node 22 runtimes. + +The benchmark matrix, immutable manifest SHA-256, environment, result hashes, +latency distributions, peak RSS, and file-operation counts are recorded in the +durable optimized receipt after the implementation and review fixes settle. + +## Scope and consequences + +- R1-R9, AE1-AE4, the Goal Capsule, and the original stop condition remain in + force. +- Persistent indexing remains deferred. It is reconsidered only if the final + request-scoped implementation misses R1 or measured product needs justify a + separate retrieval design. +- The original plan body is preserved as the historical implementation + authority; this document makes the performance decision auditable without + rewriting its prior evidence. +- A passing R1 result does not authorize a release when parity, safety, + deterministic partial-result allocation, archive integrity, packaging, or CI + is failing. + diff --git a/packages/cli/src/commands/activate.mjs b/packages/cli/src/commands/activate.mjs index 3f4d2808..bc78fc64 100644 --- a/packages/cli/src/commands/activate.mjs +++ b/packages/cli/src/commands/activate.mjs @@ -82,7 +82,20 @@ export function bridgeAttentionLines(blockedBridges) { return lines; } -export async function activateCommand(args, { lifecycle = {} } = {}) { +export function catalogConflictActivationResult(skillsIndex) { + return { + detectedClientCount: 0, + configuredContextCount: 0, + detectedClientNames: [], + configuredClientNames: [], + blockedContextCount: 0, + blockedHermesCount: 0, + blockedCatalogCount: skillsIndex.conflicts.length, + results: skillsIndex.results + }; +} + +export async function activateCommand(args, { lifecycle = {}, quiet = false } = {}) { if (hasHelpFlag(args)) { printActivateHelp(); return; @@ -149,18 +162,12 @@ export async function activateCommand(args, { lifecycle = {} } = {}) { // untrusted collision bytes). if (!options.dryRun && skillsIndex.conflicts.length > 0) { const results = skillsIndex.results; - printResults("DotAIOS activation stopped", results); - console.error( - `Activation needs attention: preserved ${skillsIndex.conflicts.length} skill catalog collision(s); no client bridges were changed.` - ); + if (!quiet) printResults("DotAIOS activation stopped", results); + console.error(quiet + ? `Connection needs attention: preserved ${skillsIndex.conflicts.length} existing app configuration conflict(s); no app instructions were changed.` + : `Activation needs attention: preserved ${skillsIndex.conflicts.length} skill catalog collision(s); no client bridges were changed.`); process.exitCode = 1; - return { - detectedClientCount: 0, - configuredContextCount: 0, - blockedContextCount: 0, - blockedCatalogCount: skillsIndex.conflicts.length, - results - }; + return catalogConflictActivationResult(skillsIndex); } const global = await createGlobalBridges( @@ -188,30 +195,32 @@ export async function activateCommand(args, { lifecycle = {} } = {}) { } } - printResults("DotAIOS activated", results); + if (!quiet) printResults("DotAIOS activated", results); const refreshAction = options.dryRun ? "would refresh" : "refreshed"; - console.log(`[${refreshAction}] ${skillsIndex.path} and ${skillsIndex.resolverPath} (${skillsIndex.count} workflow(s) indexed)`); - if (skillsIndex.unresolvedProjections?.length > 0) { + if (!quiet) console.log(`[${refreshAction}] ${skillsIndex.path} and ${skillsIndex.resolverPath} (${skillsIndex.count} workflow(s) indexed)`); + if (!quiet && skillsIndex.unresolvedProjections?.length > 0) { console.warn( `[skills] preserved ${skillsIndex.unresolvedProjections.length} unmanaged native projection collision(s); run \`dotaios skills inventory\` to review adoption candidates.` ); } - if (skillsFirst) { + if (skillsFirst && !quiet) { const verb = options.dryRun ? "would inline" : "inline"; console.log(`[skills-first] bridge files ${verb} the current skill catalog.`); } - if (global.installedCount === 0) { + if (global.installedCount === 0 && !quiet) { console.log("\nNo known AI tools were detected on this machine."); console.log("DotAIOS connects a tool automatically once it is installed — re-run `dotaios activate` then."); console.log("To connect every known tool anyway, run `dotaios activate --all`."); } - console.log("\nUsing another local AI tool that can read files? Paste this line into it:"); - console.log(` Read ${path.join(aiosPath, "AGENTS.md")} first and follow it.`); - console.log("Browser chats cannot open that path. Attach AGENTS.md or paste a reviewed `dotaios brief --compact` instead."); + if (!quiet) { + console.log("\nUsing another local AI tool that can read files? Paste this line into it:"); + console.log(` Read ${path.join(aiosPath, "AGENTS.md")} first and follow it.`); + console.log("Browser chats cannot open that path. Attach AGENTS.md or paste a reviewed `dotaios brief --compact` instead."); + } - if (!options.project) { + if (!options.project && !quiet) { console.log("\nFor Cursor project rules, run `dotaios attach ` inside a project."); } @@ -220,20 +229,28 @@ export async function activateCommand(args, { lifecycle = {} } = {}) { const blockedHermesCount = results.filter((entry) => entry.action === "hermes:conflict").length; if (blockedContextCount > 0) { process.exitCode = 1; - for (const line of bridgeAttentionLines(blockedBridges)) console.error(line); + if (quiet) { + console.error(`Connection needs attention: preserved ${blockedContextCount} existing app instruction file(s). Run \`dotaios activate\` for details.`); + } else { + for (const line of bridgeAttentionLines(blockedBridges)) console.error(line); + } } if (skillStoreFailure) { process.exitCode = 1; - console.error( - `Activation needs attention: managed skill links could not be published. ${describeFileError(skillStoreFailure)}` - ); - console.error("The client bridges above were still written. Fix that folder, then run `dotaios activate` again."); + if (quiet) { + console.error("Connection needs attention: app workflows could not be connected safely. Run `dotaios activate` for details."); + } else { + console.error( + `Activation needs attention: managed skill links could not be published. ${describeFileError(skillStoreFailure)}` + ); + console.error("The client bridges above were still written. Fix that folder, then run `dotaios activate` again."); + } } if (blockedHermesCount > 0) { process.exitCode = 1; - console.error( - `Activation needs attention: preserved ${blockedHermesCount} concurrent Hermes config edit(s). Re-run activation after reviewing the file.` - ); + console.error(quiet + ? `Connection needs attention: preserved ${blockedHermesCount} app configuration edit(s). Run \`dotaios activate\` for details.` + : `Activation needs attention: preserved ${blockedHermesCount} concurrent Hermes config edit(s). Re-run activation after reviewing the file.`); } return { @@ -242,6 +259,8 @@ export async function activateCommand(args, { lifecycle = {} } = {}) { blockedContextCount, blockedHermesCount, blockedCatalogCount: 0, + detectedClientNames: global.installedAgentNames, + configuredClientNames: global.configuredAgentNames, results }; } @@ -396,7 +415,8 @@ async function createGlobalBridges( let installedCount = 0; let configuredContextCount = 0; const blockedBridges = []; - const installedAgentNames = new Set(); + const installedAgentNames = []; + const configuredAgentNames = []; for (const agent of registry) { const destination = bridgePath(homePath, agent) || path.join(homePath, agent.detect); @@ -407,7 +427,7 @@ async function createGlobalBridges( continue; } installedCount += 1; - installedAgentNames.add(agent.name.toLowerCase()); + installedAgentNames.push(agent.name); if (!agent.bridge) { results.push({ @@ -441,6 +461,7 @@ async function createGlobalBridges( results.push(result); if (isConfiguredBridgeAction(result.action)) { configuredContextCount += 1; + configuredAgentNames.push(agent.name); } else { blockedBridges.push(result); } @@ -456,6 +477,8 @@ async function createGlobalBridges( return { results: [...results, ...skills], installedCount, + installedAgentNames: Object.freeze(installedAgentNames), + configuredAgentNames: Object.freeze(configuredAgentNames), configuredContextCount, blockedBridges }; diff --git a/packages/cli/src/commands/doctor.mjs b/packages/cli/src/commands/doctor.mjs index b858586d..3b2cabea 100644 --- a/packages/cli/src/commands/doctor.mjs +++ b/packages/cli/src/commands/doctor.mjs @@ -46,6 +46,7 @@ follow along. Options: --path Check an AIOS folder other than ~/aios --home Check agent bridges somewhere other than your home + --verbose Show paths and operator-level diagnostic detail `; export async function doctorCommand(args, { detection } = {}) { @@ -54,7 +55,8 @@ export async function doctorCommand(args, { detection } = {}) { return; } - const options = parsePathHomeOptions(args); + const verbose = args.includes("--verbose"); + const options = parsePathHomeOptions(args.filter((arg) => arg !== "--verbose")); const target = path.resolve(expandHome(options.path || defaultAiosPath())); const homePath = path.resolve(expandHome(options.home || os.homedir())); @@ -69,19 +71,21 @@ export async function doctorCommand(args, { detection } = {}) { checks.push(await checkMemoryHealth(target)); checks.push(await checkContextFreshness(target)); checks.push(await checkLatestVersion({ currentVersion: INSTALLED_VERSION })); - checks.push(...await checkAgentBridges(target, homePath, detection)); + checks.push(...await checkAgentBridges(target, homePath, detection, { verbose })); console.log("DotAIOS doctor"); console.log(""); - for (const check of checks) { - console.log(`${tag(check.status)} ${check.name}`); - if (check.detail) console.log(` ${check.detail}`); - if (check.fix) console.log(` Fix: ${check.fix}`); + for (const rawCheck of checks) { + const check = verbose ? rawCheck : conciseDoctorCheck(rawCheck); + const display = (value) => verbose ? String(value || "") : humanizeDoctorPath(value, target, homePath); + console.log(`${tag(check.status)} ${display(check.name)}`); + if (check.detail) console.log(` ${display(check.detail)}`); + if (check.fix) console.log(` Fix: ${display(check.fix)}`); } console.log(""); console.log("Using another local AI tool that can read files? Paste this line into it:"); - console.log(` Read ${path.join(target, "AGENTS.md")} first and follow it.`); + console.log(` Read ${verbose ? path.join(target, "AGENTS.md") : humanizeDoctorPath(path.join(target, "AGENTS.md"), target, homePath)} first and follow it.`); console.log(" Browser chats need an attached file or a pasted, reviewed brief."); console.log(""); @@ -479,7 +483,7 @@ function skipReason(skipped) { return "update check unavailable"; } -async function checkAgentBridges(target, homePath, detection = {}) { +async function checkAgentBridges(target, homePath, detection = {}, { verbose = false } = {}) { const results = []; let foundBridge = false; let foundNativeRuntime = false; @@ -489,11 +493,13 @@ async function checkAgentBridges(target, homePath, detection = {}) { const runtimes = await readNativeRuntimes(target, homePath, detection); for (const agent of registry) { if (!await isAgentInstalled(homePath, agent, detection)) { - results.push({ - name: `${agent.name} (not installed)`, - status: "ok", - detail: "Not detected on this machine — nothing to connect." - }); + if (verbose) { + results.push({ + name: `${agent.name} (not installed)`, + status: "ok", + detail: "Not detected on this machine — nothing to connect." + }); + } continue; } anyInstalled = true; @@ -597,9 +603,9 @@ async function checkAgentBridges(target, homePath, detection = {}) { if (!anyInstalled) { results.push({ - name: "At least one AI tool installed", + name: "Local AI apps", status: "warn", - detail: "No known AI tools detected on this machine.", + detail: "No supported local AI app was detected on this machine.", fix: "Install Claude Code, Cursor, Codex, or Gemini, then run `npx dotaios activate`." }); } else if (!foundBridge && !foundNativeRuntime) { @@ -614,6 +620,64 @@ async function checkAgentBridges(target, homePath, detection = {}) { return results; } +function conciseDoctorCheck(check) { + const name = check.name + .replace(/ bridge$/, "") + .replace(/ native skills$/, ""); + return { + ...check, + name, + detail: conciseDoctorDetail(check) + }; +} + +function conciseDoctorDetail(check) { + const detail = check.detail; + if (!detail) return detail; + const exact = new Map([ + ["Bridge points to a different AIOS folder.", "This app is connected to a different AIOS folder."], + ["An AI tool is installed but no managed bridge points at this AIOS folder yet.", + "A local AI app is installed but is not connected to this AIOS folder yet."], + ["Managed bridge predates v1.23 and still calls the retired read_session_digest surface.", + "This app connection is from an older DotAIOS version."], + ["Bridge is from an older version and still loads your whole AIOS folder into every session.", + "This app connection is from an older version and still loads your whole AIOS folder into every session."], + ["Bridge is from an older version and does not match what this release writes.", + "This app connection is from an older DotAIOS version."], + ["Managed bridge markers are malformed; DotAIOS preserved the file.", + "The DotAIOS connection markers are damaged; your file was preserved."], + ["Could not read this AIOS folder, so its native skill configuration is unverified.", + "Could not read this AIOS folder, so the app connection is unverified."] + ]); + if (exact.has(detail)) return exact.get(detail); + if (check.name.endsWith(" bridge")) { + const missingPrefix = `${check.name.slice(0, -" bridge".length)} is installed but not connected yet (no bridge at `; + if (detail.startsWith(missingPrefix) && detail.endsWith(").")) { + const opaquePath = detail.slice(missingPrefix.length, -2); + return `${check.name.slice(0, -" bridge".length)} is installed but not connected yet (no connection at ${opaquePath}).`; + } + const entrypointPrefix = "Bridge points at this AIOS folder, but its entrypoint is missing ("; + if (detail.startsWith(entrypointPrefix) && detail.endsWith(").")) { + const opaquePath = detail.slice(entrypointPrefix.length, -2); + return `This app is connected to this AIOS folder, but its entrypoint is missing (${opaquePath}).`; + } + } + return detail; +} + +function humanizeDoctorPath(value, target, homePath) { + let output = String(value || ""); + const resolvedHome = path.resolve(homePath); + const resolvedTarget = path.resolve(target); + const targetRelative = path.relative(resolvedHome, resolvedTarget); + if (targetRelative && !targetRelative.startsWith("..") && !path.isAbsolute(targetRelative)) { + output = output.split(resolvedTarget).join(`~/${targetRelative.split(path.sep).join("/")}`); + } else if (resolvedTarget === resolvedHome) { + output = output.split(resolvedTarget).join("~"); + } + return output.split(resolvedHome).join("~"); +} + // A runtime with no bridge file keeps its skills in its own configuration, so // the only honest answer comes from reading that configuration. inspectSkillHealth // already does exactly this for every mode; doctor consumes it rather than diff --git a/packages/cli/src/commands/search.mjs b/packages/cli/src/commands/search.mjs index 1f2dbeb9..735116cf 100644 --- a/packages/cli/src/commands/search.mjs +++ b/packages/cli/src/commands/search.mjs @@ -65,10 +65,23 @@ export async function searchCommand(args) { totalResults += group.results.length; } + const omittedScopes = [...new Set(groups.omissions.map((omission) => omission.scope))]; + const incompleteSuffix = omittedScopes.length > 0 + ? ` Search incomplete; omitted scope(s): ${omittedScopes.join(", ")}.` + : ""; if (totalResults === 0) { - console.log("No results found."); + console.log(`${groups.omissions.length > 0 ? "No results found in inspected sources." : "No results found."}${incompleteSuffix}`); } else { - console.log(`${totalResults} result(s) found.`); + console.log(`${totalResults} result(s) found.${incompleteSuffix}`); + } + if (groups.omissions.length > 0) { + for (const omission of groups.omissions) { + console.error( + `Search incomplete for ${omission.scope}: ${omission.recovery.message} ` + + `(reason: ${omission.reason})` + ); + } + process.exitCode = 2; } } diff --git a/packages/cli/src/commands/setup.mjs b/packages/cli/src/commands/setup.mjs index 9ecbc748..faab9bc7 100644 --- a/packages/cli/src/commands/setup.mjs +++ b/packages/cli/src/commands/setup.mjs @@ -28,7 +28,11 @@ import { resolveLightpanda } from "../../../core/src/lightpanda.mjs"; import { initCommand } from "./init.mjs"; -import { activateCommand, plannedActivationConfigPatch } from "./activate.mjs"; +import { + activateCommand, + BRIDGE_COLLISION_REMEDY, + plannedActivationConfigPatch +} from "./activate.mjs"; import { revealCommand } from "./reveal.mjs"; import { emitReliabilityMetric, @@ -47,6 +51,7 @@ Options: --vault-path Use an external vault for long-term knowledge --yes, -y Use placeholder answers for non-interactive setup --dry-run Preview files, trust boundaries, and removal without changes + --verbose Show paths and operator-level setup details --skip-reveal Do not open the folder when finished --install-lightpanda Install the optional verified browser helper @@ -62,15 +67,17 @@ export async function setupCommand(args, { lifecycle = {} } = {}) { return; } + validateSetupOptions(args); assertUniqueOptions(args, ["--path", "--vault-path"]); - const passthrough = args.filter((arg) => !["--dry-run", "--skip-reveal", "--install-lightpanda"].includes(arg)); + const verbose = args.includes("--verbose"); + const passthrough = args.filter((arg) => !["--dry-run", "--skip-reveal", "--install-lightpanda", "--verbose"].includes(arg)); const skipReveal = args.includes("--skip-reveal"); const installLightpandaRequested = args.includes("--install-lightpanda"); const nonInteractive = args.includes("--yes") || args.includes("-y"); const aiosPath = path.resolve(expandHome(extractPath(args) || defaultAiosPath())); if (args.includes("--dry-run")) { - await printSetupPreview(aiosPath, args); + await printSetupPreview(aiosPath, args, { verbose }); return; } const startedAt = Date.now(); @@ -97,7 +104,9 @@ export async function setupCommand(args, { lifecycle = {} } = {}) { try { // init creates the ~/aios folder that holds the metrics store, so the // init phase markers can only be written once init has succeeded. - setupTransactionActive = await runInitWithRecovery(passthrough, aiosPath, lifecycle); + setupTransactionActive = await runInitWithRecovery(passthrough, aiosPath, lifecycle, { + quiet: !verbose + }); await lifecycle.afterInit?.({ aiosPath, setupTransactionActive }); if (setupTransactionActive) { await assertCompletedSetupTransactionTree(aiosPath, setupTransactionActive.transaction, passthrough); @@ -142,21 +151,26 @@ export async function setupCommand(args, { lifecycle = {} } = {}) { let detectedClientCount = 0; let blockedContextCount = 0; let blockedCatalogCount = 0; + let blockedContextNeedsMerge = false; + let configuredClientNames = []; console.log(""); console.log("DotAIOS setup — step 2 of 3: connect your AI tools"); console.log(""); await emitReliabilityMetric(aiosPath, { type: "setup_phase_start", phase: "activate", run_id: runId }); try { const activation = await activateCommand(passthrough, { + quiet: !verbose, lifecycle: { ...lifecycle.activation, skillsIndexWriteMode: passthrough.includes("--overwrite") ? "overwrite" : "preserve" } }); detectedClientCount = activation.detectedClientCount; + configuredClientNames = activation.configuredClientNames || []; configuredContextCount = activation.configuredContextCount; blockedContextCount = activation.blockedContextCount; blockedCatalogCount = activation.blockedCatalogCount; + blockedContextNeedsMerge = activation.results?.some((entry) => entry.action === "kept") || false; if (blockedContextCount > 0 || blockedCatalogCount > 0) { activateOk = false; process.exitCode = 1; @@ -173,16 +187,13 @@ export async function setupCommand(args, { lifecycle = {} } = {}) { if (!activateOk) { console.log(""); console.log("Folder created. Tool connection needs attention; setup stopped before optional features."); - // Never send them back to `dotaios setup`: the folder now exists, so setup - // has nothing left to do and will only report that. `activate` is the one - // command that can still finish the job, and for a preserved collision the - // non-destructive form is the one activate already named above. - // activate has already named what is blocked and how to fix each one. - // Restating it here is how setup came to call a permission error a - // "collision" the user could not find anywhere on screen. + // The folder now exists; activation is the recovery command for this phase. if (blockedCatalogCount > 0) { - console.log(`Preserved ${blockedCatalogCount} skill catalog collision(s).`); + console.log(verbose + ? `Preserved ${blockedCatalogCount} skill catalog collision(s).` + : `Preserved ${blockedCatalogCount} existing app configuration conflict(s).`); } + if (blockedContextNeedsMerge) console.log(BRIDGE_COLLISION_REMEDY); if (blockedContextCount === 0 && blockedCatalogCount === 0) { console.log("Fix the problem reported above, then run `dotaios activate`."); } @@ -272,19 +283,16 @@ export async function setupCommand(args, { lifecycle = {} } = {}) { } console.log("To explore the folder now:"); } else { - console.log(`Folder ready. Connected context for ${configuredContextCount} local AI app${configuredContextCount === 1 ? "" : "s"}.`); + const clients = formatClientNames(configuredClientNames); + console.log(`Folder ready. ${clients || `${configuredContextCount} local AI app${configuredContextCount === 1 ? "" : "s"}`} can now use your context.`); console.log("To get started:"); } console.log(" 1. Open your AI agent — Claude Code, Codex, Gemini CLI, Cursor, or any other."); - console.log(" 2. Open the ~/aios folder or make it your working directory."); + console.log(` 2. Open the ${displayHomePath(aiosPath, os.homedir())} folder or make it your working directory.`); console.log(' 3. Ask: "Read my context and tell me what I am working on."'); console.log(" 4. Update context any time: dotaios interview --review"); - // Opening Finder pulls the foreground away from this terminal, so it has to be - // the very last thing that happens. It used to run before four optional y/N - // prompts, which meant the window appeared while the terminal was still - // waiting behind it — people reasonably concluded setup had finished and never - // saw the instructions above. + // Reveal last so optional prompts retain terminal focus. // Step 3: reveal (best-effort, never blocks) await emitReliabilityMetric(aiosPath, { type: "setup_phase_start", phase: "reveal", run_id: runId }); if (!skipReveal) { @@ -340,7 +348,7 @@ async function isCompletedInstall(aiosPath) { return true; } -async function printSetupPreview(aiosPath, args) { +async function printSetupPreview(aiosPath, args, { verbose = false } = {}) { const unsupported = [ "--all", "--force", @@ -362,11 +370,15 @@ async function printSetupPreview(aiosPath, args) { const homePath = path.resolve(expandHome(extractOption(args, "--home") || os.homedir())); const target = await previewSetupTarget(aiosPath); - console.log("DotAIOS Setup preview - no DotAIOS-managed changes made"); - console.log("Scope: AIOS target, detected global bridge files, and managed skill-link directories."); + console.log(verbose + ? "DotAIOS Setup preview - no DotAIOS-managed changes made" + : "DotAIOS Setup preview - no changes made"); + console.log(verbose + ? "Scope: AIOS target, detected global bridge files, and managed skill-link directories." + : `Your context stays in ${displayHomePath(aiosPath, homePath)}. Existing app instructions are preserved.`); console.log(""); console.log("Target:"); - console.log(`${target.action} ${aiosPath}${target.note ? ` (${target.note})` : ""}`); + console.log(`${target.action} ${displayHomePath(aiosPath, homePath)}${target.note ? ` (${target.note})` : ""}`); console.log(""); console.log("Detected client actions:"); @@ -374,26 +386,35 @@ async function printSetupPreview(aiosPath, args) { console.log("[would skip] activation because setup would stop at the target check above"); process.exitCode = 1; } else { - await printClientPreview(aiosPath, homePath); + const detectedClientCount = await printClientPreview(aiosPath, homePath, { verbose }); + if (!verbose && detectedClientCount === 0) { + console.log("No supported local AI app was detected. Install Claude Code, Codex, or Gemini CLI, then run setup again."); + } } console.log(""); console.log("Safety boundaries:"); - console.log("- Setup preserves unmanaged files; bridge collisions are reported instead of replaced."); + console.log(verbose + ? "- Setup preserves unmanaged files; bridge collisions are reported instead of replaced." + : "- Your existing app instructions are preserved; conflicts stop for review."); console.log("- Private GitHub sync stays off unless you explicitly run `dotaios sync setup` later."); console.log("- DotAIOS does not copy credentials into the AIOS folder or a Git remote URL."); console.log("- DotAIOS does not start a hosted account or upload context to a DotAIOS service."); - console.log("- This command does not create the AIOS folder or change client configuration or sync."); + console.log("- This preview does not create the AIOS folder or change app settings or sync."); console.log("- When invoked through npx, npm may download and cache the named package."); console.log(""); console.log("Optional prompts after core setup (all default No):"); console.log("- Private GitHub sync: creates and connects a private mirror you control."); console.log("- Daily brief: enables the bundled local schedule entry."); - console.log("- Conversation saving: may add a managed client hook; 30-day backfill reads local history into ~/aios."); + console.log(verbose + ? "- Conversation saving: may add a managed client hook; 30-day backfill reads local history into ~/aios." + : "- Conversation saving: can connect supported apps; 30-day backfill reads local history into ~/aios."); console.log(`- Lightpanda ${LIGHTPANDA_VERSION}: downloads an optional SHA-256-verified local browser binary.`); console.log(""); console.log("After setup, verify with: dotaios doctor"); - console.log("To remove it, first disable capture and sync, then remove only DotAIOS-managed bridges and archive or delete the AIOS folder. Unmanaged client configuration is left alone."); + if (verbose) { + console.log("To remove it, first disable capture and sync, then remove only DotAIOS-managed bridges and archive or delete the AIOS folder. Unmanaged client configuration is left alone."); + } } async function previewSetupTarget(aiosPath) { @@ -419,29 +440,41 @@ async function previewSetupTarget(aiosPath) { }; } -async function printClientPreview(aiosPath, homePath) { +async function printClientPreview(aiosPath, homePath, { verbose = false } = {}) { const registry = await loadAgentRegistry(null); // The writer projects skills into every symlink target regardless of what is // detected. The preview must read the same set, or it promises less than the // run delivers. const skillDirs = symlinkTargets(registry).map((target) => target.dir); + let detectedClientCount = 0; for (const agent of registry) { const destination = bridgePath(homePath, agent) || path.join(homePath, agent.detect); const installed = await isAgentInstalled(homePath, agent); if (!installed) { - console.log(`[would skip] ${destination} (${agent.name} not detected${skillLinkCaveat(agent, homePath, skillDirs)})`); + if (verbose) console.log(`[would skip] ${destination} (${agent.name} not detected${skillLinkCaveat(agent, homePath, skillDirs)})`); continue; } + detectedClientCount += 1; if (!agent.bridge) { - console.log(`[detected] ${destination} (${agent.name} uses native or project-specific configuration; external config is outside this preview)`); + if (verbose) { + console.log(`[detected] ${destination} (${agent.name} uses native or project-specific configuration; external config is outside this preview)`); + } else { + console.log(`[detected] ${agent.name} — needs native or project-specific setup before it can use your context.`); + } + continue; + } + + if (!verbose) { + console.log(`[detected] ${agent.name} — setup will connect it to your context.`); continue; } console.log(await previewManagedBridge(destination)); } + if (!verbose) return detectedClientCount; for (const relativeDir of skillDirs) { const targetDir = path.join(homePath, relativeDir); const stats = await lstatIfPresent(targetDir); @@ -453,6 +486,43 @@ async function printClientPreview(aiosPath, homePath) { console.log(`[would add missing managed skill links] ${targetDir} (existing unmanaged entries preserved)`); } } + return detectedClientCount; +} + +function displayHomePath(value, homePath) { + const resolved = path.resolve(value); + const resolvedHome = path.resolve(homePath); + const relative = path.relative(resolvedHome, resolved); + return relative === "" ? "~" : relative && !relative.startsWith("..") && !path.isAbsolute(relative) + ? `~/${relative.split(path.sep).join("/")}` + : resolved; +} + +function formatClientNames(names) { + const values = [...new Set(names || [])]; + if (values.length < 2) return values[0] || ""; + if (values.length === 2) return values.join(" and "); + return `${values.slice(0, -1).join(", ")}, and ${values.at(-1)}`; +} + +function validateSetupOptions(args) { + const valued = new Set(["--path", "--home", "--vault-path", "--project"]); + const flags = new Set([ + "--all", "--dry-run", "--force", "--install-lightpanda", "--merge", + "--no-skills-first", "--overwrite", "--prune-aliases", "--skip-reveal", + "--skills-first", "--verbose", "--yes", "-y" + ]); + for (let index = 0; index < args.length; index += 1) { + const arg = args[index]; + if (valued.has(arg)) { + if (!args[index + 1] || args[index + 1].startsWith("--")) { + throw new Error(`${arg} requires a value`); + } + index += 1; + continue; + } + if (!flags.has(arg)) throw new Error(`Unknown option: ${arg}`); + } } // Some agents keep their skill-link directory underneath their own detect path, @@ -497,7 +567,7 @@ async function lstatIfPresent(target) { // recorded the complete expected tree before createBaseTree, and every path // left behind still matches that record. Only that narrow case gets an internal // --force retry. The metrics-only recognizer below remains for 1.27.1 upgrades. -async function runInitWithRecovery(passthrough, aiosPath, lifecycle = {}) { +async function runInitWithRecovery(passthrough, aiosPath, lifecycle = {}, { quiet = false } = {}) { if (await hasSetupTransaction(aiosPath)) { if (passthrough.includes("--force") || passthrough.includes("--overwrite")) { throw new Error( @@ -508,6 +578,7 @@ async function runInitWithRecovery(passthrough, aiosPath, lifecycle = {}) { let transactionStarted = null; try { await initCommand(passthrough, { + quiet, beforeScaffold: async (plan) => { transactionStarted = await beginSetupTransaction(aiosPath, passthrough, plan, lifecycle); }, @@ -524,6 +595,7 @@ async function runInitWithRecovery(passthrough, aiosPath, lifecycle = {}) { const transaction = await readRecoverableSetupTransaction(aiosPath, passthrough); if (transaction) { await initCommand([...passthrough, "--force"], { + quiet, plan: transaction.transaction.plan, allowSetupTransactionRecovery: true }); @@ -532,7 +604,7 @@ async function runInitWithRecovery(passthrough, aiosPath, lifecycle = {}) { return { markerStats: transaction.markerStats, transaction: transaction.transaction }; } if (!(await isFailedSetupResidue(aiosPath))) throw error; - await initCommand([...passthrough, "--force"]); + await initCommand([...passthrough, "--force"], { quiet }); console.log("Recovered an unfinished folder from an earlier run and completed it in place."); return false; } diff --git a/packages/cli/src/commands/update.mjs b/packages/cli/src/commands/update.mjs index 14ccffe9..1548c905 100644 --- a/packages/cli/src/commands/update.mjs +++ b/packages/cli/src/commands/update.mjs @@ -1,5 +1,6 @@ import readline from "node:readline/promises"; import { stdin as input, stdout as output } from "node:process"; +import crypto from "node:crypto"; import path from "node:path"; import { hasHelpFlag } from "../lib/args.mjs"; import { defaultAiosPath, ensureAiosFolder, expandHome } from "../../../core/src/paths.mjs"; @@ -54,8 +55,16 @@ export async function updateCommand(args) { const attribution = project ? { project: project.slug, ...(project.id ? { project_id: project.id } : {}) } : {}; - await appendSignal(signalsDir, { type: "update", summary: note.trim(), source: "dotaios update", ...attribution }); - await appendEvent(eventsPath, { type: "update", summary: note.trim(), source: "dotaios update", ...attribution }); + const recordId = crypto.randomUUID(); + const record = { + type: "update", + summary: note.trim(), + source: "dotaios update", + record_id: recordId, + ...attribution + }; + await appendSignal(signalsDir, record); + await appendEvent(eventsPath, record); console.log("Saved."); } diff --git a/packages/core/src/contained-read.mjs b/packages/core/src/contained-read.mjs index 2e4396b3..4632038a 100644 --- a/packages/core/src/contained-read.mjs +++ b/packages/core/src/contained-read.mjs @@ -53,7 +53,7 @@ export function createContainedReadFilesystem(root, filesystem = localFilesystem } /** Create one synchronous reservation ledger shared by a contained projection. */ -export function createContainedReadBudget({ maxBytes, maxFiles, maxEntries }) { +export function createContainedReadBudget({ maxBytes, maxFiles, maxEntries, dimensionCodes = false }) { const limits = { maxBytes: normalizeFiniteLimit(maxBytes), maxFiles: normalizeFiniteLimit(maxFiles), @@ -63,8 +63,15 @@ export function createContainedReadBudget({ maxBytes, maxFiles, maxEntries }) { return Object.freeze({ reserveFile(size) { const bytes = normalizeFiniteLimit(size); - if (used.files + 1 > limits.maxFiles || used.bytes + bytes > limits.maxBytes) { - throw new ContainedReadError("DOTAIOS_PROJECTION_READ_BUDGET_EXCEEDED"); + if (used.files + 1 > limits.maxFiles) { + throw new ContainedReadError(dimensionCodes + ? "DOTAIOS_PROJECTION_FILE_COUNT_EXCEEDED" + : "DOTAIOS_PROJECTION_READ_BUDGET_EXCEEDED"); + } + if (used.bytes + bytes > limits.maxBytes) { + throw new ContainedReadError(dimensionCodes + ? "DOTAIOS_PROJECTION_BYTE_BUDGET_EXCEEDED" + : "DOTAIOS_PROJECTION_READ_BUDGET_EXCEEDED"); } used.files += 1; used.bytes += bytes; @@ -72,10 +79,42 @@ export function createContainedReadBudget({ maxBytes, maxFiles, maxEntries }) { reserveEntries(count = 1) { const entries = normalizeFiniteLimit(count); if (used.entries + entries > limits.maxEntries) { - throw new ContainedReadError("DOTAIOS_PROJECTION_READ_BUDGET_EXCEEDED"); + throw new ContainedReadError(dimensionCodes + ? "DOTAIOS_PROJECTION_ENTRY_COUNT_EXCEEDED" + : "DOTAIOS_PROJECTION_READ_BUDGET_EXCEEDED"); + } + used.entries += entries; + }, + reserveDemand(demand = {}) { + const bytes = normalizeFiniteLimit(demand.bytes); + const files = normalizeFiniteLimit(demand.files); + const entries = normalizeFiniteLimit(demand.entries); + if (used.files + files > limits.maxFiles) { + throw new ContainedReadError(dimensionCodes + ? "DOTAIOS_PROJECTION_FILE_COUNT_EXCEEDED" + : "DOTAIOS_PROJECTION_READ_BUDGET_EXCEEDED"); + } + if (used.bytes + bytes > limits.maxBytes) { + throw new ContainedReadError(dimensionCodes + ? "DOTAIOS_PROJECTION_BYTE_BUDGET_EXCEEDED" + : "DOTAIOS_PROJECTION_READ_BUDGET_EXCEEDED"); + } + if (used.entries + entries > limits.maxEntries) { + throw new ContainedReadError(dimensionCodes + ? "DOTAIOS_PROJECTION_ENTRY_COUNT_EXCEEDED" + : "DOTAIOS_PROJECTION_READ_BUDGET_EXCEEDED"); } + used.bytes += bytes; + used.files += files; used.entries += entries; }, + remaining() { + return Object.freeze({ + bytes: limits.maxBytes - used.bytes, + files: limits.maxFiles - used.files, + entries: limits.maxEntries - used.entries + }); + }, snapshot() { return Object.freeze({ ...used }); } @@ -131,6 +170,103 @@ export async function inspectContainedPathEntry(root, filePath, options = {}) { return Object.freeze({ type, ...snapshot }); } +/** + * Observe one final component relative to a corpus transaction's already + * canonicalized root and exact parent snapshot. + * + * The transaction, not this inspector, owns root/ancestor/directory + * validation. Keeping that proof request-owned means accepted sibling files + * do not each repeat the same ancestor walk. + */ +export async function inspectContainedSnapshotPathEntry(root, filePath, options = {}) { + const filesystem = options.filesystem || localFilesystem; + const resolvedRoot = path.resolve(root); + const resolvedFile = path.resolve(filePath); + const resolvedParent = path.resolve(options.parentPath || path.dirname(resolvedFile)); + if ( + !isPathWithinLexically(resolvedRoot, resolvedFile) + || resolvedParent !== path.dirname(resolvedFile) + || !options.parentSnapshot?.stats + ) { + throw new ContainedReadError(); + } + + let current; + try { + current = await filesystem.lstat(resolvedFile); + } catch (error) { + if (error?.code === "ENOENT" || error?.code === "ENOTDIR") { + if (Object.hasOwn(options, "expectedSnapshot") && options.expectedSnapshot === null) { + return null; + } + throw new ContainedReadError("DOTAIOS_CONTEXT_SOURCE_CHANGED"); + } + throw error; + } + const type = current.isSymbolicLink() + ? "symbolic-link" + : current.isFile() + ? "regular-file" + : "other"; + const snapshot = Object.freeze({ type, stats: current, ancestors: [] }); + if ( + Object.hasOwn(options, "expectedSnapshot") + && ( + options.expectedSnapshot === null + || options.expectedSnapshot.type !== snapshot.type + || !sameFile(options.expectedSnapshot.stats, snapshot.stats) + ) + ) { + throw new ContainedReadError("DOTAIOS_CONTEXT_SOURCE_CHANGED"); + } + return snapshot; +} + +/** Final-check each observed file once; shared directories are checked elsewhere. */ +export async function assertContainedPathEntrySnapshotsUnchanged(root, observations, options = {}) { + const concurrency = Math.max(1, Math.min(128, Number(options.concurrency) || 64)); + const unique = new Map(); + for (const observation of observations || []) { + const filePath = path.resolve(observation.filePath || observation.path); + if (!isPathWithinLexically(root, filePath) || !Object.hasOwn(observation, "snapshot")) { + throw new ContainedReadError(); + } + const retained = unique.get(filePath); + if (retained) { + const unchanged = retained.snapshot === null && observation.snapshot === null + || ( + retained.snapshot?.type === observation.snapshot?.type + && sameFile(retained.snapshot.stats, observation.snapshot.stats) + ); + if (!unchanged) throw new ContainedReadError("DOTAIOS_CONTEXT_SOURCE_CHANGED"); + continue; + } + unique.set(filePath, { filePath, snapshot: observation.snapshot }); + } + const values = [...unique.values()]; + for (let index = 0; index < values.length; index += concurrency) { + await Promise.all(values.slice(index, index + concurrency).map(async ({ filePath, snapshot }) => { + let current; + try { + current = await (options.filesystem || localFilesystem).lstat(filePath); + } catch (error) { + if ((error?.code === "ENOENT" || error?.code === "ENOTDIR") && snapshot === null) return; + if (error?.code === "ENOENT" || error?.code === "ENOTDIR") { + throw new ContainedReadError("DOTAIOS_CONTEXT_SOURCE_CHANGED"); + } + throw error; + } + if ( + snapshot === null + || snapshot.type !== (current.isSymbolicLink() ? "symbolic-link" : current.isFile() ? "regular-file" : "other") + || !sameFile(snapshot.stats, current) + ) { + throw new ContainedReadError("DOTAIOS_CONTEXT_SOURCE_CHANGED"); + } + })); + } +} + /** Snapshot regular-file metadata without opening or reading source bytes. */ export async function inspectContainedFileMetadata(root, filePath, options = {}) { const filesystem = options.filesystem || localFilesystem; @@ -379,11 +515,298 @@ export async function inspectContainedDirectory(root, directoryPath, options = { return snapshot?.stats || null; } +/** Snapshot the closest existing directory that bounds a missing contained tail. */ +export async function inspectNearestContainedDirectory(root, candidate, options = {}) { + const filesystem = options.filesystem || localFilesystem; + const resolvedRoot = path.resolve(root); + let current = path.resolve(candidate); + if (!isPathWithinLexically(resolvedRoot, current)) throw new ContainedReadError(); + await assertMissingTailContained(resolvedRoot, current, filesystem); + + while (isPathWithinLexically(resolvedRoot, current)) { + try { + const snapshot = await inspectContainedDirectorySnapshot(resolvedRoot, current, { filesystem }); + if (snapshot) return Object.freeze({ path: current, snapshot }); + } catch (error) { + if (error instanceof ContainedReadError) throw error; + if (error?.code !== "ENOENT" && error?.code !== "ENOTDIR") throw error; + } + if (current === resolvedRoot) break; + current = path.dirname(current); + } + throw new ContainedReadError("DOTAIOS_CONTEXT_SOURCE_CHANGED"); +} + /** Compare two contained directory observations from the same path. */ export function sameContainedDirectorySnapshot(left, right) { return sameDirectorySnapshot(left, right); } +/** + * Read one file that was accepted by a separately validated directory walk. + * + * This is intentionally narrower than readContainedFile: callers must supply + * the canonical authorized root and the exact parent-directory observation + * that admitted the path. It lets a request-scoped corpus transaction reuse + * traversal invariants while retaining a no-follow handle, expected-entry / + * handle identity, exact-parent identity, bounded bytes, UTF-8, and mutation + * checks for every file. Canonical containment is owned by the transaction's + * directory walk and final generation validation rather than repeated here. + */ +export async function readContainedSnapshotFile(root, filePath, options = {}) { + const filesystem = options.filesystem || localFilesystem; + const resolvedRoot = path.resolve(root); + const resolvedFile = path.resolve(filePath); + const resolvedParent = path.resolve(options.parentPath || path.dirname(resolvedFile)); + if ( + !isPathWithinLexically(resolvedRoot, resolvedFile) + || resolvedParent !== path.dirname(resolvedFile) + || !options.parentSnapshot?.stats + ) { + throw new ContainedReadError(); + } + if (typeof filesystem.open !== "function") { + throw new ContainedReadError("DOTAIOS_BOUNDED_FILE_READ_UNAVAILABLE"); + } + const noFollowFlag = Object.hasOwn(options, "noFollowFlag") + ? options.noFollowFlag + : fsConstants.O_NOFOLLOW; + if (!Number.isSafeInteger(noFollowFlag) || noFollowFlag <= 0) { + throw new ContainedReadError("DOTAIOS_BOUNDED_FILE_READ_UNAVAILABLE"); + } + + let handle; + try { + handle = await filesystem.open( + resolvedFile, + fsConstants.O_RDONLY + | noFollowFlag + | (fsConstants.O_NONBLOCK || 0) + ); + const opened = await handle.stat(); + const currentParent = await lstatAfterObservation(filesystem, resolvedParent, { bigint: true }); + const allowedRootParentLink = resolvedParent === resolvedRoot && currentParent.isSymbolicLink(); + if ( + !opened.isFile() + || (options.expectedSnapshot && ( + options.expectedSnapshot.type !== "regular-file" + || !sameFile(options.expectedSnapshot.stats, opened) + )) + || !sameBigIntFile(options.parentSnapshot.stats, currentParent) + || (!allowedRootParentLink && (!currentParent.isDirectory() || currentParent.isSymbolicLink())) + ) { + throw new ContainedReadError("DOTAIOS_CONTEXT_SOURCE_CHANGED"); + } + if (Number.isFinite(options.maxBytes) && opened.size > options.maxBytes) { + throw new ContainedReadError(options.tooLargeCode || "DOTAIOS_CONTEXT_SOURCE_TOO_LARGE"); + } + + options.budget?.reserveFile(opened.size); + const bytes = Number.isFinite(options.maxBytes) + ? await readBoundedHandle( + handle, + options.maxBytes, + options.tooLargeCode || "DOTAIOS_CONTEXT_SOURCE_TOO_LARGE", + opened.size + ) + : await handle.readFile(); + const completed = await handle.stat(); + if (!sameFile(opened, completed)) { + throw new ContainedReadError("DOTAIOS_CONTEXT_SOURCE_CHANGED"); + } + return Object.freeze({ + content: decodeContainedBytes(bytes, options.encoding), + stats: completed + }); + } catch (error) { + if (["ENOENT", "ENOTDIR", "ELOOP"].includes(error?.code)) { + throw new ContainedReadError("DOTAIOS_CONTEXT_SOURCE_CHANGED"); + } + throw error; + } finally { + if (handle) await handle.close().catch(() => {}); + } +} + +/** + * Enumerate one directory already admitted by a corpus transaction. Parent + * identity and canonical-root binding replace repeated full ancestor walks; + * before/open/after path observations still surround the portable opendir + * handle, whose identity Node does not expose directly. + */ +export async function readContainedSnapshotDirectory(root, directoryPath, options = {}) { + const filesystem = options.filesystem || localFilesystem; + const resolvedRoot = path.resolve(root); + const resolvedDirectory = path.resolve(directoryPath); + const canonicalRoot = path.resolve(options.canonicalRoot || root); + if (!isPathWithinLexically(resolvedRoot, resolvedDirectory)) { + throw new ContainedReadError(); + } + if (typeof filesystem.opendir !== "function") { + throw new ContainedReadError("DOTAIOS_BOUNDED_DIRECTORY_READ_UNAVAILABLE"); + } + + let directory; + try { + const before = await lstatAfterObservation(filesystem, resolvedDirectory, { bigint: true }); + const allowedRootLink = resolvedDirectory === resolvedRoot && before.isSymbolicLink(); + if ( + (!allowedRootLink && (!before.isDirectory() || before.isSymbolicLink())) + || (options.expectedSnapshot && !sameBigIntFile(options.expectedSnapshot.stats, before)) + ) { + throw new ContainedReadError("DOTAIOS_CONTEXT_SOURCE_CHANGED"); + } + directory = await filesystem.opendir(resolvedDirectory, { + encoding: options.nameEncoding || "utf8" + }); + const opened = await lstatAfterObservation(filesystem, resolvedDirectory, { bigint: true }); + if (!sameBigIntFile(before, opened)) { + throw new ContainedReadError("DOTAIOS_CONTEXT_SOURCE_CHANGED"); + } + + if (resolvedDirectory !== resolvedRoot) { + const resolvedParent = path.resolve(options.parentPath || path.dirname(resolvedDirectory)); + if (resolvedParent !== path.dirname(resolvedDirectory) || !options.parentSnapshot?.stats) { + throw new ContainedReadError(); + } + const currentParent = await lstatAfterObservation(filesystem, resolvedParent, { bigint: true }); + const allowedRootParentLink = resolvedParent === resolvedRoot && currentParent.isSymbolicLink(); + if ( + (!allowedRootParentLink && (!currentParent.isDirectory() || currentParent.isSymbolicLink())) + || !sameBigIntFile(options.parentSnapshot.stats, currentParent) + ) { + throw new ContainedReadError("DOTAIOS_CONTEXT_SOURCE_CHANGED"); + } + } + + let canonicalDirectory; + try { + canonicalDirectory = await filesystem.realpath(resolvedDirectory); + } catch (error) { + if (["ENOENT", "ENOTDIR", "ELOOP"].includes(error?.code)) { + throw new ContainedReadError("DOTAIOS_CONTEXT_SOURCE_CHANGED"); + } + throw error; + } + if (!isPathWithinLexically(canonicalRoot, canonicalDirectory)) { + throw new ContainedReadError("DOTAIOS_CONTEXT_SOURCE_CHANGED"); + } + + const entries = []; + while (true) { + const entry = await directory.read(); + if (!entry) break; + options.budget?.reserveEntries(1); + entries.push(options.readdirOptions?.withFileTypes ? entry : entry.name); + if (Number.isFinite(options.maxEntries) && entries.length > options.maxEntries) { + throw new ContainedReadError(options.tooManyCode || "DOTAIOS_DIRECTORY_LIMIT_EXCEEDED"); + } + } + const after = await lstatAfterObservation(filesystem, resolvedDirectory, { bigint: true }); + if (!sameBigIntFile(opened, after)) { + throw new ContainedReadError("DOTAIOS_CONTEXT_SOURCE_CHANGED"); + } + return { + entries, + snapshot: { + stats: after, + ancestors: options.expectedSnapshot?.ancestors || [] + } + }; + } catch (error) { + if (["ENOENT", "ENOTDIR", "ELOOP"].includes(error?.code)) { + throw new ContainedReadError("DOTAIOS_CONTEXT_SOURCE_CHANGED"); + } + throw error; + } finally { + if (directory) { + await directory.close().catch((error) => { + if (error?.code !== "ERR_DIR_CLOSED") throw error; + }); + } + } +} + +/** + * Revalidate one request's complete observed directory/ancestor generation. + * Paths shared by many files or directories are checked once, so this work is + * proportional to the observed directory set rather than to files multiplied + * by their ancestor depth. + */ +export async function assertContainedDirectorySnapshotsUnchanged(root, observations, options = {}) { + const filesystem = options.filesystem || localFilesystem; + const resolvedRoot = path.resolve(root); + const expectedPaths = new Map(); + + for (const observation of observations || []) { + const observedPath = path.resolve(observation.path); + if (!isPathWithinLexically(resolvedRoot, observedPath) || !observation.snapshot) { + throw new ContainedReadError(); + } + const observedRootAncestor = observedPath === resolvedRoot + ? observation.snapshot.ancestors?.find( + (ancestor) => path.resolve(ancestor.path) === resolvedRoot + ) + : null; + rememberExpectedDirectory(expectedPaths, observedPath, { + stats: observation.snapshot.stats, + resolvedPath: observedRootAncestor?.resolvedPath || observedPath, + resolvedStats: observedRootAncestor?.resolvedStats || observation.snapshot.stats + }); + for (const ancestor of observation.snapshot.ancestors || []) { + rememberExpectedDirectory(expectedPaths, ancestor.path, ancestor); + } + } + + for (const [expectedPath, expected] of expectedPaths) { + const actual = await lstatAfterObservation(filesystem, expectedPath, { bigint: true }); + if (!sameBigIntFile(expected.stats, actual)) { + throw new ContainedReadError("DOTAIOS_CONTEXT_SOURCE_CHANGED"); + } + if (expectedPath === resolvedRoot && actual.isSymbolicLink()) { + let actualResolvedPath; + try { + actualResolvedPath = await filesystem.realpath(expectedPath); + } catch (error) { + if (["ENOENT", "ENOTDIR", "ELOOP"].includes(error?.code)) { + throw new ContainedReadError("DOTAIOS_CONTEXT_SOURCE_CHANGED"); + } + throw error; + } + const actualResolvedStats = await lstatAfterObservation( + filesystem, + actualResolvedPath, + { bigint: true } + ); + if ( + path.resolve(actualResolvedPath) !== path.resolve(expected.resolvedPath) + || !sameBigIntFile(expected.resolvedStats, actualResolvedStats) + || !actualResolvedStats.isDirectory() + || actualResolvedStats.isSymbolicLink() + ) { + throw new ContainedReadError("DOTAIOS_CONTEXT_SOURCE_CHANGED"); + } + continue; + } + if (!actual.isDirectory() || actual.isSymbolicLink()) { + throw new ContainedReadError("DOTAIOS_CONTEXT_SOURCE_CHANGED"); + } + } +} + +function rememberExpectedDirectory(expectedPaths, expectedPath, expected) { + const resolvedPath = path.resolve(expectedPath); + const current = expectedPaths.get(resolvedPath); + if (current && ( + path.resolve(current.resolvedPath) !== path.resolve(expected.resolvedPath) + || !sameBigIntFile(current.stats, expected.stats) + || !sameBigIntFile(current.resolvedStats, expected.resolvedStats) + )) { + throw new ContainedReadError("DOTAIOS_CONTEXT_SOURCE_CHANGED"); + } + if (!current) expectedPaths.set(resolvedPath, expected); +} + async function inspectContainedDirectorySnapshot(root, directoryPath, options = {}) { const filesystem = options.filesystem || localFilesystem; if (!isPathWithinLexically(root, directoryPath)) throw new ContainedReadError(); @@ -400,7 +823,10 @@ async function inspectContainedDirectorySnapshot(root, directoryPath, options = if (!await isPathWithin(root, directoryPath, { fileSystem: filesystem })) { throw new ContainedReadError(); } - if (!before.isDirectory() || before.isSymbolicLink()) throw new ContainedReadError(); + const allowedRootLink = path.resolve(directoryPath) === path.resolve(root) && before.isSymbolicLink(); + if (!allowedRootLink && (!before.isDirectory() || before.isSymbolicLink())) { + throw new ContainedReadError(); + } const ancestors = await inspectContainedAncestors(root, directoryPath, filesystem); const confirmed = await lstatAfterObservation(filesystem, directoryPath, { bigint: true }); if (!sameBigIntFile(before, confirmed)) { diff --git a/packages/core/src/evidence-reader.mjs b/packages/core/src/evidence-reader.mjs index 303e665f..dbed1374 100644 --- a/packages/core/src/evidence-reader.mjs +++ b/packages/core/src/evidence-reader.mjs @@ -2,12 +2,20 @@ import localFilesystem from "node:fs/promises"; import path from "node:path"; import { + assertContainedPathEntrySnapshotsUnchanged, + assertContainedDirectorySnapshotsUnchanged, ContainedReadError, createContainedReadBudget, inspectContainedDirectory, + inspectNearestContainedDirectory, inspectContainedPathEntry, + inspectContainedSnapshotPathEntry, readContainedDirectory, - readContainedFile + readContainedFile, + readContainedSnapshotDirectory, + readContainedSnapshotFile, + sameContainedDirectorySnapshot, + sameContainedFileMetadataSnapshot } from "./contained-read.mjs"; import { isPathWithinLexically } from "./paths.mjs"; @@ -50,9 +58,25 @@ const EVIDENCE_READ_FAILURES = { + "Split that file, or move it out of the folder.", DOTAIOS_EVIDENCE_DIRECTORY_TOO_LARGE: "DotAIOS stopped reading: one directory holds more entries than the safe limit. " - + "Move some of its entries elsewhere." + + "Move some of its entries elsewhere.", + DOTAIOS_EVIDENCE_BYTE_BUDGET_EXCEEDED: + "DotAIOS stopped reading: this folder is past the safe aggregate byte limit for one request.", + DOTAIOS_EVIDENCE_FILE_COUNT_EXCEEDED: + "DotAIOS stopped reading: this folder contains more files than one request can safely inspect.", + DOTAIOS_EVIDENCE_ENTRY_COUNT_EXCEEDED: + "DotAIOS stopped reading: this folder contains more entries than one request can safely inspect." }; +const SKIPPABLE_SCOPE_CODES = new Set([ + "DOTAIOS_EVIDENCE_FILE_TOO_LARGE", + "DOTAIOS_EVIDENCE_DIRECTORY_TOO_LARGE", + "DOTAIOS_EVIDENCE_BYTE_BUDGET_EXCEEDED", + "DOTAIOS_EVIDENCE_FILE_COUNT_EXCEEDED", + "DOTAIOS_EVIDENCE_ENTRY_COUNT_EXCEEDED" +]); + +const OMITTED_SCOPE_LIMIT = 32; + export class EvidenceReadError extends Error { constructor(code = "DOTAIOS_EVIDENCE_READ_FAILED") { super(EVIDENCE_READ_FAILURES[code] || "DotAIOS could not read the evidence corpus safely."); @@ -82,7 +106,19 @@ export function createEvidenceReader({ roots, filesystem = localFilesystem, limi } function createEvidenceReaderView(roots, state) { - const { filesystem, effectiveLimits, budget, observedDirectories } = state; + const { + filesystem, + effectiveLimits, + budget, + observedDirectories, + collectRequestObservations = false, + dimensionCodes = false, + scopeDiscovery = false, + executionBudget = budget, + demandBudget = budget, + observedFiles = new Map(), + transactionState = null + } = state; const authorizedRoots = [...new Set((roots || []).map((root) => path.resolve(root)))]; if (authorizedRoots.length === 0) throw new TypeError("Evidence readers require an authorized root."); @@ -98,6 +134,7 @@ function createEvidenceReaderView(roots, state) { throw new EvidenceReadError("DOTAIOS_EVIDENCE_PATH_UNSAFE"); } try { + await rememberReadParent(authorizedRoot, filePath); return await readContainedFile(authorizedRoot, filePath, { filesystem, encoding: "utf8", @@ -105,6 +142,7 @@ function createEvidenceReaderView(roots, state) { maxBytes: options.maxBytes ?? effectiveLimits.maxFileBytes, tooLargeCode: "DOTAIOS_EVIDENCE_FILE_TOO_LARGE", returnSnapshot: options.returnSnapshot === true, + expectedSnapshot: options.expectedEntry, expectedDirectories: expectedDirectoriesFor(authorizedRoot, filePath) }); } catch (error) { @@ -133,6 +171,106 @@ function createEvidenceReaderView(roots, state) { return entries; } + async function prepareJsonl(root, filePath, options = {}) { + return materializePreparedJsonl(await prepareJsonlMetadata(root, filePath, options)); + } + + async function prepareJsonlMetadata(root, filePath, options = {}) { + const prepared = await prepareTextMetadata(root, filePath, options); + return Object.freeze({ kind: "jsonl-metadata", prepared }); + } + + async function materializePreparedJsonl(metadata) { + if (metadata?.kind !== "jsonl-metadata") { + throw new TypeError("Prepared JSONL metadata is invalid."); + } + const prepared = await materializePreparedText(metadata.prepared); + if (prepared.content === null) return Object.freeze({ kind: "jsonl", entries: [] }); + const entries = []; + for (const line of prepared.content.split("\n")) { + if (!line.trim()) continue; + try { + reserveDiscoveredEntries(1); + } catch (error) { + throw normalizeEvidenceReadError(error); + } + try { + entries.push(JSON.parse(line)); + } catch { + // Keep corrupt canonical lines out of this derived search view. + } + } + return Object.freeze({ kind: "jsonl", entries }); + } + + function readPreparedJsonl(prepared) { + if (prepared?.kind !== "jsonl" || !Array.isArray(prepared.entries)) { + throw new TypeError("Prepared JSONL evidence is invalid."); + } + return prepared.entries; + } + + async function prepareTextContent(root, filePath, options = {}) { + return materializePreparedText(await prepareTextMetadata(root, filePath, options)); + } + + async function prepareTextMetadata(root, filePath, options = {}) { + if (!scopeDiscovery) throw new EvidenceReadError("DOTAIOS_EVIDENCE_TRANSACTION_CLOSED"); + const authorizedRoot = assertAuthorizedRoot(root); + await rememberReadParent(authorizedRoot, filePath); + const expectedEntry = await inspectEntry(root, filePath); + if (expectedEntry?.type === "symbolic-link") { + throw new EvidenceReadError("DOTAIOS_EVIDENCE_PATH_UNSAFE"); + } + if (expectedEntry && expectedEntry.type !== "regular-file") { + throw new EvidenceReadError("DOTAIOS_EVIDENCE_NOT_REGULAR_FILE"); + } + if (expectedEntry && expectedEntry.stats.size > (options.maxBytes ?? effectiveLimits.maxFileBytes)) { + throw new EvidenceReadError("DOTAIOS_EVIDENCE_FILE_TOO_LARGE"); + } + if (expectedEntry) { + try { + demandBudget.reserveFile(expectedEntry.stats.size); + } catch (error) { + throw normalizeEvidenceReadError(error); + } + } + return Object.freeze({ + kind: "text-metadata", + root: authorizedRoot, + filePath: path.resolve(filePath), + expectedEntry, + maxBytes: options.maxBytes ?? effectiveLimits.maxFileBytes + }); + } + + async function materializePreparedText(metadata) { + if (metadata?.kind !== "text-metadata") { + throw new TypeError("Prepared text metadata is invalid."); + } + if (!metadata.expectedEntry) return Object.freeze({ kind: "text", content: null }); + let content; + try { + content = await readContainedFile(metadata.root, metadata.filePath, { + filesystem, + encoding: "utf8", + budget: executionBudget, + maxBytes: metadata.maxBytes, + tooLargeCode: "DOTAIOS_EVIDENCE_FILE_TOO_LARGE", + expectedSnapshot: metadata.expectedEntry, + expectedDirectories: expectedDirectoriesFor(metadata.root, metadata.filePath) + }); + } catch (error) { + throw normalizeEvidenceReadError(error); + } + return Object.freeze({ kind: "text", content }); + } + + function readPreparedText(prepared) { + if (prepared?.kind !== "text") throw new TypeError("Prepared text evidence is invalid."); + return prepared.content; + } + async function readJson(root, filePath, options = {}) { const content = await readText(root, filePath, options); if (content === null) { @@ -185,13 +323,15 @@ function createEvidenceReaderView(roots, state) { throw new EvidenceReadError("DOTAIOS_EVIDENCE_PATH_UNSAFE"); } try { - return await inspectContainedPathEntry(authorizedRoot, filePath, { + const observation = await inspectContainedPathEntry(authorizedRoot, filePath, { filesystem, expectedDirectories: expectedDirectoriesFor(authorizedRoot, filePath), ...(Object.hasOwn(options, "expectedEntry") ? { expectedSnapshot: options.expectedEntry } : {}) }); + rememberFileObservation(authorizedRoot, filePath, observation); + return observation; } catch (error) { throw normalizeEvidenceReadError(error); } @@ -246,6 +386,17 @@ function createEvidenceReaderView(roots, state) { throw new EvidenceReadError("DOTAIOS_EVIDENCE_PATH_UNSAFE"); } try { + if (collectRequestObservations) { + const before = await inspectContainedDirectory(authorizedRoot, directoryPath, { + filesystem, + returnSnapshot: true + }); + if (before !== null) rememberDirectory(authorizedRoot, directoryPath, before); + if (before === null) return []; + if (scopeDiscovery && budget.remaining().entries === 0) { + throw new ContainedReadError("DOTAIOS_PROJECTION_ENTRY_COUNT_EXCEEDED"); + } + } const observed = await readContainedDirectory(authorizedRoot, directoryPath, { filesystem, budget, @@ -263,7 +414,307 @@ function createEvidenceReaderView(roots, state) { } } + /** + * Enumerate and read one text corpus inside an evidence-reader-owned + * transaction. The callback may derive any request result it needs, but the + * outer promise cannot resolve successfully until the complete observed + * root/directory/ancestor generation has been revalidated. + */ + async function withTextCorpus(root, directoryPath, options, callback) { + const prepared = await observeTextCorpus(root, directoryPath, options, false); + return withPreparedTextCorpus(prepared, callback); + } + + async function prepareTextCorpus(root, directoryPath, options) { + if (!scopeDiscovery) { + throw new EvidenceReadError("DOTAIOS_EVIDENCE_TRANSACTION_CLOSED"); + } + return observeTextCorpus(root, directoryPath, options, true); + } + + async function observeTextCorpus(root, directoryPath, options, measureFiles) { + const authorizedRoot = assertAuthorizedRoot(root); + if (!isPathWithinLexically(authorizedRoot, directoryPath)) { + throw new EvidenceReadError("DOTAIOS_EVIDENCE_PATH_UNSAFE"); + } + if (!options || typeof options !== "object" || Array.isArray(options)) { + throw new TypeError("Evidence corpus options must be an object."); + } + let canonicalRoot; + let observation; + try { + const rootSnapshot = await inspectContainedDirectory(authorizedRoot, authorizedRoot, { + filesystem, + returnSnapshot: true + }); + if (rootSnapshot === null) { + throw new ContainedReadError("DOTAIOS_CONTEXT_SOURCE_CHANGED"); + } + canonicalRoot = await filesystem.realpath(authorizedRoot); + observation = { + directories: [{ path: authorizedRoot, snapshot: rootSnapshot }], + files: [] + }; + rememberDirectory(authorizedRoot, authorizedRoot, rootSnapshot); + const resolvedDirectory = path.resolve(directoryPath); + const startingSnapshot = resolvedDirectory === authorizedRoot + ? rootSnapshot + : await inspectContainedDirectory(authorizedRoot, resolvedDirectory, { + filesystem, + returnSnapshot: true + }); + if (startingSnapshot !== null) { + rememberDirectory(authorizedRoot, resolvedDirectory, startingSnapshot); + await walkTextCorpus( + authorizedRoot, + resolvedDirectory, + options, + observation, + { + canonicalRoot, + expectedSnapshot: startingSnapshot, + measureFiles, + ...containedParentObservation(resolvedDirectory, authorizedRoot, startingSnapshot) + } + ); + } else { + const nearest = await inspectNearestContainedDirectory( + authorizedRoot, + resolvedDirectory, + { filesystem } + ); + rememberCorpusDirectory(observation, nearest.path, nearest.snapshot); + rememberDirectory(authorizedRoot, nearest.path, nearest.snapshot); + } + observation.files.sort((left, right) => left.filePath.localeCompare(right.filePath)); + } catch (error) { + throw normalizeEvidenceReadError(error); + } + + return Object.freeze({ + root: authorizedRoot, + canonicalRoot, + observation, + options: Object.freeze({ ...options }) + }); + } + + async function withPreparedTextCorpus(prepared, callback) { + if (!prepared?.observation || !prepared?.canonicalRoot || !prepared?.root) { + throw new TypeError("Evidence corpus preparation is invalid."); + } + if (typeof callback !== "function") { + throw new TypeError("Evidence corpus transactions require a callback."); + } + const { root: authorizedRoot, observation, options } = prepared; + assertAuthorizedRoot(authorizedRoot); + let active = true; + let consumed = false; + let mappingPromise = null; + const completedFiles = new Map(); + const transaction = Object.freeze({ + mapFiles(mapper) { + if (!active) throw new EvidenceReadError("DOTAIOS_EVIDENCE_TRANSACTION_CLOSED"); + if (consumed) throw new EvidenceReadError("DOTAIOS_EVIDENCE_TRANSACTION_CONSUMED"); + if (typeof mapper !== "function") { + throw new TypeError("Evidence corpus mapping requires a callback."); + } + consumed = true; + mappingPromise = mapObservedTextFiles( + authorizedRoot, + observation.files, + options, + mapper, + executionBudget, + (filePath, snapshot) => { + completedFiles.set(path.resolve(filePath), snapshot); + rememberFileObservation(authorizedRoot, filePath, snapshot); + } + ); + return mappingPromise; + } + }); + + let result; + let callbackError; + try { + result = await callback(transaction); + } catch (error) { + callbackError = error; + } + active = false; + if (mappingPromise) { + try { + await mappingPromise; + } catch (error) { + if (!callbackError) callbackError = error; + } + } + if (callbackError) throw callbackError; + try { + if (!collectRequestObservations) { + await assertContainedPathEntrySnapshotsUnchanged( + authorizedRoot, + [...completedFiles].map(([filePath, snapshot]) => ({ filePath, snapshot })), + { filesystem } + ); + } + await assertContainedDirectorySnapshotsUnchanged( + authorizedRoot, + observation.directories, + { filesystem } + ); + return result; + } catch (error) { + throw normalizeEvidenceReadError(error); + } + } + + async function walkTextCorpus(root, directoryPath, options, observation, containment) { + if (containment.expectedSnapshot) { + rememberDirectory(root, directoryPath, containment.expectedSnapshot); + } + if (scopeDiscovery && budget.remaining().entries === 0) { + throw new EvidenceReadError("DOTAIOS_EVIDENCE_ENTRY_COUNT_EXCEEDED"); + } + const observed = await readContainedSnapshotDirectory(root, directoryPath, { + filesystem, + canonicalRoot: containment.canonicalRoot, + expectedSnapshot: containment.expectedSnapshot, + parentPath: containment.parentPath, + parentSnapshot: containment.parentSnapshot, + budget, + maxEntries: options.maxDirectoryEntries ?? effectiveLimits.maxDirectoryEntries, + tooManyCode: "DOTAIOS_EVIDENCE_DIRECTORY_TOO_LARGE", + readdirOptions: { withFileTypes: true }, + returnSnapshot: true + }); + if (observed === null) return; + rememberCorpusDirectory(observation, directoryPath, observed.snapshot); + + const entries = [...observed.entries] + .sort((left, right) => left.name.localeCompare(right.name)); + for (const entry of entries) { + if (options.skipEntry?.(entry.name)) continue; + const filePath = path.join(directoryPath, entry.name); + if (entry.isSymbolicLink()) { + const acceptedLink = options.includeFile + ? options.includeFile(filePath) + : !options.extensions || options.extensions.includes(path.extname(entry.name).toLowerCase()); + if (acceptedLink) throw new EvidenceReadError("DOTAIOS_EVIDENCE_PATH_UNSAFE"); + continue; + } + if (entry.isDirectory()) { + if (options.recursive !== false) { + const childSnapshot = await inspectContainedDirectory(root, filePath, { + filesystem, + returnSnapshot: true + }); + if (childSnapshot === null) throw new EvidenceReadError("DOTAIOS_EVIDENCE_CHANGED"); + await walkTextCorpus(root, filePath, options, observation, { + canonicalRoot: containment.canonicalRoot, + expectedSnapshot: childSnapshot, + measureFiles: containment.measureFiles, + parentPath: path.resolve(directoryPath), + parentSnapshot: observed.snapshot + }); + } + continue; + } + + const accepted = options.includeFile + ? options.includeFile(filePath) + : !options.extensions || options.extensions.includes(path.extname(entry.name).toLowerCase()); + if (!accepted) continue; + if (!entry.isFile()) throw new EvidenceReadError("DOTAIOS_EVIDENCE_NOT_REGULAR_FILE"); + if (observation.files.length >= effectiveLimits.maxFiles) { + throw new EvidenceReadError(dimensionCodes + ? "DOTAIOS_EVIDENCE_FILE_COUNT_EXCEEDED" + : "DOTAIOS_EVIDENCE_BUDGET_EXCEEDED"); + } + let expectedEntry = null; + if (containment.measureFiles) { + expectedEntry = await inspectContainedSnapshotPathEntry(root, filePath, { + filesystem, + parentPath: path.resolve(directoryPath), + parentSnapshot: observed.snapshot + }); + rememberFileObservation(root, filePath, expectedEntry); + if (!expectedEntry || expectedEntry.type !== "regular-file") { + throw new EvidenceReadError("DOTAIOS_EVIDENCE_NOT_REGULAR_FILE"); + } + if (expectedEntry.stats.size > (options.maxBytes ?? effectiveLimits.maxFileBytes)) { + throw new EvidenceReadError("DOTAIOS_EVIDENCE_FILE_TOO_LARGE"); + } + try { + demandBudget.reserveFile(expectedEntry.stats.size); + } catch (error) { + throw normalizeEvidenceReadError(error); + } + } + observation.files.push(Object.freeze({ + filePath, + parentPath: path.resolve(directoryPath), + parentSnapshot: observed.snapshot, + expectedEntry + })); + } + } + + async function mapObservedTextFiles( + root, + files, + options, + mapper, + readBudget = budget, + observeCompleted = () => {} + ) { + const concurrency = normalizeCorpusConcurrency(options.concurrency); + const mapped = new Array(files.length); + for (let index = 0; index < files.length; index += concurrency) { + const batch = files.slice(index, index + concurrency); + const values = await Promise.all(batch.map(async (file) => { + let observed; + try { + observed = await readContainedSnapshotFile(root, file.filePath, { + filesystem, + parentPath: file.parentPath, + parentSnapshot: file.parentSnapshot, + expectedSnapshot: file.expectedEntry, + encoding: "utf8", + budget: readBudget, + maxBytes: options.maxBytes ?? effectiveLimits.maxFileBytes, + tooLargeCode: "DOTAIOS_EVIDENCE_FILE_TOO_LARGE" + }); + } catch (error) { + throw normalizeEvidenceReadError(error); + } + observeCompleted(file.filePath, Object.freeze({ + type: "regular-file", + stats: observed.stats, + ancestors: [] + })); + return mapper(Object.freeze({ + filePath: file.filePath, + content: observed.content, + mtimeMs: observed.stats.mtimeMs + })); + })); + for (const [offset, value] of values.entries()) mapped[index + offset] = value; + } + return mapped; + } + async function walkDirectory(root, directoryPath, files, options) { + if (scopeDiscovery && budget.remaining().entries === 0) { + const existing = await inspectContainedDirectory(root, directoryPath, { + filesystem, + returnSnapshot: true + }); + if (existing === null) return; + rememberDirectory(root, directoryPath, existing); + throw new EvidenceReadError("DOTAIOS_EVIDENCE_ENTRY_COUNT_EXCEEDED"); + } const observed = await readContainedDirectory(root, directoryPath, { filesystem, budget, @@ -302,36 +753,358 @@ function createEvidenceReaderView(roots, state) { } } + /** + * Discover scope demand under one non-releasable physical ledger, allocate + * the remaining work deterministically, then execute only admitted scopes. + * Prepared values remain callback-scoped and cannot resolve successfully + * until every observed file/root/directory generation is revalidated. + */ + async function withScopePreflight(scopes, inspectScope, discoverScope, executeScope, callback) { + if (!Array.isArray(scopes) || scopes.length === 0 || scopes.some((scope) => typeof scope !== "string" || !scope)) { + throw new TypeError("Evidence scope preflight requires named logical scopes."); + } + if (new Set(scopes).size !== scopes.length) { + throw new TypeError("Evidence scope preflight requires unique logical scopes."); + } + if ( + typeof inspectScope !== "function" + || typeof discoverScope !== "function" + || typeof executeScope !== "function" + || typeof callback !== "function" + ) { + throw new TypeError("Evidence scope preflight requires inspection, discovery, execution, and result callbacks."); + } + + const available = budget.remaining(); + const scopeLimits = Object.freeze({ + ...effectiveLimits, + maxBytes: available.bytes, + maxFiles: available.files, + maxEntries: available.entries + }); + const prepared = scopes.map((scope) => { + const scopeDemandBudget = createContainedReadBudget({ + maxBytes: scopeLimits.maxBytes, + maxFiles: scopeLimits.maxFiles, + maxEntries: scopeLimits.maxEntries, + dimensionCodes: true + }); + return { + scope, + value: null, + demandBudget: scopeDemandBudget, + materialized: emptyDemand(), + error: null, + reader: null, + observedDirectories: new Map(), + observedFiles: new Map(), + transactionState: { active: true } + }; + }); + const createScopeReader = (item, phaseBudget = null) => createEvidenceReaderView(authorizedRoots, { + filesystem, + effectiveLimits: scopeLimits, + budget: phaseBudget + ? combineContainedReadBudgets(budget, item.demandBudget, phaseBudget) + : combineContainedReadBudgets(budget, item.demandBudget), + executionBudget: phaseBudget + ? combineContainedReadBudgets(budget, phaseBudget) + : budget, + demandBudget: item.demandBudget, + observedDirectories: item.observedDirectories, + observedFiles: item.observedFiles, + transactionState: item.transactionState, + collectRequestObservations: true, + dimensionCodes: true, + scopeDiscovery: true + }); + + try { + const inspectionPhases = createFairPhaseBudgets(budget.remaining(), prepared.length); + for (const item of prepared) { + const phase = inspectionPhases.next(); + const beforeInspection = budget.snapshot(); + item.reader = createScopeReader(item, phase.budget); + try { + item.value = await inspectScope(item.scope, item.reader); + } catch (error) { + item.error = normalizeEvidenceReadError(error); + } finally { + phase.settle(); + } + item.materialized = subtractDemand(budget.snapshot(), beforeInspection); + } + + let fatal = prepared.find(({ error }) => error && !SKIPPABLE_SCOPE_CODES.has(error.code)); + if (fatal) throw fatal.error; + + const preliminary = allocateScopeDemands( + prepared.map((item) => ({ + ...item, + observedDemand: item.demandBudget.snapshot(), + demand: subtractDemand(item.demandBudget.snapshot(), item.materialized) + })), + budget.remaining(), + available + ); + + const candidates = prepared.filter((item) => preliminary.admitted.has(item.scope)); + const discoveryPhases = createFairPhaseBudgets(budget.remaining(), candidates.length); + for (const item of candidates) { + const phase = discoveryPhases.next(); + const beforeDiscovery = budget.snapshot(); + item.reader = createScopeReader(item, phase.budget); + try { + item.value = await discoverScope(item.scope, item.reader, item.value); + } catch (error) { + item.error = normalizeEvidenceReadError(error); + } finally { + phase.settle(); + } + item.materialized = addDemand( + item.materialized, + subtractDemand(budget.snapshot(), beforeDiscovery) + ); + } + + fatal = prepared.find(({ error }) => error && !SKIPPABLE_SCOPE_CODES.has(error.code)); + if (fatal) throw fatal.error; + + for (const item of prepared) { + for (const [key, observation] of item.reader.observations()) { + const retained = observedDirectories.get(key); + if (retained && !sameContainedDirectorySnapshot(retained.snapshot, observation.snapshot)) { + throw new EvidenceReadError("DOTAIOS_EVIDENCE_CHANGED"); + } + if (!retained) observedDirectories.set(key, observation); + } + } + + const finalAllocation = allocateScopeDemands( + candidates.map((item) => ({ + ...item, + observedDemand: item.demandBudget.snapshot(), + demand: subtractDemand(item.demandBudget.snapshot(), item.materialized) + })), + budget.remaining(), + available + ); + const admitted = new Map(); + const executionOmissions = []; + for (const item of candidates) { + if (!finalAllocation.admitted.has(item.scope)) continue; + item.reader = createScopeReader(item); + try { + admitted.set(item.scope, await executeScope(item.scope, item.reader, item.value)); + } catch (error) { + const normalized = normalizeEvidenceReadError(error); + if (!SKIPPABLE_SCOPE_CODES.has(normalized.code)) throw normalized; + executionOmissions.push(createScopeOmission( + item.scope, + omissionReasonForError(normalized.code), + item.demandBudget.snapshot(), + scopeLimits + )); + } + } + const omissions = freezeOmissions([ + ...preliminary.omissions, + ...finalAllocation.omissions, + ...executionOmissions + ]); + let active = true; + const transaction = Object.freeze({ + has(scope) { + if (!active) throw new EvidenceReadError("DOTAIOS_EVIDENCE_TRANSACTION_CLOSED"); + return admitted.has(scope); + }, + get(scope) { + if (!active) throw new EvidenceReadError("DOTAIOS_EVIDENCE_TRANSACTION_CLOSED"); + return admitted.get(scope); + }, + omissions + }); + + let result; + try { + result = await callback(transaction); + } finally { + active = false; + } + try { + await revalidateObservedFiles(prepared); + await revalidateObservedDirectories(); + return result; + } catch (error) { + throw normalizeEvidenceReadError(error); + } + } finally { + for (const item of prepared) item.transactionState.active = false; + } + } + return Object.freeze({ roots: Object.freeze([...authorizedRoots]), withAuthorizedRoots(additionalRoots) { + assertViewActive(); const additions = Array.isArray(additionalRoots) ? additionalRoots : [additionalRoots]; if (additions.some((root) => typeof root !== "string" || root.length === 0)) { throw new TypeError("Authorized evidence roots must be non-empty paths."); } return createEvidenceReaderView([...authorizedRoots, ...additions], state); }, - readText, - readJson, - readJsonl, - readFrontmatter, - inspectEntry, - listFiles, - listDirectories, - listDirectory, - snapshot: () => budget.snapshot() + readText: guardReaderOperation(readText), + readJson: guardReaderOperation(readJson), + readJsonl: guardReaderOperation(readJsonl), + prepareJsonl: guardReaderOperation(prepareJsonl), + prepareJsonlMetadata: guardReaderOperation(prepareJsonlMetadata), + materializePreparedJsonl: guardReaderOperation(materializePreparedJsonl), + readPreparedJsonl: guardReaderOperation(readPreparedJsonl), + prepareTextContent: guardReaderOperation(prepareTextContent), + prepareTextMetadata: guardReaderOperation(prepareTextMetadata), + materializePreparedText: guardReaderOperation(materializePreparedText), + readPreparedText: guardReaderOperation(readPreparedText), + readFrontmatter: guardReaderOperation(readFrontmatter), + inspectEntry: guardReaderOperation(inspectEntry), + listFiles: guardReaderOperation(listFiles), + listDirectories: guardReaderOperation(listDirectories), + listDirectory: guardReaderOperation(listDirectory), + withTextCorpus: guardReaderOperation(withTextCorpus), + prepareTextCorpus: guardReaderOperation(prepareTextCorpus), + withPreparedTextCorpus: guardReaderOperation(withPreparedTextCorpus), + withScopePreflight: guardReaderOperation(withScopePreflight), + observations: guardReaderOperation(() => new Map(observedDirectories)), + fileObservations: guardReaderOperation(() => [...observedFiles.values()]), + snapshot: guardReaderOperation(() => budget.snapshot()) }); + function assertViewActive() { + if (transactionState && !transactionState.active) { + throw new EvidenceReadError("DOTAIOS_EVIDENCE_TRANSACTION_CLOSED"); + } + } + + function guardReaderOperation(operation) { + return (...args) => { + assertViewActive(); + return operation(...args); + }; + } + + function reserveDiscoveredEntries(count) { + if (executionBudget.remaining().entries < count) { + throw new ContainedReadError("DOTAIOS_PROJECTION_ENTRY_COUNT_EXCEEDED"); + } + if (demandBudget.remaining().entries < count) { + throw new ContainedReadError("DOTAIOS_PROJECTION_ENTRY_COUNT_EXCEEDED"); + } + demandBudget.reserveEntries(count); + executionBudget.reserveEntries(count); + } + function rememberDirectory(root, directoryPath, snapshot) { const resolvedRoot = path.resolve(root); const resolvedDirectory = path.resolve(directoryPath); - observedDirectories.set(`${resolvedRoot}\0${resolvedDirectory}`, { + const key = `${resolvedRoot}\0${resolvedDirectory}`; + const retained = observedDirectories.get(key); + if (retained) { + if (collectRequestObservations && !sameContainedDirectorySnapshot(retained.snapshot, snapshot)) { + throw new EvidenceReadError("DOTAIOS_EVIDENCE_CHANGED"); + } + return; + } + observedDirectories.set(key, { root: resolvedRoot, path: resolvedDirectory, snapshot }); } + function rememberFileObservation(root, filePath, expectedEntry) { + if (!collectRequestObservations) return; + const resolvedRoot = path.resolve(root); + const resolvedFile = path.resolve(filePath); + const key = `${resolvedRoot}\0${resolvedFile}`; + const retained = observedFiles.get(key); + if (retained) { + const unchanged = (retained.expectedEntry === null && expectedEntry === null) + || ( + retained.expectedEntry?.type === expectedEntry?.type + && sameContainedFileMetadataSnapshot(retained.expectedEntry, expectedEntry) + ); + if (!unchanged) throw new EvidenceReadError("DOTAIOS_EVIDENCE_CHANGED"); + return; + } + observedFiles.set(key, Object.freeze({ + root: resolvedRoot, + filePath: resolvedFile, + expectedEntry + })); + } + + async function rememberReadParent(root, filePath) { + if (!collectRequestObservations) return; + const parentPath = path.dirname(path.resolve(filePath)); + const key = `${path.resolve(root)}\0${parentPath}`; + if (observedDirectories.has(key)) return; + const snapshot = await inspectContainedDirectory(root, parentPath, { + filesystem, + returnSnapshot: true + }); + if (snapshot !== null) { + rememberDirectory(root, parentPath, snapshot); + return; + } + const nearest = await inspectNearestContainedDirectory(root, parentPath, { filesystem }); + rememberDirectory(root, nearest.path, nearest.snapshot); + } + + async function revalidateObservedDirectories() { + for (const root of authorizedRoots) { + const directories = [...observedDirectories.values()] + .filter((entry) => entry.root === root) + .map(({ path: directoryPath, snapshot }) => ({ path: directoryPath, snapshot })); + if (directories.length > 0) { + await assertContainedDirectorySnapshotsUnchanged(root, directories, { filesystem }); + } + } + } + + async function revalidateObservedFiles(scopeItems) { + for (const root of authorizedRoots) { + const files = new Map(); + for (const item of scopeItems) { + for (const observation of item.reader.fileObservations()) { + if (observation.root !== root) continue; + const retained = files.get(observation.filePath); + if ( + retained + && !( + retained.expectedEntry === null && observation.expectedEntry === null + || ( + retained.expectedEntry?.type === observation.expectedEntry?.type + && sameContainedFileMetadataSnapshot(retained.expectedEntry, observation.expectedEntry) + ) + ) + ) { + throw new EvidenceReadError("DOTAIOS_EVIDENCE_CHANGED"); + } + if (!retained) files.set(observation.filePath, observation); + } + } + if (files.size > 0) { + await assertContainedPathEntrySnapshotsUnchanged( + root, + [...files.values()].map(({ filePath, expectedEntry }) => ({ + filePath, + snapshot: expectedEntry + })), + { filesystem } + ); + } + } + } + function expectedDirectoriesFor(root, filePath) { const resolvedRoot = path.resolve(root); const resolvedFile = path.resolve(filePath); @@ -341,6 +1114,298 @@ function createEvidenceReaderView(roots, state) { } } +function allocateScopeDemands(prepared, capacity, reportedCapacity = capacity) { + const requestedCount = prepared.length; + const protectedShare = { + bytes: Math.floor(Math.floor(capacity.bytes / 2) / requestedCount), + files: Math.floor(Math.floor(capacity.files / 2) / requestedCount), + entries: Math.floor(Math.floor(capacity.entries / 2) / requestedCount) + }; + const reservations = new Map(); + let pool = { ...capacity }; + + for (const item of prepared) { + if (item.error) continue; + const protectedDemand = minDemand(item.demand, protectedShare); + reservations.set(item.scope, protectedDemand); + pool = subtractDemand(pool, protectedDemand); + } + + const admitted = new Set(); + for (const item of prepared) { + if (item.error) continue; + const reserved = reservations.get(item.scope) || emptyDemand(); + const remainder = subtractDemand(item.demand, reserved); + if (!fitsDemand(remainder, pool)) continue; + reservations.set(item.scope, item.demand); + pool = subtractDemand(pool, remainder); + admitted.add(item.scope); + } + + for (const item of prepared) { + if (item.error || admitted.has(item.scope)) continue; + pool = addDemand(pool, reservations.get(item.scope) || emptyDemand()); + reservations.delete(item.scope); + } + + for (const item of prepared) { + if (item.error || admitted.has(item.scope) || !fitsDemand(item.demand, pool)) continue; + pool = subtractDemand(pool, item.demand); + reservations.set(item.scope, item.demand); + admitted.add(item.scope); + } + + const omissions = []; + for (const item of prepared) { + if (admitted.has(item.scope)) continue; + const reason = item.error + ? omissionReasonForError(item.error.code) + : firstExceededDimension(item.demand, pool); + omissions.push(createScopeOmission( + item.scope, + reason, + item.observedDemand || item.demand, + reportedCapacity + )); + } + return Object.freeze({ admitted, omissions: Object.freeze(omissions) }); +} + +function createFairPhaseBudgets(capacity, requestedCount) { + if (requestedCount === 0) { + return Object.freeze({ + next() { + throw new RangeError("No evidence phases remain."); + } + }); + } + const protectedShare = { + bytes: Math.floor(Math.floor(capacity.bytes / 2) / requestedCount), + files: Math.floor(Math.floor(capacity.files / 2) / requestedCount), + entries: Math.floor(Math.floor(capacity.entries / 2) / requestedCount) + }; + let pool = subtractDemand(capacity, multiplyDemand(protectedShare, requestedCount)); + let remainingScopes = requestedCount; + return Object.freeze({ + next() { + if (remainingScopes === 0) throw new RangeError("No evidence phases remain."); + remainingScopes -= 1; + const phaseBudget = createContainedReadBudget({ + maxBytes: protectedShare.bytes + pool.bytes, + maxFiles: protectedShare.files + pool.files, + maxEntries: protectedShare.entries + pool.entries, + dimensionCodes: true + }); + let settled = false; + return Object.freeze({ + budget: phaseBudget, + settle() { + if (settled) return; + settled = true; + const used = phaseBudget.snapshot(); + pool = { + bytes: pool.bytes - Math.max(0, used.bytes - protectedShare.bytes) + + Math.max(0, protectedShare.bytes - used.bytes), + files: pool.files - Math.max(0, used.files - protectedShare.files) + + Math.max(0, protectedShare.files - used.files), + entries: pool.entries - Math.max(0, used.entries - protectedShare.entries) + + Math.max(0, protectedShare.entries - used.entries) + }; + } + }); + } + }); +} + +function omissionReasonForError(code) { + const reasons = { + DOTAIOS_EVIDENCE_FILE_TOO_LARGE: "file_too_large", + DOTAIOS_EVIDENCE_DIRECTORY_TOO_LARGE: "directory_entries_exceeded", + DOTAIOS_EVIDENCE_BYTE_BUDGET_EXCEEDED: "aggregate_bytes_exceeded", + DOTAIOS_EVIDENCE_FILE_COUNT_EXCEEDED: "file_count_exceeded", + DOTAIOS_EVIDENCE_ENTRY_COUNT_EXCEEDED: "entry_count_exceeded" + }; + return reasons[code]; +} + +function firstExceededDimension(demand, available) { + if (demand.bytes > available.bytes) return "aggregate_bytes_exceeded"; + if (demand.files > available.files) return "file_count_exceeded"; + return "entry_count_exceeded"; +} + +function createScopeOmission(scope, reason, demand, limits) { + const recovery = { + file_too_large: { + code: "split_or_move_file", + message: "Split the oversized file, or move it outside this search scope." + }, + directory_entries_exceeded: { + code: "reduce_directory_entries", + message: "Move some directory entries elsewhere, then retry the same search." + }, + aggregate_bytes_exceeded: { + code: "narrow_or_reduce_scope", + message: "Search a narrower scope, or move older material outside this scope." + }, + file_count_exceeded: { + code: "narrow_or_reduce_scope", + message: "Search a narrower scope, or move some files outside this scope." + }, + entry_count_exceeded: { + code: "narrow_or_reduce_scope", + message: "Search a narrower scope, or archive entries outside this scope." + } + }[reason]; + const observed = { + files: Math.min(demand.files, limits.files + 1), + bytes: Math.min(demand.bytes, limits.bytes + 1), + entries: Math.min(demand.entries, limits.entries + 1) + }; + if (reason === "file_count_exceeded") observed.files = limits.files + 1; + if (reason === "aggregate_bytes_exceeded") observed.bytes = limits.bytes + 1; + if (reason === "entry_count_exceeded") observed.entries = limits.entries + 1; + return { + scope, + reason, + observed, + inspection: reason === "directory_entries_exceeded" ? "partially_enumerated" : "not_searched", + recovery + }; +} + +function freezeOmissions(omissions) { + const selected = omissions.slice(0, OMITTED_SCOPE_LIMIT).map(deepFreezeOmission); + if (omissions.length > OMITTED_SCOPE_LIMIT) { + selected.push(deepFreezeOmission({ + scope: "all", + reason: "omissions_truncated", + observed: { files: 0, bytes: 0, entries: omissions.length - OMITTED_SCOPE_LIMIT }, + inspection: "not_searched", + recovery: { + code: "narrow_scope", + message: "Search one logical scope at a time to inspect every omission." + } + })); + } + return Object.freeze(selected); +} + +function deepFreezeOmission(omission) { + return Object.freeze({ + ...omission, + observed: Object.freeze({ ...omission.observed }), + recovery: Object.freeze({ ...omission.recovery }) + }); +} + +function emptyDemand() { + return { bytes: 0, files: 0, entries: 0 }; +} + +function minDemand(left, right) { + return { + bytes: Math.min(left.bytes, right.bytes), + files: Math.min(left.files, right.files), + entries: Math.min(left.entries, right.entries) + }; +} + +function addDemand(left, right) { + return { + bytes: left.bytes + right.bytes, + files: left.files + right.files, + entries: left.entries + right.entries + }; +} + +function multiplyDemand(demand, multiplier) { + return { + bytes: demand.bytes * multiplier, + files: demand.files * multiplier, + entries: demand.entries * multiplier + }; +} + +function subtractDemand(left, right) { + return { + bytes: Math.max(0, left.bytes - right.bytes), + files: Math.max(0, left.files - right.files), + entries: Math.max(0, left.entries - right.entries) + }; +} + +function combineContainedReadBudgets(...budgets) { + return Object.freeze({ + reserveFile(size) { + const remaining = minDemands(budgets.map((budget) => budget.remaining())); + if (remaining.files < 1) throw new ContainedReadError("DOTAIOS_PROJECTION_FILE_COUNT_EXCEEDED"); + if (remaining.bytes < size) throw new ContainedReadError("DOTAIOS_PROJECTION_BYTE_BUDGET_EXCEEDED"); + for (const budget of budgets) budget.reserveFile(size); + }, + reserveEntries(count = 1) { + if (minDemands(budgets.map((budget) => budget.remaining())).entries < count) { + throw new ContainedReadError("DOTAIOS_PROJECTION_ENTRY_COUNT_EXCEEDED"); + } + for (const budget of budgets) budget.reserveEntries(count); + }, + reserveDemand(demand = {}) { + const remaining = minDemands(budgets.map((budget) => budget.remaining())); + if (remaining.files < Number(demand.files || 0)) { + throw new ContainedReadError("DOTAIOS_PROJECTION_FILE_COUNT_EXCEEDED"); + } + if (remaining.bytes < Number(demand.bytes || 0)) { + throw new ContainedReadError("DOTAIOS_PROJECTION_BYTE_BUDGET_EXCEEDED"); + } + if (remaining.entries < Number(demand.entries || 0)) { + throw new ContainedReadError("DOTAIOS_PROJECTION_ENTRY_COUNT_EXCEEDED"); + } + for (const budget of budgets) budget.reserveDemand(demand); + }, + remaining() { + return minDemands(budgets.map((budget) => budget.remaining())); + }, + snapshot() { + return budgets.at(-1).snapshot(); + } + }); +} + +function minDemands(demands) { + return demands.reduce((minimum, demand) => minDemand(minimum, demand)); +} + +function fitsDemand(demand, capacity) { + return demand.bytes <= capacity.bytes + && demand.files <= capacity.files + && demand.entries <= capacity.entries; +} + +function rememberCorpusDirectory(observation, directoryPath, snapshot) { + const resolvedDirectory = path.resolve(directoryPath); + observation.directories.push(Object.freeze({ path: resolvedDirectory, snapshot })); +} + +function containedParentObservation(directoryPath, root, snapshot) { + const resolvedDirectory = path.resolve(directoryPath); + if (resolvedDirectory === path.resolve(root)) return {}; + const parentPath = path.dirname(resolvedDirectory); + const parent = snapshot.ancestors?.find( + (ancestor) => path.resolve(ancestor.path) === parentPath + ); + if (!parent) throw new ContainedReadError(); + return { parentPath, parentSnapshot: { stats: parent.stats } }; +} + +function normalizeCorpusConcurrency(value) { + if (value === undefined) return 32; + const normalized = Number(value); + if (!Number.isSafeInteger(normalized) || normalized < 1 || normalized > 64) { + throw new TypeError("Evidence corpus concurrency must be an integer from 1 to 64."); + } + return normalized; +} + function findFrontmatterEnd(bytes) { const startsWithLf = bytes.subarray(0, 4).toString("ascii") === "---\n"; const startsWithCrlf = bytes.subarray(0, 5).toString("ascii") === "---\r\n"; @@ -384,6 +1449,9 @@ function normalizeEvidenceReadError(error) { DOTAIOS_CONTEXT_SOURCE_CHANGED: "DOTAIOS_EVIDENCE_CHANGED", DOTAIOS_INVALID_UTF8: "DOTAIOS_EVIDENCE_INVALID_UTF8", DOTAIOS_PROJECTION_READ_BUDGET_EXCEEDED: "DOTAIOS_EVIDENCE_BUDGET_EXCEEDED", + DOTAIOS_PROJECTION_BYTE_BUDGET_EXCEEDED: "DOTAIOS_EVIDENCE_BYTE_BUDGET_EXCEEDED", + DOTAIOS_PROJECTION_FILE_COUNT_EXCEEDED: "DOTAIOS_EVIDENCE_FILE_COUNT_EXCEEDED", + DOTAIOS_PROJECTION_ENTRY_COUNT_EXCEEDED: "DOTAIOS_EVIDENCE_ENTRY_COUNT_EXCEEDED", DOTAIOS_EVIDENCE_FILE_TOO_LARGE: "DOTAIOS_EVIDENCE_FILE_TOO_LARGE", DOTAIOS_EVIDENCE_DIRECTORY_TOO_LARGE: "DOTAIOS_EVIDENCE_DIRECTORY_TOO_LARGE", DOTAIOS_BOUNDED_FILE_READ_UNAVAILABLE: "DOTAIOS_EVIDENCE_BOUNDED_READ_UNAVAILABLE", diff --git a/packages/core/src/memory.mjs b/packages/core/src/memory.mjs index 81348a3d..ae0848d1 100644 --- a/packages/core/src/memory.mjs +++ b/packages/core/src/memory.mjs @@ -1,9 +1,16 @@ import crypto from "node:crypto"; +import { constants as fsConstants } from "node:fs"; import fs from "node:fs/promises"; import path from "node:path"; import { formatJsonlEntry, parseJsonlLine, readJsonl } from "./jsonl.mjs"; import { ContainedReadError, readContainedDirectory, readContainedFile } from "./contained-read.mjs"; import { refreshLiveOkf, writeProjectLog } from "./okf-live.mjs"; +import { + assertOwnedFileStats, + publishOwnedFileExclusive, + recoverOwnedFileExclusivePublication, + syncOwnedDirectory +} from "./owned-state.mjs"; import { searchMarkdownDir, searchMemoryDir } from "./search.mjs"; export { formatJsonlEntry, parseJsonlLine, readJsonl }; @@ -152,6 +159,7 @@ export class EventStoreLockError extends Error { export async function appendEventRecord(eventsPath, entry, options = {}) { const fileSystem = options.filesystem || fs; await withEventStoreLock(eventsPath, async () => { + await recoverPendingArchive(eventsPath, fileSystem); await fileSystem.appendFile(eventsPath, formatJsonlEntry(entry)); }, options); return entry; @@ -234,6 +242,14 @@ const LOCK_STALE_MS = 5 * 60_000; const LOCK_RETRY_DELAYS_MS = [50, 150, 300]; const LOCK_MAX_ATTEMPTS = 8; const ARCHIVE_TAIL_BYTES = 262_144; +const ARCHIVE_ROTATE_BYTES = 2 * 1024 * 1024; +const ARCHIVE_LINE_MAX_BYTES = 4 * 1024 * 1024; +const EVENT_COMPACTION_PENDING_CONTRACT = "dotaios-event-compaction/v1"; +const EVENT_COMPACTION_PENDING_MAGIC_PREFIX = "#!dotaios-event-compaction"; +const EVENT_COMPACTION_PENDING_MAGIC = `${EVENT_COMPACTION_PENDING_MAGIC_PREFIX}/v1`; +// Distinguishes marker-protocol generations from ambiguous overlap left by the +// previous markerless rotator. It is durable before any new rotation begins. +const ARCHIVE_ROTATION_FORMAT = '{"version":1,"protocol":"durable-marker"}\n'; function archivePathFor(eventsPath) { return eventsPath.replace(/\.jsonl$/, "-archive.jsonl"); @@ -356,23 +372,30 @@ async function malformedLockIsStale(lockPath, fileSystem, staleMs, now) { async function fsyncFile(fileSystem, filePath) { if (typeof fileSystem.open !== "function") return; + const handle = await fileSystem.open(filePath, "r+"); try { - const handle = await fileSystem.open(filePath, "r+"); - try { - await handle.sync(); - } finally { - await handle.close(); - } - } catch { - // fsync is best-effort; the commit point below is the atomicity boundary. + await handle.sync(); + } finally { + await handle.close(); } } async function writeFileDurable(fileSystem, filePath, content) { - await fileSystem.writeFile(filePath, content); + await fileSystem.writeFile(filePath, content, { flag: "wx", mode: 0o600 }); + await fsyncFile(fileSystem, filePath); +} + +async function stagePendingArchive(fileSystem, filePath, content) { + assertArchiveLineBounds(content); + await fileSystem.writeFile(filePath, content, { flag: "wx", mode: 0o600 }); await fsyncFile(fileSystem, filePath); } +async function stageEventPendingArchive(fileSystem, filePath, content) { + assertArchiveLineBounds(content); + await publishOwnedFileExclusive(filePath, content, { filesystem: fileSystem }); +} + async function fileExists(fileSystem, filePath) { try { await fileSystem.stat(filePath); @@ -382,85 +405,837 @@ async function fileExists(fileSystem, filePath) { } } -// A crash can leave a staged archive batch behind. Decide whether the batch -// committed (rename happened → events no longer holds it → finish the flush) -// or not (events still holds it → the staging file is stale, drop it). +// A crash can leave one self-describing pending artifact behind. `prepared` +// evidence exists before archive inspection and can never authorize a live +// rename. `ready` binds the exact ordered archive pre-state and can recover +// publication after the live-file commit point. Raw legacy batches retain the +// conservative compatibility path below. async function recoverPendingArchive(eventsPath, fileSystem) { const archivePath = archivePathFor(eventsPath); const pendingPath = pendingPathFor(archivePath); let pendingContent; + let pendingStats; try { - pendingContent = await fileSystem.readFile(pendingPath, "utf8"); + ({ content: pendingContent, stats: pendingStats } = await readOwnedArchiveFile(pendingPath, fileSystem, { + allowLegacyMode: true + })); } catch (error) { if (error.code === "ENOENT") return; throw error; } - const pendingLines = pendingContent.split("\n").filter((line) => line.trim()); + const artifact = parsePendingArchiveArtifact(pendingContent); + const pendingLines = artifact.pendingLines; if (pendingLines.length === 0) { - await fileSystem.unlink(pendingPath); + await removeObservedArchiveFile(pendingPath, pendingStats, fileSystem); return; } - let eventsContent = ""; - try { - eventsContent = await fileSystem.readFile(eventsPath, "utf8"); - } catch (error) { - if (error.code !== "ENOENT") throw error; + const eventsContent = await readEventsContent(eventsPath, fileSystem); + if (artifact.kind === "transaction") { + const transaction = artifact.transaction; + const liveState = identifyTransactionLiveState(eventsContent, transaction); + if (transaction.phase === "prepared") { + if (liveState !== "before") throw archiveStateError(); + await removeObservedArchiveFile(pendingPath, pendingStats, fileSystem); + return; + } + if (liveState === "before") { + const archiveState = await inspectArchiveRecordState(archivePath, fileSystem); + if ( + archiveState.records !== transaction.archiveRecordsBefore + || archiveState.chain !== transaction.archiveChainBefore + ) throw archiveStateError(); + await removeObservedArchiveFile(pendingPath, pendingStats, fileSystem); + return; + } + if (liveState !== "after") throw archiveStateError(); + await flushPendingArchive(archivePath, fileSystem, { + archiveRecordsBefore: transaction.archiveRecordsBefore, + archiveChainBefore: transaction.archiveChainBefore + }); + return; } + + // Legacy raw pending batches predate the self-describing envelope. Preserve + // their old conservative recovery rule; every newly staged event batch is + // governed by the phase and exact-chain evidence above. const eventLines = new Set(eventsContent.split("\n").filter((line) => line.trim())); const first = pendingLines[0]; const last = pendingLines[pendingLines.length - 1]; if (eventLines.has(first) && eventLines.has(last)) { - await fileSystem.unlink(pendingPath); + await removeObservedArchiveFile(pendingPath, pendingStats, fileSystem); return; } await flushPendingArchive(archivePath, fileSystem); } -// Append the staged batch to the archive, line-idempotently: a crashed earlier -// flush may already have written part or all of it. Returns only after the -// append is on disk and fsynced, so callers may treat it as the point after -// which deleting the source is safe. -async function flushPendingArchive(archivePath, fileSystem) { +function identifyTransactionLiveState(eventsContent, transaction) { + const eventsHash = archiveContentHash(eventsContent); + if (eventsHash === transaction.beforeHash) return "before"; + if (eventsHash === transaction.afterHash) return "after"; + + // Validate each potential suffix record exactly once, then hash the live + // bytes once from left to right. Hash.copy() snapshots the prefix state at a + // newline without re-hashing every earlier byte for every boundary. + if (!eventsContent.endsWith("\n")) return null; + const records = splitArchiveRecords(eventsContent); + const validSuffix = new Array(records.length + 1).fill(false); + validSuffix[records.length] = true; + for (let index = records.length - 1; index >= 0; index -= 1) { + validSuffix[index] = validSuffix[index + 1] && isCompleteJsonlRecord(records[index]); + } + + const prefixHash = crypto.createHash("sha256"); + for (let index = 0; index < records.length - 1; index += 1) { + prefixHash.update(records[index]); + if (!validSuffix[index + 1]) continue; + const digest = prefixHash.copy().digest("hex"); + if (digest === transaction.beforeHash) return "before"; + if (digest === transaction.afterHash) return "after"; + } + return null; +} + +function isCompleteJsonlRecord(record) { + if (!record.endsWith("\n")) return false; + const line = record.slice(0, -1); + if (!line.trim()) return false; + try { + const entry = JSON.parse(line); + return entry !== null && typeof entry === "object" && !Array.isArray(entry); + } catch { + return false; + } +} + +// Publish the staged batch into the active archive and, when needed, immutable +// numbered shards. Event envelopes prove exact ordered progress; legacy signal +// batches retain their bounded compatibility behavior. Returns only after every +// accepted line is fsynced, so callers may safely remove its source. +async function flushPendingArchive(archivePath, fileSystem, options = {}) { const pendingPath = pendingPathFor(archivePath); let pendingContent; + let pendingStats; try { - pendingContent = await fileSystem.readFile(pendingPath, "utf8"); + ({ content: pendingContent, stats: pendingStats } = await readOwnedArchiveFile(pendingPath, fileSystem, { + allowLegacyMode: true + })); } catch (error) { if (error.code === "ENOENT") return; throw error; } - const pendingLines = pendingContent.split("\n").filter((line) => line.trim()); - let archiveContent = ""; + const artifact = parsePendingArchiveArtifact(pendingContent); + const pendingLines = artifact.pendingLines; + assertArchiveLineBounds(artifact.payload); + if (artifact.kind === "transaction" && ( + artifact.transaction.phase !== "ready" + || artifact.transaction.archiveRecordsBefore !== options.archiveRecordsBefore + || artifact.transaction.archiveChainBefore !== options.archiveChainBefore + )) throw archiveStateError(); + + let shards = await inspectArchiveShards(archivePath, fileSystem); + let active = await readArchiveActive(archivePath, fileSystem); + assertArchiveLineBounds(active.content); + let missing; + if ( + Number.isSafeInteger(options.archiveRecordsBefore) + && typeof options.archiveChainBefore === "string" + ) { + // Finish any marker-governed rotation first. During the shard-before-active + // publication window the same logical records temporarily exist in both + // generations, so counting before recovery would mistake overlap for new + // records from this batch. + await appendArchiveLines(archivePath, active, shards, [], fileSystem); + shards = await inspectArchiveShards(archivePath, fileSystem); + active = await readArchiveActive(archivePath, fileSystem); + // A transaction records the archive's logical record count before this + // exact batch. The count delta tells us how much of the ordered batch was + // published before a crash without confusing equal bytes for one record. + const archiveState = await inspectArchiveRecordState(archivePath, fileSystem, { active, shards }); + const published = archiveState.records - options.archiveRecordsBefore; + if (published < 0 || published > pendingLines.length) throw archiveStateError(); + let expectedChain = options.archiveChainBefore; + for (const line of pendingLines.slice(0, published)) { + expectedChain = extendArchiveRecordChain(expectedChain, line); + } + if (archiveState.chain !== expectedChain) throw archiveStateError(); + missing = pendingLines.slice(published); + } else { + const tailLines = await collectArchiveTailLines( + archivePath, + active, + shards, + Math.max(ARCHIVE_TAIL_BYTES, Buffer.byteLength(pendingContent) + 1024), + fileSystem + ); + missing = pendingLines.filter((line) => !tailLines.has(line)); + } + await appendArchiveLines(archivePath, active, shards, missing, fileSystem); + + if (artifact.kind === "transaction") { + const settled = await inspectArchiveRecordState(archivePath, fileSystem); + if ( + settled.records !== artifact.transaction.archiveRecordsBefore + artifact.transaction.pendingRecords + || settled.chain !== artifact.transaction.archiveChainAfter + ) throw archiveStateError(); + } + + await removeObservedArchiveFile(pendingPath, pendingStats, fileSystem); +} + +async function readEventsContent(eventsPath, fileSystem) { + try { + return await fileSystem.readFile(eventsPath, "utf8"); + } catch (error) { + if (error.code === "ENOENT") return ""; + throw error; + } +} + +async function inspectArchiveRecordState(archivePath, fileSystem, observed = {}) { + const shards = observed.shards || await inspectArchiveShards(archivePath, fileSystem); + const active = observed.active || await readArchiveActive(archivePath, fileSystem); + let records = 0; + let chain = "0".repeat(64); + for (const shard of shards) { + const content = (await readOwnedArchiveFile(shard.path, fileSystem)).content; + for (const line of content.split("\n").filter((item) => item.trim())) { + chain = extendArchiveRecordChain(chain, line); + records += 1; + } + } + for (const line of active.content.split("\n").filter((item) => item.trim())) { + chain = extendArchiveRecordChain(chain, line); + records += 1; + } + return { records, chain }; +} + +function extendArchiveRecordChain(chain, line) { + const bytes = Buffer.from(line); + const length = Buffer.allocUnsafe(4); + length.writeUInt32BE(bytes.length); + return crypto.createHash("sha256") + .update(Buffer.from(chain, "hex")) + .update(length) + .update(bytes) + .digest("hex"); +} + +function parsePendingArchiveArtifact(content) { + if (!content.startsWith(`${EVENT_COMPACTION_PENDING_MAGIC}\n`)) { + if (content.startsWith(EVENT_COMPACTION_PENDING_MAGIC_PREFIX)) throw archiveStateError(); + return parseLegacyPendingArchive(content); + } + const headerStart = EVENT_COMPACTION_PENDING_MAGIC.length + 1; + const headerEnd = content.indexOf("\n", headerStart); + if (headerEnd < 0) throw archiveStateError(); + const headerLine = content.slice(headerStart, headerEnd); + let transaction; + try { + transaction = JSON.parse(headerLine); + } catch { + throw archiveStateError(); + } + if (transaction?.artifact_contract !== EVENT_COMPACTION_PENDING_CONTRACT) throw archiveStateError(); + const payload = content.slice(headerEnd + 1); + const pendingLines = payload.split("\n").filter((line) => line.trim()); + if ( + !["prepared", "ready"].includes(transaction.phase) + || ![ + transaction.beforeHash, + transaction.afterHash, + transaction.pendingHash + ] + .every((value) => typeof value === "string" && /^[a-f0-9]{64}$/.test(value)) + || !Number.isSafeInteger(transaction.pendingRecords) + || transaction.pendingRecords < 1 + || archiveContentHash(payload) !== transaction.pendingHash + || pendingLines.length !== transaction.pendingRecords + ) throw archiveStateError(); + if (transaction.phase === "ready" && ( + ![transaction.archiveChainBefore, transaction.archiveChainAfter] + .every((value) => typeof value === "string" && /^[a-f0-9]{64}$/.test(value)) + || !Number.isSafeInteger(transaction.archiveRecordsBefore) + || transaction.archiveRecordsBefore < 0 + )) throw archiveStateError(); + if (transaction.phase === "prepared" && [ + transaction.archiveRecordsBefore, + transaction.archiveChainBefore, + transaction.archiveChainAfter + ].some((value) => value !== undefined)) throw archiveStateError(); + return { kind: "transaction", transaction, payload, pendingLines }; +} + +function parseLegacyPendingArchive(content) { + const pendingLines = content.split("\n").filter((line) => line.trim()); + if (pendingLines.some((line) => !isCompleteJsonlRecord(`${line}\n`))) { + throw archiveStateError(); + } + return { kind: "legacy", payload: content, pendingLines }; +} + +function renderPendingArchiveArtifact(transaction, payload) { + return `${EVENT_COMPACTION_PENDING_MAGIC}\n${JSON.stringify(transaction)}\n${payload}`; +} + +async function removeObservedArchiveFile(filePath, expectedStats, fileSystem) { + const current = await fileSystem.lstat(filePath, { bigint: true }); + assertOwnedArchiveFileStats(current); + if (!sameArchiveSnapshot(expectedStats, current)) throw archiveStateError(); + await fileSystem.unlink(filePath); + await syncOwnedDirectory(path.dirname(filePath), { filesystem: fileSystem }); +} + +async function appendArchiveLines(archivePath, active, existingShards, lines, fileSystem) { + let activeContent = active.content; + let activeStats = active.stats; + let rotationMarkerRecovered; + ({ activeContent, activeStats, existingShards, rotationMarkerRecovered } = await recoverArchiveRotation( + archivePath, + activeContent, + activeStats, + existingShards, + fileSystem + )); + await ensureArchiveRotationFormat( + archivePath, + activeContent, + existingShards, + fileSystem, + rotationMarkerRecovered + ); + let nextShard = existingShards.length === 0 + ? 1 + : existingShards.at(-1).number + 1; + + // Normalize a legacy over-target active file one complete JSONL record at a + // time. The durable transition marker is the only authority allowed to + // interpret matching bytes in a shard and the active generation as overlap. + while (Buffer.byteLength(activeContent) > ARCHIVE_ROTATE_BYTES) { + const { chunk, remainder } = takeArchiveChunk(activeContent); + activeStats = await rotateArchiveActive( + archivePath, + nextShard, + activeContent, + chunk, + remainder, + activeStats, + fileSystem + ); + nextShard += 1; + activeContent = remainder; + } + + let activeParts = activeContent ? [activeContent] : []; + let activeBytes = Buffer.byteLength(activeContent); + let activeEndsWithNewline = activeContent.endsWith("\n"); + const materializeActive = () => { + activeContent = activeParts.join(""); + activeParts = activeContent ? [activeContent] : []; + return activeContent; + }; + let changed = false; + for (const line of lines) { + const record = `${line}\n`; + const recordBytes = Buffer.byteLength(record); + if (recordBytes > ARCHIVE_LINE_MAX_BYTES) throw archiveLineTooLargeError(); + const separator = activeBytes > 0 && !activeEndsWithNewline ? "\n" : ""; + const separatorBytes = separator ? 1 : 0; + if (activeBytes > 0 && activeBytes + separatorBytes + recordBytes > ARCHIVE_ROTATE_BYTES) { + const completedActive = materializeActive(); + activeStats = await replaceArchiveActive(archivePath, completedActive, activeStats, fileSystem); + activeStats = await rotateArchiveActive( + archivePath, + nextShard, + completedActive, + completedActive, + "", + activeStats, + fileSystem + ); + nextShard += 1; + activeContent = ""; + activeParts = []; + activeBytes = 0; + activeEndsWithNewline = false; + changed = false; + } + if (recordBytes > ARCHIVE_ROTATE_BYTES) { + await publishArchiveShard(archivePath, nextShard, record, fileSystem); + nextShard += 1; + continue; + } + if (separator) activeParts.push(separator); + activeParts.push(record); + activeBytes += separatorBytes + recordBytes; + activeEndsWithNewline = true; + changed = true; + } + + if (changed || !activeStats) { + await replaceArchiveActive(archivePath, materializeActive(), activeStats, fileSystem); + } +} + +async function recoverArchiveRotation( + archivePath, + activeContent, + activeStats, + existingShards, + fileSystem +) { + const markerPath = rotationMarkerPathFor(archivePath); + let markerContent; + let markerStats; + try { + ({ content: markerContent, stats: markerStats } = await readOwnedArchiveFile(markerPath, fileSystem)); + } catch (error) { + if (error.code === "ENOENT") { + return { activeContent, activeStats, existingShards, rotationMarkerRecovered: false }; + } + throw error; + } + + const marker = parseArchiveRotationMarker(markerContent); + const shard = existingShards.find(({ number }) => number === marker.shardNumber); + if (existingShards.some(({ number }) => number > marker.shardNumber)) throw archiveStateError(); + + const activeHash = archiveContentHash(activeContent); + if (activeHash === marker.beforeHash) { + const bytes = Buffer.from(activeContent); + const chunkBytes = bytes.subarray(0, marker.chunkBytes); + const remainderBytes = bytes.subarray(marker.chunkBytes); + const chunk = chunkBytes.toString("utf8"); + const remainder = remainderBytes.toString("utf8"); + if ( + chunkBytes.length !== marker.chunkBytes + || Buffer.byteLength(chunk) !== marker.chunkBytes + || archiveContentHash(chunk) !== marker.chunkHash + || archiveContentHash(remainder) !== marker.afterHash + ) throw archiveStateError(); + + if (shard) { + const shardContent = (await readOwnedArchiveFile(shard.path, fileSystem)).content; + if (archiveContentHash(shardContent) !== marker.chunkHash) throw archiveStateError(); + } else { + await publishArchiveShard(archivePath, marker.shardNumber, chunk, fileSystem); + } + activeStats = await replaceArchiveActive(archivePath, remainder, activeStats, fileSystem); + activeContent = remainder; + } else if (activeHash === marker.afterHash) { + if (!shard) throw archiveStateError(); + const shardContent = (await readOwnedArchiveFile(shard.path, fileSystem)).content; + if (archiveContentHash(shardContent) !== marker.chunkHash) throw archiveStateError(); + } else { + throw archiveStateError(); + } + + await ensureArchiveRotationFormat( + archivePath, + activeContent, + existingShards, + fileSystem, + true + ); + await removeObservedArchiveFile(markerPath, markerStats, fileSystem); + return { + activeContent, + activeStats, + existingShards: await inspectArchiveShards(archivePath, fileSystem), + rotationMarkerRecovered: true + }; +} + +async function ensureArchiveRotationFormat( + archivePath, + activeContent, + existingShards, + fileSystem, + rotationMarkerRecovered +) { + const formatPath = rotationFormatPathFor(archivePath); try { - archiveContent = await fileSystem.readFile(archivePath, "utf8"); + const { content } = await readOwnedArchiveFile(formatPath, fileSystem); + if (content !== ARCHIVE_ROTATION_FORMAT) throw archiveStateError(); + return; } catch (error) { if (error.code !== "ENOENT") throw error; } - // The dedupe window must be at least as long as the batch it checks. A batch - // bigger than a fixed window would fail to find its own already-written early - // lines on a retry and append them a second time. - const windowBytes = Math.max(ARCHIVE_TAIL_BYTES, pendingContent.length + 1024); - const tailLines = new Set(archiveContent.slice(-windowBytes).split("\n").filter((line) => line.trim())); - const missing = pendingLines.filter((line) => !tailLines.has(line)); - if (missing.length > 0) { - // A torn earlier append can leave the archive without a final newline; - // start on a fresh line so the fragment stays its own (visible) bad line. - const prefix = archiveContent.length > 0 && !archiveContent.endsWith("\n") ? "\n" : ""; - await fileSystem.appendFile(archivePath, prefix + missing.map((line) => `${line}\n`).join("")); - await fsyncFile(fileSystem, archivePath); + + if (!rotationMarkerRecovered) { + await assertNoMarkerlessArchiveOverlap(archivePath, activeContent, existingShards, fileSystem); + } + await publishOwnedFileExclusive(formatPath, ARCHIVE_ROTATION_FORMAT, { filesystem: fileSystem }); +} + +async function assertNoMarkerlessArchiveOverlap( + archivePath, + activeContent, + existingShards, + fileSystem +) { + const newest = existingShards.at(-1); + if (!newest || !activeContent) return; + const newestContent = (await readOwnedArchiveFile(newest.path, fileSystem)).content; + if (!newestContent || !activeContent.startsWith(newestContent)) return; + throw archiveLegacyRecoveryRequiredError(archivePath, newest.path); +} + +async function rotateArchiveActive( + archivePath, + shardNumber, + activeContent, + chunk, + remainder, + activeStats, + fileSystem +) { + const markerPath = rotationMarkerPathFor(archivePath); + const marker = { + version: 1, + shardNumber, + chunkBytes: Buffer.byteLength(chunk), + chunkHash: archiveContentHash(chunk), + beforeHash: archiveContentHash(activeContent), + afterHash: archiveContentHash(remainder) + }; + await publishOwnedFileExclusive( + markerPath, + `${JSON.stringify(marker)}\n`, + { filesystem: fileSystem } + ); + const markerStats = await inspectOwnedArchiveFile(markerPath, fileSystem); + await publishArchiveShard(archivePath, shardNumber, chunk, fileSystem); + const publishedStats = await replaceArchiveActive(archivePath, remainder, activeStats, fileSystem); + await removeObservedArchiveFile(markerPath, markerStats, fileSystem); + return publishedStats; +} + +function parseArchiveRotationMarker(content) { + let marker; + try { + marker = JSON.parse(content); + } catch { + throw archiveStateError(); + } + if ( + marker?.version !== 1 + || !Number.isSafeInteger(marker.shardNumber) + || marker.shardNumber < 1 + || !Number.isSafeInteger(marker.chunkBytes) + || marker.chunkBytes < 1 + || ![marker.chunkHash, marker.beforeHash, marker.afterHash] + .every((value) => typeof value === "string" && /^[a-f0-9]{64}$/.test(value)) + ) throw archiveStateError(); + return marker; +} + +function archiveContentHash(content) { + return crypto.createHash("sha256").update(content).digest("hex"); +} + +function rotationMarkerPathFor(archivePath) { + return `${archivePath}.rotation`; +} + +function rotationFormatPathFor(archivePath) { + return `${archivePath}.rotation-format`; +} + +function takeArchiveChunk(content) { + let chunkEnd = 0; + let chunkBytes = 0; + let recordStart = 0; + for (let index = 0; index <= content.length; index += 1) { + if (index < content.length && content[index] !== "\n") continue; + if (index === content.length && recordStart === content.length) break; + const recordEnd = index < content.length ? index + 1 : index; + const recordBytes = Buffer.byteLength(content.slice(recordStart, recordEnd)); + if (chunkEnd > 0 && chunkBytes + recordBytes > ARCHIVE_ROTATE_BYTES) break; + chunkBytes += recordBytes; + chunkEnd = recordEnd; + recordStart = recordEnd; + if (chunkBytes > ARCHIVE_ROTATE_BYTES) break; + } + if (chunkEnd === 0) throw archiveStateError(); + return { chunk: content.slice(0, chunkEnd), remainder: content.slice(chunkEnd) }; +} + +function splitArchiveRecords(content) { + if (!content) return []; + const records = []; + let start = 0; + for (let index = 0; index < content.length; index += 1) { + if (content[index] !== "\n") continue; + records.push(content.slice(start, index + 1)); + start = index + 1; + } + if (start < content.length) records.push(content.slice(start)); + return records; +} + +function assertArchiveLineBounds(content) { + for (const record of splitArchiveRecords(content)) { + if (Buffer.byteLength(record) > ARCHIVE_LINE_MAX_BYTES) throw archiveLineTooLargeError(); + } +} + +async function inspectArchiveShards(archivePath, fileSystem) { + const directory = path.dirname(archivePath); + const stem = path.basename(archivePath, ".jsonl"); + const matcher = new RegExp(`^${escapeRegExp(stem)}\\.(\\d{6})\\.jsonl$`); + let entries; + try { + entries = await fileSystem.readdir(directory, { withFileTypes: true }); + } catch (error) { + if (error.code === "ENOENT") return []; + throw error; + } + const shards = []; + for (const entry of entries) { + const match = entry.name.match(matcher); + if (!match) continue; + const filePath = path.join(directory, entry.name); + const stats = await inspectOwnedArchiveFile(filePath, fileSystem); + shards.push({ number: Number(match[1]), path: filePath, stats }); + } + shards.sort((left, right) => left.number - right.number); + for (let index = 1; index < shards.length; index += 1) { + if (shards[index - 1].number === shards[index].number) throw archiveStateError(); + } + return shards; +} + +async function inspectOwnedArchiveFile(filePath, fileSystem) { + let stats = await fileSystem.lstat(filePath, { bigint: true }); + if (Number(stats.nlink) === 2 && await recoverOwnedFileExclusivePublication(filePath, { + filesystem: fileSystem + })) { + stats = await fileSystem.lstat(filePath, { bigint: true }); + } + assertOwnedArchiveFileStats(stats); + return stats; +} + +async function collectArchiveTailLines(archivePath, active, shards, minimumBytes, fileSystem) { + const lines = new Set(); + let observedBytes = 0; + const sources = [ + { path: archivePath, content: active.content }, + ...[...shards].reverse().map(({ path: filePath }) => ({ path: filePath, content: null })) + ]; + for (const source of sources) { + const content = source.content === null + ? (await readOwnedArchiveFile(source.path, fileSystem)).content + : source.content; + observedBytes += Buffer.byteLength(content); + for (const line of content.split("\n")) if (line.trim()) lines.add(line); + if (observedBytes >= minimumBytes) break; + } + return lines; +} + +async function readArchiveActive(archivePath, fileSystem) { + try { + return await readOwnedArchiveFile(archivePath, fileSystem, { allowLegacyMode: true }); + } catch (error) { + if (error.code === "ENOENT") return { content: "", stats: null }; + throw error; + } +} + +async function readOwnedArchiveFile(filePath, fileSystem, { allowLegacyMode = false } = {}) { + let pathStats = await fileSystem.lstat(filePath, { bigint: true }); + if (Number(pathStats.nlink) === 2 && await recoverOwnedFileExclusivePublication(filePath, { + filesystem: fileSystem + })) { + pathStats = await fileSystem.lstat(filePath, { bigint: true }); + } + const flags = fsConstants.O_RDONLY | (fsConstants.O_NOFOLLOW || 0); + const handle = await fileSystem.open(filePath, flags); + try { + let before = await handle.stat({ bigint: true }); + ({ pathStats, handleStats: before } = await validateArchiveFileMode( + filePath, + pathStats, + before, + handle, + fileSystem, + allowLegacyMode + )); + const content = await handle.readFile("utf8"); + const after = await handle.stat({ bigint: true }); + assertOwnedArchiveFileStats(after); + if (!sameArchiveSnapshot(before, after)) throw archiveStateError(); + const current = await fileSystem.lstat(filePath, { bigint: true }); + assertOwnedArchiveFileStats(current); + if (!sameArchiveSnapshot(after, current)) throw archiveStateError(); + return { content, stats: current }; + } finally { + await handle.close(); + } +} + +async function validateArchiveFileMode( + filePath, + pathStats, + handleStats, + handle, + fileSystem, + allowLegacyMode +) { + try { + assertOwnedArchiveFileStats(pathStats); + assertOwnedArchiveFileStats(handleStats); + if (!sameArchiveSnapshot(pathStats, handleStats)) throw archiveStateError(); + return { pathStats, handleStats }; + } catch (error) { + if (!allowLegacyMode || process.platform === "win32") throw error; + assertOwnedArchiveFileStats(pathStats, 0o644); + assertOwnedArchiveFileStats(handleStats, 0o644); + if (!sameArchiveSnapshot(pathStats, handleStats)) throw archiveStateError(); + await handle.chmod(0o600); + const securedHandle = await handle.stat({ bigint: true }); + const securedPath = await fileSystem.lstat(filePath, { bigint: true }); + assertOwnedArchiveFileStats(securedHandle); + assertOwnedArchiveFileStats(securedPath); + if ( + !sameArchiveObject(handleStats, securedHandle) + || !sameArchiveSnapshot(securedHandle, securedPath) + ) throw archiveStateError(); + return { pathStats: securedPath, handleStats: securedHandle }; + } +} + +function sameArchiveObject(left, right) { + return Boolean(left && right + && left.dev === right.dev + && left.ino === right.ino + && left.nlink === right.nlink + && left.uid === right.uid); +} + +function sameArchiveIdentity(left, right) { + return Boolean(sameArchiveObject(left, right) + && left.mode === right.mode + && left.size === right.size); +} + +function sameArchiveSnapshot(left, right) { + return Boolean(left && right && [ + "dev", + "ino", + "mode", + "nlink", + "uid", + "size", + "mtimeNs", + "ctimeNs" + ].every((field) => left[field] === right[field])); +} + +function assertOwnedArchiveFileStats(stats, mode = 0o600) { + if (typeof stats?.mode !== "bigint") { + assertOwnedFileStats(stats, mode); + return; } - await fileSystem.unlink(pendingPath); + assertOwnedFileStats({ + isFile: () => stats.isFile(), + isSymbolicLink: () => stats.isSymbolicLink(), + mode: Number(stats.mode), + nlink: Number(stats.nlink), + uid: Number(stats.uid) + }, mode); +} + +async function publishArchiveShard(archivePath, number, content, fileSystem) { + assertArchiveLineBounds(content); + const shardPath = archiveShardPath(archivePath, number); + await publishOwnedFileExclusive(shardPath, content, { filesystem: fileSystem }); + return shardPath; +} + +async function replaceArchiveActive(archivePath, content, expectedStats, fileSystem) { + assertArchiveLineBounds(content); + if (!expectedStats) { + await publishOwnedFileExclusive(archivePath, content, { filesystem: fileSystem }); + return inspectOwnedArchiveFile(archivePath, fileSystem); + } + const current = await fileSystem.lstat(archivePath, { bigint: true }); + assertOwnedArchiveFileStats(current); + if (!sameArchiveSnapshot(expectedStats, current)) throw archiveStateError(); + const temporary = path.join( + path.dirname(archivePath), + `.${path.basename(archivePath)}.${crypto.randomUUID()}.tmp` + ); + let handle; + try { + handle = await fileSystem.open(temporary, "wx", 0o600); + await handle.writeFile(content, "utf8"); + await handle.sync(); + await handle.close(); + handle = null; + const temporaryStats = await fileSystem.lstat(temporary, { bigint: true }); + assertOwnedArchiveFileStats(temporaryStats); + const stillCurrent = await fileSystem.lstat(archivePath, { bigint: true }); + assertOwnedArchiveFileStats(stillCurrent); + if (!sameArchiveSnapshot(expectedStats, stillCurrent)) throw archiveStateError(); + await fileSystem.rename(temporary, archivePath); + const published = await fileSystem.lstat(archivePath, { bigint: true }); + assertOwnedArchiveFileStats(published); + if (!sameArchiveIdentity(temporaryStats, published)) throw archiveStateError(); + await syncOwnedDirectory(path.dirname(archivePath), { filesystem: fileSystem }); + return published; + } finally { + if (handle) await handle.close().catch(() => {}); + await fileSystem.rm(temporary, { force: true }).catch(() => {}); + } +} + +function archiveShardPath(archivePath, number) { + return archivePath.replace(/\.jsonl$/, `.${String(number).padStart(6, "0")}.jsonl`); +} + +function archiveLineTooLargeError() { + const error = new Error("One archive record exceeds the safe 4 MiB read ceiling."); + error.code = "DOTAIOS_ARCHIVE_LINE_TOO_LARGE"; + return error; +} + +function archiveStateError() { + const error = new Error("Memory archive state changed or is not safely owned."); + error.code = "DOTAIOS_ARCHIVE_STATE_INVALID"; + return error; +} + +function archiveLegacyRecoveryRequiredError(archivePath, shardPath) { + const error = new Error( + "Memory archive needs explicit legacy rotation recovery; authoritative archive bytes were left unchanged." + ); + error.code = "DOTAIOS_ARCHIVE_LEGACY_RECOVERY_REQUIRED"; + error.diagnostic = Object.freeze({ + kind: "markerless-rotation-overlap", + archive: path.basename(archivePath), + shard: path.basename(shardPath), + action: "preserve-and-inspect" + }); + return error; +} + +function escapeRegExp(value) { + return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); } /** * Compact events.jsonl: keep only the most recent N entries in the main file, - * archive older entries to events-archive.jsonl. + * archive older entries to the bounded events-archive generation. * - * Crash-safe transaction: kept-tail and archive batch are staged and fsynced, - * the rename over events.jsonl is the single commit point, and the archive - * append happens only after it (idempotently, so an interrupted run can be - * re-run without losing or duplicating an event). Guarded by an advisory - * lockfile next to events.jsonl. + * Crash-safe transaction: a self-describing prepared artifact is published + * before archive inspection, then ownership-safely advanced to ready with the + * exact ordered archive pre-state. Only ready may precede the rename over + * events.jsonl, the single live-file commit point. Archive publication happens + * afterward and verifies the expected ordered post-state before removing the + * artifact. Guarded by an advisory lockfile next to events.jsonl. * * Returns { archived, kept } — or { archived: 0, kept: 0, skipped: "locked" } * when another live process holds the lock. @@ -478,12 +1253,60 @@ export async function compactEvents(eventsPath, limit = RECENT_EVENT_LIMIT, opti const toArchive = all.slice(0, -limit); const toKeep = all.slice(-limit); - const tmpPath = `${eventsPath}.tmp`; + const tmpPath = path.join( + path.dirname(eventsPath), + `.${path.basename(eventsPath)}.${crypto.randomUUID()}.tmp` + ); + const archiveBatch = toArchive.map((entry) => formatJsonlEntry(entry)).join(""); + const keptBatch = toKeep.map((entry) => formatJsonlEntry(entry)).join(""); + assertArchiveLineBounds(archiveBatch); - await writeFileDurable(fileSystem, tmpPath, toKeep.map((entry) => formatJsonlEntry(entry)).join("")); - await writeFileDurable(fileSystem, pendingPathFor(archivePathFor(eventsPath)), toArchive.map((entry) => formatJsonlEntry(entry)).join("")); - await fileSystem.rename(tmpPath, eventsPath); - await flushPendingArchive(archivePathFor(eventsPath), fileSystem); + try { + const archivePath = archivePathFor(eventsPath); + const pendingPath = pendingPathFor(archivePath); + const eventsContent = await readEventsContent(eventsPath, fileSystem); + const prepared = { + artifact_contract: EVENT_COMPACTION_PENDING_CONTRACT, + phase: "prepared", + beforeHash: archiveContentHash(eventsContent), + afterHash: archiveContentHash(keptBatch), + pendingHash: archiveContentHash(archiveBatch), + pendingRecords: toArchive.length + }; + await writeFileDurable(fileSystem, tmpPath, keptBatch); + await stageEventPendingArchive( + fileSystem, + pendingPath, + renderPendingArchiveArtifact(prepared, archiveBatch) + ); + const preparedStats = await inspectOwnedArchiveFile(pendingPath, fileSystem); + const archiveState = await inspectArchiveRecordState(archivePath, fileSystem); + let archiveChainAfter = archiveState.chain; + for (const line of archiveBatch.split("\n").filter((item) => item.trim())) { + archiveChainAfter = extendArchiveRecordChain(archiveChainAfter, line); + } + const ready = { + ...prepared, + phase: "ready", + archiveRecordsBefore: archiveState.records, + archiveChainBefore: archiveState.chain, + archiveChainAfter + }; + await replaceArchiveActive( + pendingPath, + renderPendingArchiveArtifact(ready, archiveBatch), + preparedStats, + fileSystem + ); + await fileSystem.rename(tmpPath, eventsPath); + await syncOwnedDirectory(path.dirname(eventsPath), { filesystem: fileSystem }); + await flushPendingArchive(archivePath, fileSystem, { + archiveRecordsBefore: ready.archiveRecordsBefore, + archiveChainBefore: ready.archiveChainBefore + }); + } finally { + await fileSystem.rm(tmpPath, { force: true }).catch(() => {}); + } return { archived: toArchive.length, kept: toKeep.length }; }, options); @@ -497,11 +1320,12 @@ export async function compactEvents(eventsPath, limit = RECENT_EVENT_LIMIT, opti /** * Move signal files older than retentionDays out of memory/signals/ and into - * memory/signals-archive.jsonl. Retention controls what stays in the routed - * daily window — it is not a licence to destroy what the user wrote. + * the bounded memory/signals-archive generation. Retention controls what stays + * in the routed daily window — it is not a licence to destroy what the user + * wrote. * * Crash-safe transaction: the batch is staged and fsynced, appended to the - * archive line-idempotently, and only then are the source files unlinked. + * archive generation line-idempotently, and only then are the source files unlinked. * The unlink is the commit point, so every crash point leaves a line in both * places (transient duplication the next run collapses) and never in neither. * Staging order is unlink order, so a crash mid-delete leaves a suffix of the @@ -576,7 +1400,9 @@ export async function trimSignals(signalsDir, retentionDays = SIGNAL_RETENTION_D if (sources.length === 0) return nothingToDo; if (staged.length > 0) { - await writeFileDurable(fileSystem, pendingPath, staged.map((line) => `${line}\n`).join("")); + const archiveBatch = staged.map((line) => `${line}\n`).join(""); + assertArchiveLineBounds(archiveBatch); + await stagePendingArchive(fileSystem, pendingPath, archiveBatch); await flushPendingArchive(archivePath, fileSystem); } diff --git a/packages/core/src/owned-state.mjs b/packages/core/src/owned-state.mjs index d7da7795..744bf7ae 100644 --- a/packages/core/src/owned-state.mjs +++ b/packages/core/src/owned-state.mjs @@ -33,12 +33,10 @@ export async function validateOwnedDirectoryIfPresent(directory, { return true; } -export function assertOwnedFileStats(stats, mode = 0o600) { - if (!stats || !stats.isFile() || stats.isSymbolicLink()) throw ownedStateError(); - if (process.platform === "win32") return; - if (stats.nlink !== 1 || stats.uid !== currentUid() || (stats.mode & 0o777) !== mode) { - throw ownedStateError(); - } +export function assertOwnedFileStats(stats, mode = 0o600, { + platform = process.platform +} = {}) { + assertOwnedPublicationFile(stats, mode, 1, platform); } export function sameFileIdentity(left, right) { @@ -91,6 +89,64 @@ export async function publishOwnedFileExclusive(filePath, bytes, { return publishedStats; } +// link(temporary, target) is the exclusive publication point. A process death +// before unlink(temporary) leaves exactly two names for the same owned inode; +// remove only that narrowly proven temporary, then restore strict nlink=1. +export async function recoverOwnedFileExclusivePublication(filePath, { + filesystem = fs, + mode = 0o600, + platform = process.platform +} = {}) { + let targetStats; + try { + targetStats = await filesystem.lstat(filePath, { bigint: true }); + } catch (error) { + if (error?.code === "ENOENT") return false; + throw error; + } + if (!isOwnedPublicationFile(targetStats, mode, 2, platform)) return false; + + const directory = path.dirname(filePath); + const basename = path.basename(filePath); + const prefix = `.${basename}.`; + const candidates = []; + for (const entry of await filesystem.readdir(directory, { withFileTypes: true })) { + if (!entry.name.startsWith(prefix) || !entry.name.endsWith(".tmp")) continue; + const token = entry.name.slice(prefix.length, -4); + if (!isUuid(token)) continue; + const candidatePath = path.join(directory, entry.name); + let candidateStats; + try { + candidateStats = await filesystem.lstat(candidatePath, { bigint: true }); + } catch (error) { + if (error?.code === "ENOENT") continue; + throw error; + } + if ( + isOwnedPublicationFile(candidateStats, mode, 2, platform) + && sameOwnedPublicationSnapshot(targetStats, candidateStats) + ) candidates.push({ path: candidatePath, stats: candidateStats }); + } + if (candidates.length !== 1) return false; + + const [currentTarget, currentCandidate] = await Promise.all([ + filesystem.lstat(filePath, { bigint: true }), + filesystem.lstat(candidates[0].path, { bigint: true }) + ]); + if ( + !sameOwnedPublicationSnapshot(targetStats, currentTarget) + || !sameOwnedPublicationSnapshot(candidates[0].stats, currentCandidate) + || !sameOwnedPublicationSnapshot(currentTarget, currentCandidate) + ) return false; + + await filesystem.unlink(candidates[0].path); + await syncOwnedDirectory(directory, { filesystem }); + const recovered = await filesystem.lstat(filePath, { bigint: true }); + assertOwnedPublicationFile(recovered, mode, 1, platform); + if (!sameOwnedPublicationObject(targetStats, recovered)) throw ownedStateError(); + return true; +} + export async function syncOwnedDirectory(directoryPath, { filesystem = fs } = {}) { const handle = await filesystem.open(directoryPath, "r"); try { @@ -106,6 +162,47 @@ export function ownedStateError() { return error; } +function isOwnedPublicationFile(stats, mode, links, platform = process.platform) { + try { + assertOwnedPublicationFile(stats, mode, links, platform); + return true; + } catch { + return false; + } +} + +function assertOwnedPublicationFile(stats, mode, links, platform = process.platform) { + if (!stats || !stats.isFile() || stats.isSymbolicLink()) throw ownedStateError(); + const nlink = typeof stats.nlink === "bigint" ? Number(stats.nlink) : stats.nlink; + if (nlink !== links) throw ownedStateError(); + if (platform === "win32") return; + const permissions = typeof stats.mode === "bigint" + ? Number(stats.mode & 0o777n) + : stats.mode & 0o777; + const uid = typeof stats.uid === "bigint" ? Number(stats.uid) : stats.uid; + if (uid !== currentUid() || permissions !== mode) throw ownedStateError(); +} + +function sameOwnedPublicationObject(left, right) { + return Boolean(left && right + && left.dev === right.dev + && left.ino === right.ino + && left.uid === right.uid + && left.mode === right.mode + && left.size === right.size); +} + +function sameOwnedPublicationSnapshot(left, right) { + return Boolean(sameOwnedPublicationObject(left, right) + && left.nlink === right.nlink + && left.mtimeNs === right.mtimeNs + && left.ctimeNs === right.ctimeNs); +} + +function isUuid(value) { + return /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(value); +} + function assertOwnedDirectoryStats(stats, mode, sharedParent = false) { if (!stats || !stats.isDirectory() || stats.isSymbolicLink()) throw ownedStateError(); if (process.platform === "win32") return; diff --git a/packages/core/src/search.mjs b/packages/core/src/search.mjs index f7c307bb..15579faf 100644 --- a/packages/core/src/search.mjs +++ b/packages/core/src/search.mjs @@ -1,5 +1,5 @@ import path from "node:path"; -import { createEvidenceReader } from "./evidence-reader.mjs"; +import { createEvidenceReader, EvidenceReadError } from "./evidence-reader.mjs"; import { isPathWithinLexically } from "./paths.mjs"; import { resolvePortableProjectIdentity, validateProjectSelector } from "./projects.mjs"; import { haystackHasInflectionOf } from "./search-inflections.mjs"; @@ -41,10 +41,6 @@ const SECRET_FILE_PATTERNS = [ // the file mtime (markdown). Missing age means no penalty. Newer wins // when lexical relevance is otherwise close. // -// TODO(L1-5): buildCorpusStats tokenizes the scanned candidate set per query. -// The persistent incremental term-frequency cache replaces buildCorpusStats -// call sites (rebuild on changed files only) without touching rankSearchHit. - export const RECENCY_HALF_LIFE_DAYS = 30; const RANK_TIER_WEIGHT = 1_000_000; // Tiers are absolute: any phrase hit outranks every terms hit, and so on down. @@ -130,126 +126,176 @@ export async function searchAios({ ] : [scope]; - // Scopes are independent; run them concurrently. Promise.all preserves input - // order, so the returned groups stay in the same order as before. - const groups = await Promise.all( - scopes.map(async (name) => ({ - scope: name, - results: await searchScope(name, { - aiosPath: resolvedAiosPath, - vaultPath: resolvedVaultPath, - vaultRoot, - query, - limit, - projectIdentity, - sessionFilters, - reader - }) - })) + return reader.withScopePreflight( + scopes, + (name, scopeReader) => inspectSearchScope(name, { + aiosPath: resolvedAiosPath, + vaultPath: resolvedVaultPath, + vaultRoot, + query, + limit, + projectIdentity, + sessionFilters, + reader: scopeReader + }), + (name, scopeReader, prepared) => discoverSearchScope(name, prepared, { + aiosPath: resolvedAiosPath, + query, + limit, + sessionFilters, + reader: scopeReader + }), + (name, scopeReader, prepared) => executeDiscoveredScope(name, prepared, { + aiosPath: resolvedAiosPath, + query, + limit, + sessionFilters, + reader: scopeReader + }), + async (transaction) => { + const groups = scopes + .filter((name) => transaction.has(name)) + .map((name) => ({ + scope: name, + results: transaction.get(name) + })); + return attachSearchMetadata(groups, { + requested: scope, + project: projectIdentity?.slug || null, + project_id: projectIdentity?.id || null, + projects_omitted: scope === "all" && projectIdentity === null + }, transaction.omissions); + } ); - return new Proxy(groups, { - get(target, property, receiver) { - if (property === "scope") { - return Object.freeze({ - requested: scope, - project: projectIdentity?.slug || null, - project_id: projectIdentity?.id || null, - projects_omitted: scope === "all" && projectIdentity === null - }); - } - return Reflect.get(target, property, receiver); +} + +function attachSearchMetadata(groups, scope, omissions) { + Object.defineProperties(groups, { + scope: { + value: Object.freeze({ ...scope }), + enumerable: false, + configurable: false, + writable: false + }, + omissions: { + value: omissions, + enumerable: false, + configurable: false, + writable: false } }); + return groups; } -async function searchScope(scope, { +async function inspectSearchScope(scope, { aiosPath, vaultPath, vaultRoot, query, - limit = DEFAULT_LIMIT, - projectIdentity = null, - sessionFilters = {}, + limit, + projectIdentity, + sessionFilters, reader }) { if (scope === "sessions") { - return searchSessionsScope(aiosPath, query, { limit, reader, ...sessionFilters }); + return inspectSessionsScope(aiosPath, { reader }); } if (scope === "memory") { - const memoryDir = path.join(aiosPath, "memory"); - const corpora = await Promise.all([ - searchMemoryDir(memoryDir, query, { limit, reader, root: aiosPath }), - ...MEMORY_NOTE_DIRS.map((relative) => searchMarkdownDir(path.join(memoryDir, relative), query, { - limit, - sourcePrefix: `memory/${relative}`, - reader, - root: aiosPath - })) - ]); - return interleaveByRank(corpora, limit); - } - if (scope === "context") { - return searchMarkdownDir(path.join(aiosPath, "context"), query, { - limit, - sourcePrefix: "context", + return inspectMemoryScope(path.join(aiosPath, "memory"), { reader, root: aiosPath }); } + const config = markdownScopeConfig(scope, { + aiosPath, + vaultPath, + vaultRoot, + projectIdentity + }); + if (!config) return Object.freeze({ kind: "empty" }); + const prepared = await reader.prepareTextCorpus( + config.root, + config.dir, + { + extensions: config.extensions, + includeFile: config.includeFile, + skipEntry: shouldSkipEntry, + concurrency: 32 + } + ); + return Object.freeze({ kind: "markdown", config, prepared }); +} + +async function discoverSearchScope(scope, prepared, { + aiosPath, + query, + limit, + sessionFilters, + reader +}) { + if (scope === "sessions") { + return discoverSessionsScope(aiosPath, query, prepared, { limit, reader, ...sessionFilters }); + } + if (scope === "memory") { + return discoverMemoryScope(prepared, { reader }); + } + return prepared; +} + +async function executeDiscoveredScope(_scope, prepared, { + aiosPath, + query, + limit, + sessionFilters, + reader +}) { + if (prepared?.kind === "sessions") { + return prepared.results; + } + if (prepared?.kind === "memory") { + return searchDiscoveredMemory(prepared, query, { limit, reader }); + } + if (prepared?.kind === "markdown") { + return searchPreparedMarkdownDir(prepared.prepared, prepared.config, query, { limit, reader }); + } + return []; +} + +function markdownScopeConfig(scope, { aiosPath, vaultPath, vaultRoot, projectIdentity }) { + if (scope === "context") { + return Object.freeze({ dir: path.join(aiosPath, "context"), root: aiosPath, sourcePrefix: "context", extensions: [".md"] }); + } if (scope === "vault") { - return searchMarkdownDir(vaultPath, query, { - limit, - sourcePrefix: "vault", - reader, - root: vaultRoot - }); + return Object.freeze({ dir: vaultPath, root: vaultRoot, sourcePrefix: "vault", extensions: [".md"] }); } if (scope === "projects") { if (!projectIdentity) validateProjectSelector(null); - return searchMarkdownDir(path.join(aiosPath, "projects", projectIdentity.slug), query, { - limit, + return Object.freeze({ + dir: path.join(aiosPath, "projects", projectIdentity.slug), + root: aiosPath, sourcePrefix: `projects/${projectIdentity.slug}`, - reader, - root: aiosPath + extensions: [".md"] }); } if (scope === "decisions") { - return searchMarkdownDir(path.join(aiosPath, "decisions"), query, { - limit, - sourcePrefix: "decisions", - reader, - root: aiosPath - }); + return Object.freeze({ dir: path.join(aiosPath, "decisions"), root: aiosPath, sourcePrefix: "decisions", extensions: [".md"] }); } if (scope === "skills") { - return searchMarkdownDir(path.join(aiosPath, "skills"), query, { - limit, - sourcePrefix: "skills", - extensions: [".md"], - reader, - root: aiosPath - }); + return Object.freeze({ dir: path.join(aiosPath, "skills"), root: aiosPath, sourcePrefix: "skills", extensions: [".md"] }); } if (scope === "references") { - return searchMarkdownDir(path.join(aiosPath, "references"), query, { - limit, - sourcePrefix: "references", - extensions: [".md"], - reader, - root: aiosPath - }); + return Object.freeze({ dir: path.join(aiosPath, "references"), root: aiosPath, sourcePrefix: "references", extensions: [".md"] }); } if (scope === "plugins") { - return searchMarkdownDir(path.join(aiosPath, "plugins"), query, { - limit, + return Object.freeze({ + dir: path.join(aiosPath, "plugins"), + root: aiosPath, sourcePrefix: "plugins", extensions: [".md", ".json"], - includeFile: (filePath) => filePath.endsWith(".md") || path.basename(filePath) === "manifest.json", - reader, - root: aiosPath + includeFile: (filePath) => filePath.endsWith(".md") || path.basename(filePath) === "manifest.json" }); } - return []; + return null; } // Each corpus arrives already ranked and already capped at the limit, but their @@ -312,26 +358,100 @@ export async function searchMemoryDir(memoryDir, query, { root = memoryDir } = {}) { const activeReader = reader || createEvidenceReader({ roots: [path.resolve(root)] }); - const sources = [ - { filePath: path.join(memoryDir, "events.jsonl"), source: "memory/events.jsonl" }, - { filePath: path.join(memoryDir, "events-archive.jsonl"), source: "memory/events-archive.jsonl" }, - { filePath: path.join(memoryDir, "signals-archive.jsonl"), source: "memory/signals-archive.jsonl" }, - ...await listSignalSources(path.join(memoryDir, "signals"), { reader: activeReader, root }) + const sources = await memorySourceDescriptors(memoryDir, { reader: activeReader, root }); + const entriesBySource = new Map(); + for (const { filePath, source } of sources) { + entriesBySource.set(source, await activeReader.readJsonl(root, filePath)); + } + return rankMemorySources(sources, entriesBySource, query, limit); +} + +async function inspectMemoryScope(memoryDir, { reader, root }) { + const sources = await memorySourceDescriptors(memoryDir, { reader, root }); + const entriesBySource = new Map(); + for (const { filePath, source } of sources) { + entriesBySource.set(source, await reader.prepareJsonlMetadata(root, filePath)); + } + const noteCorpora = []; + for (const relative of MEMORY_NOTE_DIRS) { + const dir = path.join(memoryDir, relative); + noteCorpora.push(Object.freeze({ + relative, + dir, + prepared: await reader.prepareTextCorpus( + root, + dir, + { extensions: [".md"], skipEntry: shouldSkipEntry, concurrency: 32 } + ) + })); + } + return Object.freeze({ kind: "memory", sources, entriesBySource, noteCorpora }); +} + +async function discoverMemoryScope(prepared, { reader }) { + const entriesBySource = new Map(); + for (const [source, metadata] of prepared.entriesBySource) { + entriesBySource.set(source, await reader.materializePreparedJsonl(metadata)); + } + return Object.freeze({ ...prepared, entriesBySource }); +} + +async function searchDiscoveredMemory(prepared, query, { limit, reader }) { + const entriesBySource = new Map( + [...prepared.entriesBySource].map(([source, entries]) => [source, reader.readPreparedJsonl(entries)]) + ); + const corpora = [rankMemorySources(prepared.sources, entriesBySource, query, limit)]; + for (const note of prepared.noteCorpora) { + corpora.push(await searchPreparedMarkdownDir( + note.prepared, + { dir: note.dir, sourcePrefix: `memory/${note.relative}` }, + query, + { limit, reader } + )); + } + return interleaveByRank(corpora, limit); +} + +async function memorySourceDescriptors(memoryDir, { reader, root }) { + const archives = await listMemoryArchiveSources(memoryDir, { reader, root }); + const eventSource = { + filePath: path.join(memoryDir, "events.jsonl"), + source: "memory/events.jsonl", + family: "events", + generation: "live" + }; + const signalSources = (await listSignalSources(path.join(memoryDir, "signals"), { reader, root })) + .map((source) => ({ + ...source, + family: "signals", + generation: "live" + })); + return [ + eventSource, + ...archives, + ...signalSources ]; +} +function rankMemorySources(sources, entriesBySource, query, limit) { // Every scanned entry (matched or not) feeds the corpus so IDF reflects how // common a term actually is in this folder, not just among the hits. const docs = []; const candidates = []; - for (const { filePath, source } of sources) { - const entries = await activeReader.readJsonl(root, filePath); + const retrySuppression = buildMemoryRetrySuppression(sources, entriesBySource); + for (const { source, family } of sources) { + const suppressed = retrySuppression.get(source) || new Map(); + const entries = entriesBySource.get(source) || []; for (const entry of entries) { const text = JSON.stringify(entry); + if (consumeSerializedEntry(suppressed, text)) continue; docs.push(text); const match = matchJsonEntry(entry, query); if (!match) continue; candidates.push({ text, + family, + recordId: typeof entry?.record_id === "string" ? entry.record_id : "", result: { source, match, @@ -346,18 +466,128 @@ export async function searchMemoryDir(memoryDir, query, { const corpus = buildCorpusStats(docs); const now = Date.now(); const terms = queryTerms(query); - return candidates - .map(({ text, result }) => { + const ranked = candidates + .map(({ text, family, recordId, result }) => { const haystack = text.toLowerCase(); const matchedTerms = terms.filter((term) => haystack.includes(term)); const ageMs = result.ts ? now - Date.parse(result.ts) : null; - return { result, rank: rankSearchHit({ kind: result.match.kind, matchedTerms, corpus, ageMs }) }; + return { family, recordId, result, rank: rankSearchHit({ kind: result.match.kind, matchedTerms, corpus, ageMs }) }; + }) + .sort((a, b) => (b.rank - a.rank) || compareTimestampsDesc(a.result.ts, b.result.ts)); + const representedFamilies = new Map(); + return ranked + .filter(({ family, recordId }) => { + if (!recordId) return true; + const priorFamily = representedFamilies.get(recordId); + if (!priorFamily) { + representedFamilies.set(recordId, family); + return true; + } + // Same-family multiplicity remains observable. Only the opposite-family + // mirror of one operation is the same conceptual result. + return priorFamily === family; }) - .sort((a, b) => (b.rank - a.rank) || compareTimestampsDesc(a.result.ts, b.result.ts)) .slice(0, limit) .map(({ result }) => result); } +function buildMemoryRetrySuppression(sources, entriesBySource) { + const suppression = new Map(sources.map(({ source }) => [source, new Map()])); + for (const family of new Set(sources.map(({ family }) => family).filter(Boolean))) { + const familySources = sources.filter((source) => source.family === family); + const newestShard = familySources.filter(({ generation }) => generation === "archive-shard").at(-1); + const active = familySources.find(({ generation }) => generation === "archive-active"); + const live = familySources.filter(({ generation }) => generation === "live"); + // Later groups win equal multiplicities: live is the current canonical + // source, then the newest immutable shard, then the transitional active + // archive. Losing retry generations are removed as whole multisets. + const groups = [ + { sources: active ? [active] : [] }, + { sources: newestShard ? [newestShard] : [] }, + { sources: live } + ].map((group) => { + const sourceCounts = new Map(group.sources.map(({ source }) => [ + source, + serializedEntryCounts(entriesBySource.get(source) || []) + ])); + const counts = new Map(); + for (const entryCounts of sourceCounts.values()) { + for (const [serialized, count] of entryCounts) { + counts.set(serialized, (counts.get(serialized) || 0) + count); + } + } + return { ...group, sourceCounts, counts }; + }); + const serializedEntries = new Set(groups.flatMap(({ counts }) => [...counts.keys()])); + for (const serialized of serializedEntries) { + let winner = null; + let winnerCount = 0; + for (const group of groups) { + const count = group.counts.get(serialized) || 0; + if (count >= winnerCount && count > 0) { + winner = group; + winnerCount = count; + } + } + for (const group of groups) { + if (group === winner) continue; + for (const { source } of group.sources) { + const count = group.sourceCounts.get(source).get(serialized) || 0; + if (count > 0) suppression.get(source).set(serialized, count); + } + } + } + } + return suppression; +} + +async function listMemoryArchiveSources(memoryDir, { reader, root }) { + const entries = await reader.listDirectory(root, memoryDir); + const families = ["events-archive", "signals-archive"]; + const sources = []; + for (const family of families) { + const matcher = new RegExp(`^${family}\\.(\\d{6})\\.jsonl$`); + const shards = entries + .map((entry) => ({ entry, match: entry.name.match(matcher) })) + .filter(({ match }) => match) + .sort((left, right) => Number(left.match[1]) - Number(right.match[1])); + for (const { entry } of shards) { + const source = `memory/${entry.name}`; + sources.push({ + filePath: path.join(memoryDir, entry.name), + source, + family: family === "events-archive" ? "events" : "signals", + generation: "archive-shard" + }); + } + const source = `memory/${family}.jsonl`; + sources.push({ + filePath: path.join(memoryDir, `${family}.jsonl`), + source, + family: family === "events-archive" ? "events" : "signals", + generation: "archive-active" + }); + } + return sources; +} + +function serializedEntryCounts(entries) { + const counts = new Map(); + for (const entry of entries) { + const serialized = JSON.stringify(entry); + counts.set(serialized, (counts.get(serialized) || 0) + 1); + } + return counts; +} + +function consumeSerializedEntry(counts, serialized) { + const remaining = counts.get(serialized) || 0; + if (remaining === 0) return false; + if (remaining === 1) counts.delete(serialized); + else counts.set(serialized, remaining - 1); + return true; +} + export async function searchJsonlEntries(filePath, query, { source, reader = null, root = path.dirname(filePath) }) { const activeReader = reader || createEvidenceReader({ roots: [path.resolve(root)] }); const entries = await activeReader.readJsonl(root, filePath); @@ -385,47 +615,59 @@ export async function searchMarkdownDir(dir, query, { root = dir } = {}) { const activeReader = reader || createEvidenceReader({ roots: [path.resolve(root)] }); - const files = await activeReader.listFiles(root, dir, { extensions, includeFile, skipEntry: shouldSkipEntry }); + const CONCURRENCY = 32; + return activeReader.withTextCorpus( + root, + dir, + { extensions, includeFile, skipEntry: shouldSkipEntry, concurrency: CONCURRENCY }, + (transaction) => rankMarkdownTransaction(transaction, dir, query, sourcePrefix, limit) + ); +} + +async function searchPreparedMarkdownDir(prepared, config, query, { limit, reader }) { + return reader.withPreparedTextCorpus( + prepared, + (transaction) => rankMarkdownTransaction( + transaction, + config.dir, + query, + config.sourcePrefix, + limit + ) + ); +} + +async function rankMarkdownTransaction(transaction, dir, query, sourcePrefix, limit) { + // Every accepted file is observed once by the transaction. Matching, + // snippet construction, corpus statistics, and ranking all stay inside + // its callback, so no derived result can escape before final validation. + const observedFiles = await transaction.mapFiles((observed) => + collectSearchFile(observed, dir, query, sourcePrefix) + ); const docs = []; const candidates = []; - - // Read files concurrently in bounded batches — I/O is the bottleneck, and a - // cap keeps us well under the open-file limit on large vaults. Every read - // file feeds the IDF corpus; only files with snippets become candidates. - const CONCURRENCY = 32; - for (let i = 0; i < files.length; i += CONCURRENCY) { - const batch = await Promise.all( - files.slice(i, i + CONCURRENCY).map((filePath) => - collectSearchFile(filePath, dir, query, sourcePrefix, { reader: activeReader, root }) - ) - ); - for (const item of batch) { - if (!item) continue; - docs.push(item.content); - if (item.candidate) candidates.push(item.candidate); - } + for (const item of observedFiles) { + if (!item) continue; + docs.push(item.content); + if (item.candidate) candidates.push(item.candidate); } const corpus = buildCorpusStats(docs); const now = Date.now(); const terms = queryTerms(query); - const ranked = []; - for (let i = 0; i < candidates.length; i += CONCURRENCY) { - const batch = candidates.slice(i, i + CONCURRENCY).map((candidate) => { - let ageMs = candidate.mtimeMs === null ? null : now - candidate.mtimeMs; - const haystack = candidate.content.toLowerCase(); - const matchedTerms = terms.filter((term) => haystack.includes(term)); - const rank = rankSearchHit({ - kind: candidate.kind, - matchedTerms, - corpus, - ageMs, - structuralBoost: candidate.structuralBoost - }); - return { result: candidate.result, rank }; + const ranked = candidates.map((candidate) => { + const ageMs = candidate.mtimeMs === null ? null : now - candidate.mtimeMs; + const haystack = candidate.content.toLowerCase(); + const matchedTerms = terms.filter((term) => haystack.includes(term)); + const rank = rankSearchHit({ + kind: candidate.kind, + matchedTerms, + corpus, + ageMs, + structuralBoost: candidate.structuralBoost }); - ranked.push(...batch); - } + return { result: candidate.result, rank }; + }); return ranked .sort((a, b) => (b.rank - a.rank) || a.result.file.localeCompare(b.result.file)) @@ -433,16 +675,7 @@ export async function searchMarkdownDir(dir, query, { .map(({ result }) => result); } -async function collectSearchFile(filePath, dir, query, sourcePrefix, { reader, root = dir } = {}) { - const observed = await reader.readText(root, filePath, { returnSnapshot: true }); - if (observed === null) { - const error = new Error("Search evidence changed while it was being read."); - error.code = "DOTAIOS_EVIDENCE_CHANGED"; - throw error; - } - const { content } = observed; - const mtimeMs = observed.stats.mtimeMs; - +function collectSearchFile({ filePath, content, mtimeMs }, dir, query, sourcePrefix) { const snippets = buildMarkdownSnippets(content, query); if (snippets.length === 0) return { content, candidate: null }; @@ -742,6 +975,44 @@ async function searchSessionsScope(aiosPath, query, { })); } +async function inspectSessionsScope(aiosPath, { reader }) { + const sessionsRoot = path.resolve(aiosPath, "memory", "sessions"); + const indexPath = path.join(sessionsRoot, "index.jsonl"); + return Object.freeze({ + kind: "sessions-metadata", + index: await reader.prepareJsonlMetadata(aiosPath, indexPath) + }); +} + +async function discoverSessionsScope(aiosPath, query, inspected, { + limit, + agent, + project, + since, + reader +} = {}) { + const index = await reader.materializePreparedJsonl(inspected.index); + const bodies = new Map(); + const replayReader = { + readJsonl: async () => reader.readPreparedJsonl(index), + readText: async (_root, filePath) => { + const resolved = path.resolve(filePath); + if (!bodies.has(resolved)) { + bodies.set(resolved, reader.prepareTextContent(aiosPath, resolved)); + } + return reader.readPreparedText(await bodies.get(resolved)); + } + }; + const results = await searchSessionsScope(aiosPath, query, { + agent, + project, + since, + limit, + reader: replayReader + }); + return Object.freeze({ kind: "sessions", results }); +} + function compareTimestampsDesc(a, b) { if (!a && !b) return 0; if (!a) return 1; diff --git a/packages/core/src/working-context.mjs b/packages/core/src/working-context.mjs index a8eed34d..425d549f 100644 --- a/packages/core/src/working-context.mjs +++ b/packages/core/src/working-context.mjs @@ -166,8 +166,8 @@ export async function selectWorkingContext(aiosPath, options = {}, dependencies const deduped = dedupeUpdateChannels(signals, events); const candidates = { - identity: compactHeader(identity), - priorities: compactHeader(priorities), + identity: compactHeader(stripMarkdownFrontmatter(identity)), + priorities: compactHeader(stripMarkdownFrontmatter(priorities)), decisions: recentDecisions(decisionsLog, MAX_DECISION_ITEMS), today: { focus: firstLine(readSection(todayNote, "Focus")), @@ -420,6 +420,14 @@ function compactHeader(content) { : visible; } +function stripMarkdownFrontmatter(content) { + const source = String(content || ""); + if (!source.startsWith("---\n") && !source.startsWith("---\r\n")) return source; + const lines = source.split(/\r?\n/); + const closing = lines.findIndex((line, index) => index > 0 && line.trim() === "---"); + return closing === -1 ? source : lines.slice(closing + 1).join("\n").replace(/^\n+/, ""); +} + function firstLine(content) { return String(content || "").split(/\r?\n/).map((line) => line.trim()).find(Boolean) || ""; } diff --git a/packages/mcp/src/server.mjs b/packages/mcp/src/server.mjs index e65d2a64..af87afec 100755 --- a/packages/mcp/src/server.mjs +++ b/packages/mcp/src/server.mjs @@ -9,7 +9,10 @@ import { WORKING_CONTEXT_OPERATIONAL_OVERHEAD_LIMIT, buildWorkingContextEnvelope } from "../../core/src/working-context-envelope.mjs"; -import { createEvidenceReader } from "../../core/src/evidence-reader.mjs"; +import { + DEFAULT_EVIDENCE_READ_LIMITS, + createEvidenceReader, +} from "../../core/src/evidence-reader.mjs"; import { defaultAiosPath, expandHome, resolveVaultPath } from "../../core/src/paths.mjs"; import { SEARCH_SCOPES, searchAios } from "../../core/src/search.mjs"; import { validateProjectSelector } from "../../core/src/projects.mjs"; @@ -22,6 +25,8 @@ const SERVER_VERSION = JSON.parse( ).version; const DEFAULT_RESULT_BUDGET = 6000; const MAX_SEARCH_CONFIG_BYTES = 1024 * 1024; +const SEARCH_QUERY_TRUNCATION_MARKER = "[query truncated]"; +const MIN_SEARCH_RESULT_BUDGET = deriveMinimumSearchResultBudget(); async function main(argv = process.argv.slice(2)) { if (argv.includes("--help") || argv.includes("-h")) { @@ -209,7 +214,7 @@ class DotaiosMcpServer { const limit = args.limit === undefined ? 10 : boundedInteger(args.limit, "limit", 1, 20); const budget = args.budget === undefined ? DEFAULT_RESULT_BUDGET - : boundedInteger(args.budget, "budget", 256, 32000); + : boundedInteger(args.budget, "budget", MIN_SEARCH_RESULT_BUDGET, 32000); const scope = optionalString(args.scope) || "all"; if (!SEARCH_SCOPES.includes(scope)) { throw protocolError(-32602, `scope must be one of: ${SEARCH_SCOPES.join(", ")}`); @@ -254,6 +259,7 @@ class DotaiosMcpServer { ? { projects: "omitted" } : null, groups, + omissions: groups.omissions, limit: budget }); } @@ -307,7 +313,13 @@ function tools() { description: "Optional canonical project slug or stable id. Required for project-only scope." }, limit: { type: "integer", minimum: 1, maximum: 20, default: 10 }, - budget: { type: "integer", minimum: 256, maximum: 32000, default: 6000 }, + budget: { + type: "integer", + minimum: MIN_SEARCH_RESULT_BUDGET, + maximum: 32000, + default: DEFAULT_RESULT_BUDGET, + description: "Character budget for the complete serialized search response, including full omission metadata.", + }, }, required: ["query"], }, @@ -330,14 +342,17 @@ function tools() { ]; } -function serializeBoundedSearchResults({ query, scope, scopeDetail, groups, limit }) { +function serializeBoundedSearchResults({ query, scope, scopeDetail, groups, omissions = [], limit }) { + const complete = omissions.length === 0; const selected = []; let truncated = false; outer: for (const group of groups) { for (const rawResult of group.results || []) { const result = sanitizeResultValue(rawResult); const candidate = [...selected, { scope: group.scope, ...result }]; - const candidateText = serializeSearchEnvelope({ query, scope, scopeDetail, results: candidate, limit, truncated: false }); + const candidateText = serializeSearchEnvelope({ + query, scope, scopeDetail, results: candidate, complete, omissions, limit, truncated: false + }); if (candidateText.length > limit) { truncated = true; break outer; @@ -347,25 +362,30 @@ function serializeBoundedSearchResults({ query, scope, scopeDetail, groups, limi } let boundedQuery = query; - let serialized = serializeSearchEnvelope({ query: boundedQuery, scope, scopeDetail, results: selected, limit, truncated }); + let serialized = serializeSearchEnvelope({ + query: boundedQuery, scope, scopeDetail, results: selected, complete, omissions, limit, truncated + }); while (serialized.length > limit && selected.length > 0) { selected.pop(); truncated = true; - serialized = serializeSearchEnvelope({ query: boundedQuery, scope, scopeDetail, results: selected, limit, truncated }); + serialized = serializeSearchEnvelope({ + query: boundedQuery, scope, scopeDetail, results: selected, complete, omissions, limit, truncated + }); } if (serialized.length > limit) { - const marker = "[query truncated]"; truncated = true; const fitted = fitSerializedString({ value: boundedQuery, - marker, + marker: SEARCH_QUERY_TRUNCATION_MARKER, limit, serialize: (candidate) => serializeSearchEnvelope({ query: candidate, scope, scopeDetail, results: selected, + complete, + omissions, limit, truncated }) @@ -443,22 +463,64 @@ function serializeSkillEnvelope({ intent, matches, limit, truncated }) { throw new Error("Could not stabilize skill response budget metadata"); } -function serializeSearchEnvelope({ query, scope, scopeDetail, results, limit, truncated }) { +function serializeSearchEnvelope({ query, scope, scopeDetail, results, complete, omissions, limit, truncated }) { + const envelope = (used) => ({ + query, + scope, + ...(scopeDetail ? { scope_selection: scopeDetail } : {}), + results, + complete, + omissions, + budget: { limit, used, truncated }, + }); + // Keep the representation fixed so changing `used` cannot oscillate across the limit. + const usePretty = JSON.stringify(envelope(limit), null, 2).length <= limit; let used = 0; for (let attempt = 0; attempt < 8; attempt += 1) { - const serialized = JSON.stringify({ - query, - scope, - ...(scopeDetail ? { scope_selection: scopeDetail } : {}), - results, - budget: { limit, used, truncated }, - }, null, 2); + const value = envelope(used); + const serialized = usePretty ? JSON.stringify(value, null, 2) : JSON.stringify(value); if (serialized.length === used) return serialized; used = serialized.length; } throw new Error("Could not stabilize search response budget metadata"); } +function deriveMinimumSearchResultBudget() { + const maximumSelector = "\u{10400}".repeat(200); + const maximumOmissions = SEARCH_SCOPES + .filter((scope) => scope !== "all") + .map((scope) => ({ + scope, + reason: "directory_entries_exceeded", + observed: { + files: DEFAULT_EVIDENCE_READ_LIMITS.maxFiles + 1, + bytes: DEFAULT_EVIDENCE_READ_LIMITS.maxBytes + 1, + entries: DEFAULT_EVIDENCE_READ_LIMITS.maxEntries + 1, + }, + inspection: "partially_enumerated", + recovery: { + code: "reduce_directory_entries", + message: "Move some directory entries elsewhere, then retry the same search.", + }, + })); + let candidate = 0; + for (let attempt = 0; attempt < 8; attempt += 1) { + const serialized = serializeSearchEnvelope({ + query: SEARCH_QUERY_TRUNCATION_MARKER, + scope: "all", + scopeDetail: { project: maximumSelector, project_id: maximumSelector }, + results: [], + complete: false, + omissions: maximumOmissions, + limit: candidate, + truncated: true, + }); + if (serialized.length === candidate) return candidate; + candidate = serialized.length; + } + throw new Error("Could not derive the minimum search response budget"); +} + function fitSerializedString({ value, marker, limit, serialize }) { const codePoints = Array.from(value); let lower = 0; diff --git a/scripts/bench-search.mjs b/scripts/bench-search.mjs new file mode 100644 index 00000000..281e5bf3 --- /dev/null +++ b/scripts/bench-search.mjs @@ -0,0 +1,1067 @@ +#!/usr/bin/env node + +import { createHash } from "node:crypto"; +import { constants as fsConstants, existsSync, realpathSync } from "node:fs"; +import fs from "node:fs/promises"; +import path from "node:path"; +import process from "node:process"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +import { createEvidenceReader } from "../packages/core/src/evidence-reader.mjs"; +import { searchAios, searchMarkdownDir } from "../packages/core/src/search.mjs"; + +const repoRoot = realpathSync(fileURLToPath(new URL("..", import.meta.url))); +const defaultManifestPath = path.join(repoRoot, "benchmarks", "search", "manifest.json"); +const PUBLIC_SEARCH_DIRECTORY_OPERATION_ALLOWANCE = Object.freeze({ + lstat: 512, + realpath: 256, + open: 16 +}); +const PUBLIC_SEARCH_OPERATION_PER_FILE_CEILING = Object.freeze({ + lstat: 4, + realpath: 0, + open: 1 +}); +const PUBLIC_SEARCH_OPERATION_DIRECTORY_MARGIN = Object.freeze({ + lstat: 1, + realpath: 3, + open: 0 +}); +const PROSE_WORDS = [ + "account", "action", "agent", "archive", "brief", "campaign", "client", "context", + "decision", "delivery", "evidence", "experiment", "feedback", "handoff", "identity", "insight", + "launch", "market", "memory", "metric", "note", "outcome", "owner", "pipeline", + "plan", "priority", "project", "proposal", "question", "receipt", "research", "review", + "roadmap", "scope", "search", "session", "signal", "source", "strategy", "summary", + "task", "timeline", "update", "validation", "workflow", "workspace", "writer", "year" +]; + +export async function loadManifest(filePath = defaultManifestPath) { + const manifest = JSON.parse(await fs.readFile(filePath, "utf8")); + validateManifest(manifest); + return manifest; +} + +export function manifestReceipt(manifest) { + validateManifest(manifest); + return sha256(canonicalJson(manifest)); +} + +export async function generateFixture({ manifest, destination, selection }) { + validateManifest(manifest); + validateSelection(manifest, selection); + const fixtureRoot = path.resolve(destination); + assertOutsideRepository(fixtureRoot); + await ensureEmptyDirectory(fixtureRoot); + + const expectedByQuery = Object.fromEntries(manifest.queries.map(({ id }) => [id, []])); + const inventory = []; + const concurrency = Math.max(1, Math.min(64, manifest.protocol.concurrency)); + const fixedMtime = new Date(manifest.corpus.generator.fixedMtime); + + for (let start = 0; start < selection.fileCount; start += concurrency) { + const generated = await Promise.all( + Array.from({ length: Math.min(concurrency, selection.fileCount - start) }, async (_, offset) => { + const index = start + offset; + const relativePath = fixtureRelativePath(index, selection, manifest); + const content = fixtureContent(index, selection, manifest); + const absolutePath = path.join(fixtureRoot, ...relativePath.split("/")); + await fs.mkdir(path.dirname(absolutePath), { recursive: true }); + await fs.writeFile(absolutePath, content, { flag: "wx" }); + await fs.utimes(absolutePath, fixedMtime, fixedMtime); + return { + path: relativePath, + bytes: Buffer.byteLength(content), + sha256: sha256(content), + mtime: manifest.corpus.generator.fixedMtime + }; + }) + ); + inventory.push(...generated); + } + + inventory.sort((left, right) => left.path.localeCompare(right.path)); + for (const query of manifest.queries) { + expectedByQuery[query.id] = expectedFilesForQuery(query, inventory) + .map((relativePath) => `vault/${relativePath}`); + } + const inventorySha256 = sha256(canonicalJson(inventory)); + const totalBytes = inventory.reduce((sum, entry) => sum + entry.bytes, 0); + return { + schemaVersion: "dotaios-search-fixture-receipt/v1", + manifestSha256: manifestReceipt(manifest), + generator: { ...manifest.corpus.generator }, + selection: { ...selection }, + fileCount: inventory.length, + totalBytes, + inventorySha256, + controlledResults: expectedByQuery, + inventory + }; +} + +export function assertExactResults(actual, expected, { queryId = "controlled" } = {}) { + if (expected.length > 0 && actual.length === 0) { + throw new Error(`Benchmark rejected empty controlled result for ${queryId}.`); + } + if (actual.length !== expected.length || actual.some((value, index) => value !== expected[index])) { + throw new Error( + `Benchmark result mismatch for ${queryId}: expected ${JSON.stringify(expected)}, got ${JSON.stringify(actual)}.` + ); + } +} + +export async function runBenchmark({ manifest, fixtureRoot, fixtureReceipt }) { + validateManifest(manifest); + validateFixtureReceipt(manifest, fixtureRoot, fixtureReceipt); + validateRuntime(manifest); + await verifyFixtureInventory(fixtureRoot, fixtureReceipt); + + const searches = []; + for (const query of manifest.queries) { + searches.push(await measureOperation({ + id: query.id, + manifest, + operation: () => runPublicSearchBenchmarkSample({ manifest, fixtureRoot, fixtureReceipt, query }) + })); + } + const rawSearchControl = await measureRawSearchControl({ manifest, fixtureRoot, fixtureReceipt }); + const rawReadControl = await measureOperation({ + id: "raw-read-control", + manifest, + operation: () => runRawReadControl({ manifest, fixtureRoot, fixtureReceipt }) + }); + const safeCorpusReadControl = await measureOperation({ + id: "safe-corpus-read-control", + manifest, + operation: () => runSafeCorpusReadBenchmarkSample({ manifest, fixtureRoot, fixtureReceipt }) + }); + await verifyFixtureInventory(fixtureRoot, fixtureReceipt); + + return createBenchmarkReport({ + manifest, + fixtureReceipt, + searches, + rawSearchControl, + rawReadControl, + safeCorpusReadControl + }); +} + +export function createBenchmarkReport({ + manifest, + fixtureReceipt, + searches, + rawSearchControl, + rawReadControl, + safeCorpusReadControl +}) { + const operationGate = assertPublicSearchOperationGate(searches, safeCorpusReadControl); + return Object.freeze({ + schemaVersion: "dotaios-search-benchmark-result/v2", + benchmarkId: manifest.benchmarkId, + manifestSha256: manifestReceipt(manifest), + inventorySha256: fixtureReceipt.inventorySha256, + selection: fixtureReceipt.selection, + runtime: runtimeReceipt(), + protocol: { ...manifest.protocol }, + searches, + searchSurface: Object.freeze({ + entryPoint: "searchAios", + requestedScope: "all", + completeness: "complete" + }), + operationGate, + rawSearchControl, + rawReadControl, + safeCorpusReadControl + }); +} + +export function publicSearchOperationCeiling({ fileCount, directoryCount }) { + for (const [name, value] of Object.entries({ fileCount, directoryCount })) { + if (!Number.isSafeInteger(value) || value < 0) { + throw new Error(`Public search operation ceiling requires ${name} to be a non-negative safe integer.`); + } + } + return Object.freeze(Object.fromEntries( + ["lstat", "realpath", "open"].map((name) => [ + name, + fileCount * PUBLIC_SEARCH_OPERATION_PER_FILE_CEILING[name] + + directoryCount * PUBLIC_SEARCH_OPERATION_DIRECTORY_MARGIN[name] + ]) + )); +} + +/** + * Measure only the residual canonical search logic over an immutable fixture. + * + * UNSAFE BENCHMARK-ONLY CONTROL: this deliberately omits lexical/canonical + * containment, ancestor validation, directory-generation checks, and the + * request budget. It must never move into packages/core or a product caller. + */ +export async function runRawSearchBenchmark({ manifest, fixtureRoot, fixtureReceipt }) { + validateManifest(manifest); + validateFixtureReceipt(manifest, fixtureRoot, fixtureReceipt); + validateRuntime(manifest); + await verifyFixtureInventory(fixtureRoot, fixtureReceipt); + const rawSearchControl = await measureRawSearchControl({ manifest, fixtureRoot, fixtureReceipt }); + await verifyFixtureInventory(fixtureRoot, fixtureReceipt); + return { + schemaVersion: "dotaios-search-raw-search-control-result/v1", + benchmarkId: manifest.benchmarkId, + control: Object.freeze({ + id: "raw-search-control-v1", + safety: "unsafe-benchmark-only", + protocolAuthority: "harness-schema-v1-reusing-frozen-manifest-sampling" + }), + manifestSha256: manifestReceipt(manifest), + inventorySha256: fixtureReceipt.inventorySha256, + selection: fixtureReceipt.selection, + runtime: runtimeReceipt(), + protocol: { ...manifest.protocol }, + rawSearchControl + }; +} + +export async function verifyFixtureInventory(fixtureRoot, fixtureReceipt) { + const actual = []; + const pending = [path.resolve(fixtureRoot)]; + while (pending.length > 0) { + const directory = pending.pop(); + const entries = await fs.readdir(directory, { withFileTypes: true }); + for (const entry of entries) { + const absolutePath = path.join(directory, entry.name); + if (entry.isSymbolicLink() || (!entry.isDirectory() && !entry.isFile())) { + throw new Error(`Fixture inventory contains an unsafe entry: ${path.relative(fixtureRoot, absolutePath)}.`); + } + if (entry.isDirectory()) { + pending.push(absolutePath); + continue; + } + const stats = await fs.lstat(absolutePath); + if (!stats.isFile() || stats.isSymbolicLink()) { + throw new Error(`Fixture inventory changed while validating: ${path.relative(fixtureRoot, absolutePath)}.`); + } + const bytes = await fs.readFile(absolutePath); + actual.push({ + path: path.relative(fixtureRoot, absolutePath).split(path.sep).join("/"), + bytes: bytes.byteLength, + sha256: sha256(bytes), + mtime: stats.mtime.toISOString() + }); + } + } + actual.sort((left, right) => left.path.localeCompare(right.path)); + const actualHash = sha256(canonicalJson(actual)); + if (actualHash !== fixtureReceipt.inventorySha256) { + throw new Error( + `Fixture inventory mismatch: expected ${fixtureReceipt.inventorySha256}, got ${actualHash}.` + ); + } + return actualHash; +} + +export async function runPublicSearchBenchmarkSample({ manifest, fixtureRoot, fixtureReceipt, query }) { + const { filesystem, counts, acceptedFilePathCounts, containmentPathCounts } = countingFilesystem({ fixtureRoot, fixtureReceipt }); + const reader = createEvidenceReader({ roots: [fixtureRoot], filesystem }); + const rss = monitorRss(manifest.protocol.rssPollIntervalMs); + const started = process.hrtime.bigint(); + try { + const groups = await searchAios({ + aiosPath: fixtureRoot, + vaultPath: fixtureRoot, + query: query.text, + limit: query.expectation.resultLimit ?? manifest.protocol.resultLimit, + evidenceReader: reader + }); + if (groups.omissions.length > 0) { + throw new Error(`Public search benchmark was incomplete: ${JSON.stringify(groups.omissions)}.`); + } + const vault = groups.find((group) => group.scope === "vault"); + if (!vault) throw new Error("Public search benchmark did not return the controlled vault scope."); + const results = vault.results; + const sources = results.map(({ source }) => source); + if (query.expectation.kind !== "none" && ( + fixtureReceipt.controlledResults[query.id].length === 0 + || sources.length === 0 + )) { + throw new Error(`Public search benchmark rejected vacuous controlled hits for ${query.id}.`); + } + assertExactResults(sources, fixtureReceipt.controlledResults[query.id], { queryId: query.id }); + const surface = Object.freeze({ + entryPoint: "searchAios", + requestedScope: "all", + completeness: "complete", + omissions: Object.freeze([]), + returnedScopes: Object.freeze(groups.map(({ scope }) => scope)) + }); + return { + durationMs: elapsedMs(started), + peakRssBytes: rss.stop(), + operations: { ...counts }, + operationBreakdown: { + acceptedFilePaths: { ...acceptedFilePathCounts }, + containmentPaths: { ...containmentPathCounts } + }, + readBudget: reader.snapshot(), + surface, + exactResults: sources, + outputSha256: sha256(canonicalJson(results)) + }; + } catch (error) { + rss.stop(); + throw error; + } +} + +async function measureRawSearchControl({ manifest, fixtureRoot, fixtureReceipt }) { + const controls = []; + for (const query of manifest.queries) { + const measurement = await measureOperation({ + id: query.id, + manifest, + operation: () => runUnsafeBenchmarkOnlyRawSearchSample({ + manifest, + fixtureRoot, + fixtureReceipt, + query, + expectedResults: fixtureReceipt.controlledResults[query.id] + }) + }); + controls.push({ safety: "unsafe-benchmark-only", ...measurement }); + } + return controls; +} + +/** + * Run one deliberately unsafe benchmark-only raw-search sample. + * + * This export exists only so the benchmark regression test can prove that the + * control rejects wrong output before a duration is accepted. Product code + * must use createEvidenceReader instead. + */ +export async function runUnsafeBenchmarkOnlyRawSearchSample({ + manifest, + fixtureRoot, + fixtureReceipt, + query, + expectedResults = fixtureReceipt.controlledResults[query.id] +}) { + const reader = createUnsafeBenchmarkOnlyRawSearchReader({ fixtureRoot, fixtureReceipt }); + const rss = monitorRss(manifest.protocol.rssPollIntervalMs); + const started = process.hrtime.bigint(); + try { + const results = await searchMarkdownDir(fixtureRoot, query.text, { + limit: query.expectation.resultLimit ?? manifest.protocol.resultLimit, + sourcePrefix: "vault", + reader, + root: fixtureRoot + }); + const sources = results.map(({ source }) => source); + assertExactResults(sources, expectedResults, { queryId: `raw-search-control:${query.id}` }); + return { + durationMs: elapsedMs(started), + peakRssBytes: rss.stop(), + operations: reader.operations(), + readBudget: reader.snapshot(), + exactResults: sources, + outputSha256: sha256(canonicalJson(results)) + }; + } catch (error) { + rss.stop(); + throw error; + } +} + +function createUnsafeBenchmarkOnlyRawSearchReader({ fixtureRoot, fixtureReceipt }) { + const resolvedRoot = path.resolve(fixtureRoot); + const inventoryByPath = new Map(fixtureReceipt.inventory.map((entry) => { + const absolutePath = path.join(resolvedRoot, ...entry.path.split("/")); + return [absolutePath, entry]; + })); + const files = [...inventoryByPath.keys()]; + let openedFiles = 0; + let openedBytes = 0; + + function assertFixtureRequest(root, requestedPath) { + if (path.resolve(root) !== resolvedRoot || path.resolve(requestedPath) !== resolvedRoot) { + throw new Error("Unsafe raw-search control may read only its immutable fixture root."); + } + } + + async function readObservedFile(filePath) { + const absolutePath = path.resolve(filePath); + const inventoryEntry = inventoryByPath.get(absolutePath); + if (!inventoryEntry) throw new Error("Unsafe raw-search control refused a path outside its receipt inventory."); + const handle = await fs.open(absolutePath, fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW); + openedFiles += 1; + try { + const before = await handle.stat(); + if (!before.isFile() || before.size !== inventoryEntry.bytes) { + throw new Error(`Raw-search fixture entry changed before read: ${inventoryEntry.path}.`); + } + const bytes = await handle.readFile(); + const after = await handle.stat(); + if ( + after.dev !== before.dev + || after.ino !== before.ino + || after.size !== before.size + || after.mtimeMs !== before.mtimeMs + || bytes.byteLength !== inventoryEntry.bytes + ) { + throw new Error(`Raw-search fixture entry changed during read: ${inventoryEntry.path}.`); + } + openedBytes += bytes.byteLength; + return Object.freeze({ + filePath: absolutePath, + content: decodeBenchmarkUtf8(bytes, inventoryEntry.path), + mtimeMs: after.mtimeMs, + stats: after + }); + } finally { + await handle.close(); + } + } + + return Object.freeze({ + async withTextCorpus(root, directoryPath, options, callback) { + assertFixtureRequest(root, directoryPath); + const concurrency = options.concurrency ?? 32; + const transaction = Object.freeze({ + async mapFiles(mapper) { + const mapped = new Array(files.length); + for (let start = 0; start < files.length; start += concurrency) { + const batch = await Promise.all( + files.slice(start, start + concurrency).map(async (filePath) => mapper(await readObservedFile(filePath))) + ); + for (const [offset, value] of batch.entries()) mapped[start + offset] = value; + } + return mapped; + } + }); + return callback(transaction); + }, + snapshot() { + return Object.freeze({ files: openedFiles, bytes: openedBytes, entries: 0 }); + }, + operations() { + return Object.freeze({ lstat: 0, realpath: 0, open: openedFiles }); + } + }); +} + +function decodeBenchmarkUtf8(bytes, relativePath) { + try { + return new TextDecoder("utf-8", { fatal: true }).decode(bytes); + } catch { + throw new Error(`Raw-search fixture is not valid UTF-8: ${relativePath}.`); + } +} + +async function runRawReadControl({ manifest, fixtureRoot, fixtureReceipt }) { + const rss = monitorRss(manifest.protocol.rssPollIntervalMs); + const started = process.hrtime.bigint(); + let fileCount = 0; + let totalBytes = 0; + const concurrency = manifest.protocol.rawReadControl.concurrency; + try { + for (let start = 0; start < fixtureReceipt.inventory.length; start += concurrency) { + const batch = await Promise.all( + fixtureReceipt.inventory.slice(start, start + concurrency).map(async (entry) => { + const filePath = path.join(fixtureRoot, ...entry.path.split("/")); + const handle = await fs.open(filePath, fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW); + try { + return (await handle.readFile()).byteLength; + } finally { + await handle.close(); + } + }) + ); + fileCount += batch.length; + totalBytes += batch.reduce((sum, bytes) => sum + bytes, 0); + } + if (fileCount !== fixtureReceipt.fileCount || totalBytes !== fixtureReceipt.totalBytes) { + throw new Error( + `Raw-read control mismatch: expected ${fixtureReceipt.fileCount} files/${fixtureReceipt.totalBytes} bytes, ` + + `got ${fileCount} files/${totalBytes} bytes.` + ); + } + return { + durationMs: elapsedMs(started), + peakRssBytes: rss.stop(), + operations: { lstat: 0, realpath: 0, open: fileCount }, + readBudget: { files: fileCount, bytes: totalBytes, entries: 0 }, + exactResults: { fileCount, totalBytes }, + outputSha256: sha256(`${fileCount}\0${totalBytes}`) + }; + } catch (error) { + rss.stop(); + throw error; + } +} + +export async function runSafeCorpusReadBenchmarkSample({ manifest, fixtureRoot, fixtureReceipt }) { + const { filesystem, counts, acceptedFilePathCounts, containmentPathCounts } = countingFilesystem({ fixtureRoot, fixtureReceipt }); + const reader = createEvidenceReader({ roots: [fixtureRoot], filesystem }); + const rss = monitorRss(manifest.protocol.rssPollIntervalMs); + const started = process.hrtime.bigint(); + try { + const exactResults = await reader.withScopePreflight( + ["corpus"], + (_scope, scopeReader) => scopeReader.prepareTextCorpus( + fixtureRoot, + fixtureRoot, + { extensions: [".md"], concurrency: manifest.protocol.concurrency } + ), + (_scope, _scopeReader, prepared) => prepared, + (_scope, scopeReader, prepared) => scopeReader.withPreparedTextCorpus( + prepared, + async (transaction) => { + const byteLengths = await transaction.mapFiles(({ content }) => Buffer.byteLength(content)); + return { + fileCount: byteLengths.length, + totalBytes: byteLengths.reduce((sum, bytes) => sum + bytes, 0) + }; + } + ), + (transaction) => transaction.get("corpus") + ); + if (exactResults.fileCount !== fixtureReceipt.fileCount || exactResults.totalBytes !== fixtureReceipt.totalBytes) { + throw new Error( + `Safe corpus-read control mismatch: expected ${fixtureReceipt.fileCount} files/${fixtureReceipt.totalBytes} bytes, ` + + `got ${exactResults.fileCount} files/${exactResults.totalBytes} bytes.` + ); + } + return { + durationMs: elapsedMs(started), + peakRssBytes: rss.stop(), + operations: { ...counts }, + operationBreakdown: { + acceptedFilePaths: { ...acceptedFilePathCounts }, + containmentPaths: { ...containmentPathCounts } + }, + readBudget: reader.snapshot(), + exactResults, + outputSha256: sha256(`${exactResults.fileCount}\0${exactResults.totalBytes}`) + }; + } catch (error) { + rss.stop(); + throw error; + } +} + +async function measureOperation({ id, manifest, operation }) { + const cold = []; + for (let index = 0; index < manifest.protocol.coldSamples; index += 1) cold.push(await operation()); + for (let index = 0; index < manifest.protocol.warmupSamples; index += 1) await operation(); + const warm = []; + for (let index = 0; index < manifest.protocol.measuredSamples; index += 1) warm.push(await operation()); + + assertStableSamples(id, cold); + assertStableSamples(id, warm); + if (cold[0].outputSha256 !== warm[0].outputSha256) { + throw new Error(`Benchmark ${id} output changed between cold and warm samples.`); + } + return { + id, + cold: summarizeSamples(cold), + warm: summarizeSamples(warm), + ...(warm[0].surface ? { surface: warm[0].surface } : {}), + exactResults: warm[0].exactResults, + outputSha256: warm[0].outputSha256 + }; +} + +export function assertPublicSearchOperationGate( + searches, + safeCorpusReadControl, + allowance = PUBLIC_SEARCH_DIRECTORY_OPERATION_ALLOWANCE +) { + const control = safeCorpusReadControl?.warm?.operations; + if (!control) throw new Error("Public search operation gate requires the safe corpus-read control."); + const operationNames = ["lstat", "realpath", "open"]; + assertOperationCounters("safe corpus-read control", control, operationNames); + assertOperationCounters("directory allowance", allowance, operationNames); + for (const search of searches || []) { + const operations = search?.warm?.operations; + if (!operations) throw new Error(`Public search operation gate lacks operations for ${search?.id || "unknown"}.`); + assertOperationCounters(`measured search ${search?.id || "unknown"}`, operations, operationNames); + } + for (const search of searches || []) { + const operations = search.warm.operations; + for (const name of operationNames) { + const maximum = control[name] + allowance[name]; + if (operations[name] > maximum) { + throw new Error( + `Public search operation gate failed for ${search.id}:${name}: ` + + `${operations[name]} > safe control ${control[name]} + directory allowance ${allowance[name]}.` + ); + } + } + } + return Object.freeze({ + comparison: "safe-corpus-read-control-plus-fixed-directory-allowance", + allowance: Object.freeze({ ...allowance }), + passed: true + }); +} + +function assertOperationCounters(label, operations, names) { + for (const name of names) { + if (!Number.isSafeInteger(operations?.[name]) || operations[name] < 0) { + throw new Error( + `Public search operation gate requires ${label}.${name} to be a non-negative safe integer.` + ); + } + } +} + +function summarizeSamples(samples) { + const durations = samples.map(({ durationMs }) => durationMs).sort((left, right) => left - right); + const summary = { + samples: samples.length, + medianMs: percentile(durations, 0.5), + p95Ms: percentile(durations, 0.95), + peakRssBytes: Math.max(...samples.map(({ peakRssBytes }) => peakRssBytes)), + operations: summarizeOperationCounts(samples.map(({ operations }) => operations)), + readBudget: samples[0].readBudget + }; + if (samples.every(({ operationBreakdown }) => operationBreakdown)) { + summary.operationBreakdown = Object.fromEntries( + ["acceptedFilePaths", "containmentPaths"].map((scope) => [ + scope, + summarizeOperationCounts(samples.map(({ operationBreakdown }) => operationBreakdown[scope])) + ]) + ); + } + return summary; +} + +function summarizeOperationCounts(counts) { + return Object.fromEntries(["lstat", "realpath", "open"].map((name) => { + const values = counts.map((entry) => entry[name]); + if (values.some((value) => value !== values[0])) { + throw new Error(`Benchmark operation count changed between samples for ${name}: ${values.join(", ")}.`); + } + return [name, values[0]]; + })); +} + +function assertStableSamples(id, samples) { + if (samples.length === 0) throw new Error(`Benchmark ${id} produced no timing samples.`); + const expectedHash = samples[0].outputSha256; + for (const sample of samples) { + if (sample.outputSha256 !== expectedHash) { + throw new Error(`Benchmark ${id} output changed between timing samples.`); + } + } +} + +function expectedFilesForQuery(query, inventory) { + if (query.expectation.kind === "none") return []; + let indices; + if (query.expectation.kind === "fixed-indices") { + indices = new Set(query.expectation.fileIndices); + } else { + indices = new Set(); + for (let index = query.expectation.remainder; index < inventory.length; index += query.expectation.modulo) { + indices.add(index); + } + } + const selected = inventory.filter((entry) => indices.has(fileIndex(entry.path))); + selected.sort((left, right) => left.path.localeCompare(right.path)); + return selected.slice(0, query.expectation.resultLimit ?? selected.length).map(({ path: relativePath }) => relativePath); +} + +function fixtureRelativePath(index, selection, manifest) { + const file = `note-${String(index).padStart(5, "0")}.md`; + if (selection.layout === "shallow") { + const buckets = manifest.corpus.layouts.shallow.bucketCount; + return `bucket-${String(index % buckets).padStart(2, "0")}/${file}`; + } + const branching = manifest.corpus.layouts.nested.branchingFactor; + const first = index % branching; + const second = Math.floor(index / branching) % branching; + const third = Math.floor(index / (branching ** 2)) % branching; + return `branch-${String(third).padStart(2, "0")}/branch-${String(second).padStart(2, "0")}/branch-${String(first).padStart(2, "0")}/${file}`; +} + +function fixtureContent(index, selection, manifest) { + const distribution = manifest.corpus.distributions[selection.distribution]; + const random = seededRandom(`${manifest.corpus.generator.version}:${manifest.corpus.generator.seed}:${selection.layout}:${selection.distribution}:${index}`); + const targetBytes = distribution.targetBytes.min + + Math.floor(random() * (distribution.targetBytes.max - distribution.targetBytes.min + 1)); + const frontmatter = index % distribution.frontmatterEvery === 0 + ? `---\ntitle: Fixture note ${String(index).padStart(5, "0")}\ncategory: benchmark\n---\n` + : ""; + const controlled = manifest.queries.flatMap((query) => queryAppliesToIndex(query, index) ? [query.text] : []); + let content = `${frontmatter}# Fixture note ${String(index).padStart(5, "0")}\n\n`; + if (controlled.length > 0) content += `${controlled.join("\n")}\n\n`; + + let tokenIndex = 0; + while (Buffer.byteLength(content) < targetBytes) { + if (selection.distribution === "prose") { + const words = Array.from({ length: 14 }, () => { + const vocabularyIndex = Math.floor(random() * distribution.vocabularySize); + const word = PROSE_WORDS[vocabularyIndex % PROSE_WORDS.length]; + return vocabularyIndex < PROSE_WORDS.length ? word : `${word}-${Math.floor(vocabularyIndex / PROSE_WORDS.length)}`; + }); + content += `${words.join(" ")}.\n`; + } else { + const token = sha256(`${index}:${tokenIndex}:${random()}`).slice(0, distribution.tokenLength); + content += `${token} `; + if (tokenIndex % 8 === 7) content += "\n"; + } + tokenIndex += 1; + } + return content.endsWith("\n") ? content : `${content}\n`; +} + +function queryAppliesToIndex(query, index) { + if (query.expectation.kind === "none") return false; + if (query.expectation.kind === "fixed-indices") return query.expectation.fileIndices.includes(index); + return index % query.expectation.modulo === query.expectation.remainder; +} + +function fileIndex(relativePath) { + const match = /note-(\d+)\.md$/.exec(relativePath); + if (!match) throw new Error(`Unexpected fixture path: ${relativePath}`); + return Number(match[1]); +} + +function countingFilesystem({ fixtureRoot = null, fixtureReceipt = null } = {}) { + const counts = { lstat: 0, realpath: 0, open: 0 }; + const acceptedFilePathCounts = { lstat: 0, realpath: 0, open: 0 }; + const containmentPathCounts = { lstat: 0, realpath: 0, open: 0 }; + const acceptedFiles = fixtureRoot && fixtureReceipt + ? new Set(fixtureReceipt.inventory.map(({ path: relativePath }) => + path.join(path.resolve(fixtureRoot), ...relativePath.split("/")) + )) + : new Set(); + const filesystem = new Proxy(fs, { + get(target, property, receiver) { + const value = Reflect.get(target, property, receiver); + if (typeof value !== "function") return value; + if (Object.hasOwn(counts, property)) { + return (...args) => { + counts[property] += 1; + const requestedPath = typeof args[0] === "string" ? path.resolve(args[0]) : null; + (requestedPath && acceptedFiles.has(requestedPath) ? acceptedFilePathCounts : containmentPathCounts)[property] += 1; + return value(...args); + }; + } + return value.bind(target); + } + }); + return { filesystem, counts, acceptedFilePathCounts, containmentPathCounts }; +} + +function monitorRss(intervalMs) { + let peak = process.memoryUsage.rss(); + const timer = setInterval(() => { + peak = Math.max(peak, process.memoryUsage.rss()); + }, intervalMs); + timer.unref(); + return { + stop() { + clearInterval(timer); + return Math.max(peak, process.memoryUsage.rss()); + } + }; +} + +function validateManifest(manifest) { + if (!manifest || manifest.schemaVersion !== "dotaios-search-benchmark/v1") { + throw new Error("Unsupported search benchmark manifest schema."); + } + if (!manifest.referenceMachine?.identifier || !manifest.referenceMachine?.powerProfile?.source) { + throw new Error("Manifest must fix a reference machine identifier and power profile."); + } + if (!Array.isArray(manifest.runtime?.supportedNodeMajors) || !manifest.runtime.supportedNodeMajors.includes(20) || !manifest.runtime.supportedNodeMajors.includes(22)) { + throw new Error("Manifest must support Node 20 and 22."); + } + if (!manifest.corpus?.generator?.version || !Number.isSafeInteger(manifest.corpus?.generator?.seed)) { + throw new Error("Manifest must fix a generator version and integer seed."); + } + if (canonicalJson(manifest.corpus.fileCounts) !== canonicalJson([500, 2500, 10000])) { + throw new Error("Manifest file counts must be exactly 500, 2500, and 10000."); + } + for (const name of ["shallow", "nested"]) if (!manifest.corpus.layouts?.[name]) throw new Error(`Missing ${name} layout.`); + for (const name of ["prose", "high-entropy"]) if (!manifest.corpus.distributions?.[name]) throw new Error(`Missing ${name} distribution.`); + if (!Array.isArray(manifest.corpus.scenarioMatrix) || manifest.corpus.scenarioMatrix.length === 0) { + throw new Error("Manifest scenario matrix must be a non-empty array."); + } + const scenarioKeys = new Set(); + for (const scenario of manifest.corpus.scenarioMatrix) { + if ( + !scenario + || typeof scenario.layout !== "string" + || typeof scenario.distribution !== "string" + || !manifest.corpus.layouts[scenario.layout] + || !manifest.corpus.distributions[scenario.distribution] + ) { + throw new Error("Each scenario must name a declared layout and distribution."); + } + const key = `${scenario.layout}:${scenario.distribution}`; + if (scenarioKeys.has(key)) throw new Error(`Duplicate benchmark scenario: ${key}.`); + scenarioKeys.add(key); + } + const queryIds = new Set((manifest.queries || []).map(({ id }) => id)); + for (const id of ["no-hit", "low-hit", "high-hit"]) if (!queryIds.has(id)) throw new Error(`Missing ${id} query.`); + for (const query of manifest.queries || []) { + const expectation = query.expectation; + if (!["none", "fixed-indices", "modulo"].includes(expectation?.kind)) { + throw new Error(`Unsupported query expectation kind for ${query.id}.`); + } + if (expectation.kind === "fixed-indices" && ( + !Array.isArray(expectation.fileIndices) + || expectation.fileIndices.some((index) => !Number.isSafeInteger(index) || index < 0) + )) throw new Error(`Fixed-indices expectation for ${query.id} must contain non-negative safe integers.`); + if (expectation.kind === "modulo" && ( + !Number.isSafeInteger(expectation.modulo) + || expectation.modulo < 1 + || !Number.isSafeInteger(expectation.remainder) + || expectation.remainder < 0 + || expectation.remainder >= expectation.modulo + )) throw new Error(`Modulo expectation for ${query.id} must define a positive modulus and valid remainder.`); + } + if (!Number.isSafeInteger(manifest.protocol?.coldSamples) || manifest.protocol.coldSamples < 1) throw new Error("Cold samples must be positive."); + if (!Number.isSafeInteger(manifest.protocol?.warmupSamples) || manifest.protocol.warmupSamples < 1) throw new Error("Warm-up samples must be positive."); + if (!Number.isSafeInteger(manifest.protocol?.measuredSamples) || manifest.protocol.measuredSamples < 20) throw new Error("Measured samples must be at least 20."); + if (!Number.isSafeInteger(manifest.protocol?.concurrency) || manifest.protocol.concurrency < 1) throw new Error("Fixture concurrency must be positive."); + if (!Number.isSafeInteger(manifest.protocol?.resultLimit) || manifest.protocol.resultLimit < 1) throw new Error("Result limit must be a positive safe integer."); + if (!Number.isSafeInteger(manifest.protocol?.rssPollIntervalMs) || manifest.protocol.rssPollIntervalMs < 1) throw new Error("RSS polling interval must be a positive safe integer."); + if (!manifest.protocol?.rawReadControl?.enabled) throw new Error("Raw-read control must be enabled."); + if (!Number.isSafeInteger(manifest.protocol.rawReadControl.concurrency) || manifest.protocol.rawReadControl.concurrency < 1) { + throw new Error("Raw-read control concurrency must be positive."); + } +} + +function validateRuntime(manifest) { + const runtimeMajor = Number(process.versions.node.split(".")[0]); + if (!manifest.runtime.supportedNodeMajors.includes(runtimeMajor)) { + throw new Error(`Node ${process.versions.node} is outside the manifest's supported major versions.`); + } +} + +function runtimeReceipt() { + return { + node: process.versions.node, + platform: process.platform, + architecture: process.arch + }; +} + +function validateSelection(manifest, selection) { + if (!manifest.corpus.fileCounts.includes(selection.fileCount)) throw new Error(`Unsupported file count: ${selection.fileCount}.`); + if (!manifest.corpus.layouts[selection.layout]) throw new Error(`Unsupported layout: ${selection.layout}.`); + if (!manifest.corpus.distributions[selection.distribution]) throw new Error(`Unsupported distribution: ${selection.distribution}.`); +} + +function validateFixtureReceipt(manifest, fixtureRoot, receipt) { + const resolved = path.resolve(fixtureRoot); + assertOutsideRepository(resolved); + if (receipt.schemaVersion !== "dotaios-search-fixture-receipt/v1") throw new Error("Invalid fixture receipt schema."); + if (receipt.manifestSha256 !== manifestReceipt(manifest)) throw new Error("Fixture receipt does not match the benchmark manifest."); + if (canonicalJson(receipt.generator) !== canonicalJson(manifest.corpus.generator)) { + throw new Error("Fixture receipt generator does not match the benchmark manifest."); + } + validateSelection(manifest, receipt.selection); + if (receipt.fileCount !== receipt.selection.fileCount || receipt.inventory.length !== receipt.fileCount) { + throw new Error("Fixture receipt file count is invalid."); + } + const seenPaths = new Set(); + let totalBytes = 0; + for (const entry of receipt.inventory) { + const normalized = typeof entry.path === "string" ? path.posix.normalize(entry.path) : null; + if ( + normalized !== entry.path + || path.posix.isAbsolute(entry.path) + || entry.path === ".." + || entry.path.startsWith("../") + || !entry.path.endsWith(".md") + || seenPaths.has(entry.path) + ) { + throw new Error("Fixture inventory contains an invalid or duplicate path."); + } + if (!Number.isSafeInteger(entry.bytes) || entry.bytes < 1 || !/^[a-f0-9]{64}$/.test(entry.sha256)) { + throw new Error(`Fixture inventory metadata is invalid for ${entry.path}.`); + } + if (entry.mtime !== manifest.corpus.generator.fixedMtime) { + throw new Error(`Fixture inventory mtime is invalid for ${entry.path}.`); + } + seenPaths.add(entry.path); + totalBytes += entry.bytes; + } + if (totalBytes !== receipt.totalBytes) throw new Error("Fixture receipt byte total is invalid."); + if (sha256(canonicalJson(receipt.inventory)) !== receipt.inventorySha256) throw new Error("Fixture inventory hash is invalid."); + for (const query of manifest.queries) { + if (!Array.isArray(receipt.controlledResults?.[query.id])) throw new Error(`Fixture receipt lacks ${query.id} expectations.`); + if (query.expectation.kind !== "none" && receipt.controlledResults[query.id].length === 0) { + throw new Error(`Fixture receipt has an empty controlled result for ${query.id}.`); + } + const derived = expectedFilesForQuery(query, receipt.inventory).map((relativePath) => `vault/${relativePath}`); + if ( + derived.length !== receipt.controlledResults[query.id].length + || derived.some((value, index) => value !== receipt.controlledResults[query.id][index]) + ) { + throw new Error(`Fixture controlled-result receipt mismatch for ${query.id}.`); + } + } +} + +function assertOutsideRepository(destination) { + let existingAncestor = path.resolve(destination); + const missingSegments = []; + while (!existsSync(existingAncestor)) { + const parent = path.dirname(existingAncestor); + if (parent === existingAncestor) break; + missingSegments.unshift(path.basename(existingAncestor)); + existingAncestor = parent; + } + const canonicalDestination = path.join(realpathSync(existingAncestor), ...missingSegments); + const relative = path.relative(repoRoot, canonicalDestination); + if (relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative))) { + throw new Error(`Benchmark fixtures must stay outside the repository: ${canonicalDestination}`); + } +} + +async function preflightFixtureDestination(destination) { + const fixtureRoot = path.resolve(destination); + assertOutsideRepository(fixtureRoot); + try { + const entries = await fs.readdir(fixtureRoot); + if (entries.length > 0) throw new Error(`Fixture destination must be empty: ${fixtureRoot}`); + } catch (error) { + if (error?.code !== "ENOENT") throw error; + } +} + +async function ensureEmptyDirectory(destination) { + await fs.mkdir(destination, { recursive: true }); + const entries = await fs.readdir(destination); + if (entries.length > 0) throw new Error(`Fixture destination must be empty: ${destination}`); +} + +function seededRandom(seed) { + let state = Number.parseInt(sha256(seed).slice(0, 8), 16) >>> 0; + return () => { + state += 0x6d2b79f5; + let value = state; + value = Math.imul(value ^ (value >>> 15), value | 1); + value ^= value + Math.imul(value ^ (value >>> 7), value | 61); + return ((value ^ (value >>> 14)) >>> 0) / 4294967296; + }; +} + +function canonicalJson(value) { + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; + if (value && typeof value === "object") { + return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${canonicalJson(value[key])}`).join(",")}}`; + } + return JSON.stringify(value); +} + +function sha256(value) { + return createHash("sha256").update(value).digest("hex"); +} + +function elapsedMs(started) { + return Number(process.hrtime.bigint() - started) / 1_000_000; +} + +function percentile(sorted, quantile) { + return sorted[Math.max(0, Math.ceil(sorted.length * quantile) - 1)]; +} + +function readOption(args, name, fallback = null) { + const index = args.indexOf(name); + if (index === -1) return fallback; + if (!args[index + 1] || args[index + 1].startsWith("--")) throw new Error(`${name} requires a value.`); + return args[index + 1]; +} + +async function main(args) { + const command = args[0]; + const manifestPath = path.resolve(readOption(args, "--manifest", defaultManifestPath)); + const manifest = await loadManifest(manifestPath); + if (command === "receipt") { + process.stdout.write(`${manifestReceipt(manifest)}\n`); + return; + } + if (command === "generate") { + const destination = readOption(args, "--output"); + if (!destination) throw new Error("generate requires --output outside the repository."); + await preflightFixtureDestination(destination); + const receiptPath = path.resolve(readOption(args, "--receipt", `${destination}.receipt.json`)); + const receiptOutput = await fs.open(receiptPath, "wx"); + try { + const receipt = await generateFixture({ + manifest, + destination, + selection: { + fileCount: Number(readOption(args, "--count")), + layout: readOption(args, "--layout"), + distribution: readOption(args, "--distribution") + } + }); + await receiptOutput.writeFile(`${JSON.stringify(receipt, null, 2)}\n`); + process.stdout.write(`${JSON.stringify({ fixtureRoot: path.resolve(destination), receiptPath, ...receipt, inventory: undefined }, null, 2)}\n`); + } finally { + await receiptOutput.close(); + } + return; + } + if (command === "run") { + const fixtureRoot = readOption(args, "--fixture"); + const receiptPath = readOption(args, "--receipt"); + const outputPath = readOption(args, "--output"); + if (!fixtureRoot || !receiptPath) throw new Error("run requires --fixture and --receipt."); + const reportOutput = outputPath ? await fs.open(path.resolve(outputPath), "wx") : null; + try { + const fixtureReceipt = JSON.parse(await fs.readFile(receiptPath, "utf8")); + const report = await runBenchmark({ manifest, fixtureRoot: path.resolve(fixtureRoot), fixtureReceipt }); + const output = `${JSON.stringify(report, null, 2)}\n`; + if (reportOutput) await reportOutput.writeFile(output); + process.stdout.write(output); + } finally { + if (reportOutput) await reportOutput.close(); + } + return; + } + if (command === "raw-search") { + const fixtureRoot = readOption(args, "--fixture"); + const receiptPath = readOption(args, "--receipt"); + const outputPath = readOption(args, "--output"); + if (!fixtureRoot || !receiptPath) throw new Error("raw-search requires --fixture and --receipt."); + const reportOutput = outputPath ? await fs.open(path.resolve(outputPath), "wx") : null; + try { + const fixtureReceipt = JSON.parse(await fs.readFile(receiptPath, "utf8")); + const report = await runRawSearchBenchmark({ + manifest, + fixtureRoot: path.resolve(fixtureRoot), + fixtureReceipt + }); + const output = `${JSON.stringify(report, null, 2)}\n`; + if (reportOutput) await reportOutput.writeFile(output); + process.stdout.write(output); + } finally { + if (reportOutput) await reportOutput.close(); + } + return; + } + throw new Error( + "Usage: bench-search.mjs receipt | generate --output --count <500|2500|10000> " + + "--layout --distribution [--receipt ] | " + + "run --fixture --receipt [--output ] | " + + "raw-search --fixture --receipt [--output ]" + ); +} + +if (process.argv[1] && pathToFileURL(path.resolve(process.argv[1])).href === import.meta.url) { + main(process.argv.slice(2)).catch((error) => { + console.error(`search benchmark failed: ${error.message}`); + process.exitCode = 1; + }); +} diff --git a/tests/cli/activate.test.mjs b/tests/cli/activate.test.mjs index f5c4b4bd..4dc5969c 100644 --- a/tests/cli/activate.test.mjs +++ b/tests/cli/activate.test.mjs @@ -50,6 +50,13 @@ describe("activateCommand — symlinks", () => { ]); assert.ok(activation.configuredContextCount > 0); assert.ok(activation.detectedClientCount > 0); + assert.deepEqual( + activation.configuredClientNames, + ["Claude Code", "Codex", "Gemini"], + "only clients with configured context bridges belong in the configured-name list" + ); + assert.ok(activation.detectedClientNames.includes("Cursor"), "bridge-less apps remain visible as detected"); + assert.equal(activation.configuredClientNames.includes("Cursor"), false); const symlinkPath = path.join(dirs.homePath, ".claude", "skills", "test-skill"); const stat = await fs.lstat(symlinkPath); @@ -64,6 +71,27 @@ describe("activateCommand — symlinks", () => { assert.ok(content.includes("Test Skill"), "symlink should resolve to skill content"); }); + it("returns the full stable activation result shape for a catalog conflict", async () => { + const { catalogConflictActivationResult } = await import( + path.join(repoRoot, "packages/cli/src/commands/activate.mjs") + ); + const results = [{ action: "kept", path: "skills/INDEX.md" }]; + + assert.deepEqual( + catalogConflictActivationResult({ conflicts: [{ path: "skills/INDEX.md" }], results }), + { + detectedClientCount: 0, + configuredContextCount: 0, + detectedClientNames: [], + configuredClientNames: [], + blockedContextCount: 0, + blockedHermesCount: 0, + blockedCatalogCount: 1, + results + } + ); + }); + it("refuses to connect a temporary AIOS into the real home", async () => { const { activateCommand } = await import( path.join(repoRoot, "packages/cli/src/commands/activate.mjs") diff --git a/tests/cli/connect_gemini_bridge.test.mjs b/tests/cli/connect_gemini_bridge.test.mjs index 5ab6d196..4a6c8b4e 100644 --- a/tests/cli/connect_gemini_bridge.test.mjs +++ b/tests/cli/connect_gemini_bridge.test.mjs @@ -328,8 +328,8 @@ test("doctor stays green after connect gemini", () => { env: { ...process.env, HOME: base, DOTAIOS_NO_UPDATE_CHECK: "1" } }); - assert.match(doctor.stdout, /\[ok\] Gemini bridge/); - assert.doesNotMatch(doctor.stdout, /Bridge points to a different AIOS folder/); + assert.match(doctor.stdout, /\[ok\] Gemini\n/); + assert.doesNotMatch(doctor.stdout, /Connection points to a different AIOS folder/); } finally { fs.rmSync(base, { recursive: true, force: true }); } diff --git a/tests/cli/doctor.test.mjs b/tests/cli/doctor.test.mjs index 13297c77..c3936804 100644 --- a/tests/cli/doctor.test.mjs +++ b/tests/cli/doctor.test.mjs @@ -31,7 +31,7 @@ async function runDoctor({ aiosPath, homePath, detection }) { const prevExitCode = process.exitCode; console.log = (...args) => lines.push(args.join(" ")); try { - await doctorCommand(["--path", aiosPath, "--home", homePath], { detection }); + await doctorCommand(["--verbose", "--path", aiosPath, "--home", homePath], { detection }); return { output: lines.join("\n"), exitCode: process.exitCode }; } finally { console.log = origLog; @@ -306,7 +306,7 @@ describe("doctorCommand", () => { const origLog = console.log.bind(console); console.log = (...args) => lines.push(args.join(" ")); try { - await doctorCommand(["--path", aiosPath, "--home", tmpHome]); + await doctorCommand(["--verbose", "--path", aiosPath, "--home", tmpHome]); } finally { console.log = origLog; await fs.rm(tmpHome, { recursive: true, force: true }); diff --git a/tests/cli/first_run_language.test.mjs b/tests/cli/first_run_language.test.mjs new file mode 100644 index 00000000..03441fc0 --- /dev/null +++ b/tests/cli/first_run_language.test.mjs @@ -0,0 +1,264 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { spawnSync } from "node:child_process"; +import test from "node:test"; +import assert from "node:assert/strict"; + +const repoRoot = path.resolve(new URL("../..", import.meta.url).pathname); +const cli = path.join(repoRoot, "packages", "cli", "src", "index.mjs"); +const quietEnv = { ...process.env, PATH: "/usr/bin:/bin", DOTAIOS_NO_UPDATE_CHECK: "1" }; + +test("default setup preview names only detected clients and keeps operator detail behind --verbose", () => { + const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "dotaios-first-run-setup-")); + const homePath = path.join(tempRoot, "home"); + const aiosPath = path.join(homePath, "aios"); + fs.mkdirSync(path.join(homePath, ".claude"), { recursive: true }); + + try { + const concise = run(["setup", "--dry-run", "--path", aiosPath, "--home", homePath]); + assert.equal(concise.status, 0, concise.stderr); + assert.match(concise.stdout, /Claude Code/); + assert.match(concise.stdout, /Your context stays in|After setup/i); + assert.match(concise.stdout, /~\/aios/); + assert.doesNotMatch(concise.stdout, /not detected|managed (?:bridge|skill)|projection|DotAIOS-managed/i); + assert.doesNotMatch(concise.stdout, new RegExp(escapeRegExp(tempRoot))); + + const verbose = run(["setup", "--dry-run", "--verbose", "--path", aiosPath, "--home", homePath]); + assert.equal(verbose.status, 0, verbose.stderr); + assert.match(verbose.stdout, /managed (?:bridge|skill)|DotAIOS-managed/i); + assert.match(verbose.stdout, new RegExp(escapeRegExp(tempRoot))); + } finally { + fs.rmSync(tempRoot, { recursive: true, force: true }); + } +}); + +test("default setup preview gives a clear next step when no supported app is detected", () => { + const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "dotaios-first-run-no-client-")); + const homePath = path.join(tempRoot, "home"); + const aiosPath = path.join(homePath, "aios"); + fs.mkdirSync(homePath, { recursive: true }); + + try { + const result = run(["setup", "--dry-run", "--path", aiosPath, "--home", homePath]); + assert.equal(result.status, 0, result.stderr); + assert.match(result.stdout, /No supported local AI app was detected/); + assert.match(result.stdout, /Install Claude Code, Codex, or Gemini CLI, then run setup again/); + assert.doesNotMatch(result.stdout, /managed (?:bridge|skill)|projection|DotAIOS-managed/i); + assert.doesNotMatch(result.stdout, new RegExp(escapeRegExp(tempRoot))); + } finally { + fs.rmSync(tempRoot, { recursive: true, force: true }); + } +}); + +test("real setup keeps init detail concise by default and restores it with --verbose", () => { + const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "dotaios-first-run-real-")); + const conciseHome = path.join(tempRoot, "concise-home"); + const conciseAios = path.join(conciseHome, "aios"); + const verboseHome = path.join(tempRoot, "verbose-home"); + const verboseAios = path.join(verboseHome, "aios"); + fs.mkdirSync(path.join(conciseHome, ".claude"), { recursive: true }); + fs.mkdirSync(path.join(verboseHome, ".claude"), { recursive: true }); + + try { + const concise = run([ + "setup", "--yes", "--skip-reveal", + "--path", conciseAios, "--home", conciseHome + ]); + assert.equal(concise.status, 0, `${concise.stdout}\n${concise.stderr}`); + const conciseOutput = `${concise.stdout}\n${concise.stderr}`; + assert.match(conciseOutput, /Folder ready\. Claude Code can now use your context\./); + assert.match( + conciseOutput, + new RegExp(`Open the ${escapeRegExp(conciseAios)} folder or make it your working directory`) + ); + assert.doesNotMatch( + conciseOutput.replaceAll(conciseAios, ""), + new RegExp(escapeRegExp(tempRoot)) + ); + assert.doesNotMatch(conciseOutput, /AIOS path:|Vault path:|Files: \d+ created|\nNext steps:\n/); + assert.doesNotMatch(conciseOutput, /not detected on this machine/); + + const verbose = run([ + "setup", "--yes", "--skip-reveal", "--verbose", + "--path", verboseAios, "--home", verboseHome + ]); + assert.equal(verbose.status, 0, `${verbose.stdout}\n${verbose.stderr}`); + assert.match(verbose.stdout, new RegExp(`AIOS path: ${escapeRegExp(verboseAios)}`)); + assert.match(verbose.stdout, new RegExp(`Vault path: ${escapeRegExp(path.join(verboseAios, "vault"))}`)); + assert.match(verbose.stdout, /Files: \d+ created, \d+ updated, \d+ kept/); + assert.match(verbose.stdout, /\nNext steps:\n/); + } finally { + fs.rmSync(tempRoot, { recursive: true, force: true }); + } +}); + +test("direct init retains its detailed completion output", () => { + const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "dotaios-first-run-init-")); + const aiosPath = path.join(tempRoot, "aios"); + + try { + const result = run(["init", "--yes", "--path", aiosPath]); + assert.equal(result.status, 0, `${result.stdout}\n${result.stderr}`); + assert.match(result.stdout, new RegExp(`AIOS path: ${escapeRegExp(aiosPath)}`)); + assert.match(result.stdout, new RegExp(`Vault path: ${escapeRegExp(path.join(aiosPath, "vault"))}`)); + assert.match(result.stdout, /Files: \d+ created, \d+ updated, \d+ kept/); + assert.match(result.stdout, /\nNext steps:\n/); + } finally { + fs.rmSync(tempRoot, { recursive: true, force: true }); + } +}); + +test("real setup names only clients whose context bridge was configured", () => { + const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "dotaios-first-run-mixed-clients-")); + const homePath = path.join(tempRoot, "home"); + const aiosPath = path.join(homePath, "aios"); + fs.mkdirSync(path.join(homePath, ".claude"), { recursive: true }); + fs.mkdirSync(path.join(homePath, ".cursor"), { recursive: true }); + + try { + const result = run([ + "setup", "--yes", "--skip-reveal", + "--path", aiosPath, "--home", homePath + ]); + assert.equal(result.status, 0, `${result.stdout}\n${result.stderr}`); + assert.match(result.stdout, /Folder ready\. Claude Code can now use your context\./); + assert.doesNotMatch(result.stdout, /Folder ready\.[^\n]*Cursor[^\n]*can now use your context/); + } finally { + fs.rmSync(tempRoot, { recursive: true, force: true }); + } +}); + +test("setup preview distinguishes bridge clients from bridge-less detected apps", () => { + const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "dotaios-first-run-mixed-preview-")); + const homePath = path.join(tempRoot, "home"); + const aiosPath = path.join(homePath, "aios"); + fs.mkdirSync(path.join(homePath, ".claude"), { recursive: true }); + fs.mkdirSync(path.join(homePath, ".cursor"), { recursive: true }); + + try { + const result = run(["setup", "--dry-run", "--path", aiosPath, "--home", homePath]); + assert.equal(result.status, 0, `${result.stdout}\n${result.stderr}`); + assert.match(result.stdout, /\[detected\] Claude Code — setup will connect it to your context\./); + assert.match( + result.stdout, + /\[detected\] Cursor — needs native or project-specific setup before it can use your context\./ + ); + assert.doesNotMatch(result.stdout, /\[detected\] Cursor — setup will connect it to your context\./); + } finally { + fs.rmSync(tempRoot, { recursive: true, force: true }); + } +}); + +test("default doctor reports the user outcome without listing absent clients", () => { + const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "dotaios-first-run-doctor-")); + const homePath = path.join(tempRoot, "home"); + const aiosPath = path.join(homePath, "aios"); + fs.mkdirSync(homePath, { recursive: true }); + fs.mkdirSync(aiosPath, { recursive: true }); + fs.writeFileSync(path.join(aiosPath, "aios.json"), `${JSON.stringify({ schema_version: "1.2.0", ai_tools: [] })}\n`); + fs.writeFileSync(path.join(aiosPath, "AGENTS.md"), "# AIOS\n"); + + try { + const concise = run(["doctor", "--path", aiosPath, "--home", homePath]); + assert.equal(concise.status, 0, concise.stderr); + assert.match(concise.stdout, /No supported local AI app was detected|No local AI app is connected/i); + assert.match(concise.stdout, /install|activate/i); + assert.match(concise.stdout, /~\/aios/); + assert.doesNotMatch(concise.stdout, /\(not installed\)|native skills|managed bridge|projection/i); + assert.doesNotMatch(concise.stdout, new RegExp(escapeRegExp(tempRoot))); + + const verbose = run(["doctor", "--verbose", "--path", aiosPath, "--home", homePath]); + assert.equal(verbose.status, 0, verbose.stderr); + assert.match(verbose.stdout, /\(not installed\)|native skills/i); + assert.match(verbose.stdout, new RegExp(escapeRegExp(tempRoot))); + } finally { + fs.rmSync(tempRoot, { recursive: true, force: true }); + } +}); + +test("setup and doctor document --verbose and continue rejecting unknown options", () => { + for (const command of ["setup", "doctor"]) { + const help = run([command, "--help"]); + assert.equal(help.status, 0, help.stderr); + assert.match(help.stdout, /--verbose/); + + const invalid = run([command, "--definitely-unknown"]); + assert.equal(invalid.status, 1); + assert.match(invalid.stderr, /Unknown option|unknown/i); + } +}); + +test("default doctor blocking output states the outcome and one safe next action", () => { + const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "dotaios-first-run-blocked-")); + const homePath = path.join(tempRoot, "home"); + const missingAios = path.join(homePath, "aios"); + fs.mkdirSync(homePath, { recursive: true }); + + try { + const result = run(["doctor", "--path", missingAios, "--home", homePath]); + assert.equal(result.status, 1); + assert.match(result.stdout, /blocking issue/i); + assert.match(result.stdout, /npx dotaios setup/); + assert.match(result.stdout, /~\/aios/); + assert.doesNotMatch(result.stdout, /\(not installed\)|managed (?:bridge|skill)|projection/i); + assert.doesNotMatch(result.stdout, new RegExp(escapeRegExp(tempRoot))); + } finally { + fs.rmSync(tempRoot, { recursive: true, force: true }); + } +}); + +test("default doctor preserves bridge and projection words inside folder paths", () => { + const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "dotaios-first-run-doctor-path-")); + const homePath = path.join(tempRoot, "home"); + const missingAios = path.join(homePath, "my-bridge-folder", "projection-data", "aios"); + fs.mkdirSync(homePath, { recursive: true }); + + try { + const result = run(["doctor", "--path", missingAios, "--home", homePath]); + assert.equal(result.status, 1); + assert.match(result.stdout, /~\/my-bridge-folder\/projection-data\/aios/); + assert.doesNotMatch(result.stdout, /my-connection-folder|connection-data/); + } finally { + fs.rmSync(tempRoot, { recursive: true, force: true }); + } +}); + +test("default doctor explains a wrong-folder connection without operator jargon", () => { + const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "dotaios-first-run-doctor-connection-")); + const homePath = path.join(tempRoot, "home"); + const aiosPath = path.join(homePath, "aios"); + const otherAios = path.join(tempRoot, "other-aios"); + const claudeBridge = path.join(homePath, ".claude", "CLAUDE.md"); + + try { + const initialized = run(["init", "--yes", "--path", aiosPath]); + assert.equal(initialized.status, 0, initialized.stderr); + fs.mkdirSync(path.dirname(claudeBridge), { recursive: true }); + fs.writeFileSync(claudeBridge, [ + "", + `Read ${path.join(otherAios, "AGENTS.md")} first.`, + "" + ].join("\n")); + + const result = run(["doctor", "--path", aiosPath, "--home", homePath]); + assert.equal(result.status, 0, result.stderr); + assert.match(result.stdout, /connected to a different AIOS folder/i); + assert.match(result.stdout, /installed but is not connected to this AIOS folder/i); + assert.doesNotMatch(result.stdout, /\bbridge\b|managed bridge|projection/i); + } finally { + fs.rmSync(tempRoot, { recursive: true, force: true }); + } +}); + +function run(args) { + return spawnSync(process.execPath, [cli, ...args], { + cwd: repoRoot, + encoding: "utf8", + env: quietEnv + }); +} + +function escapeRegExp(value) { + return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} diff --git a/tests/cli/memory_promotion.test.mjs b/tests/cli/memory_promotion.test.mjs index 791cdb2d..ed572c66 100644 --- a/tests/cli/memory_promotion.test.mjs +++ b/tests/cli/memory_promotion.test.mjs @@ -323,7 +323,10 @@ test("promotion and compaction share one writer lock without losing the receipt" const compactionFilesystem = { ...fsp, async rename(source, destination) { - if (source === `${eventsPath}.tmp` && destination === eventsPath) { + const isReplacement = path.dirname(source) === path.dirname(eventsPath) + && path.basename(source).startsWith(`.${path.basename(eventsPath)}.`) + && path.basename(source).endsWith(".tmp"); + if (isReplacement && destination === eventsPath) { renameStarted(); await allowRename; } diff --git a/tests/cli/release_safety_regressions.test.mjs b/tests/cli/release_safety_regressions.test.mjs index 91b76363..a1c3bcc4 100644 --- a/tests/cli/release_safety_regressions.test.mjs +++ b/tests/cli/release_safety_regressions.test.mjs @@ -105,9 +105,9 @@ for (const [label, bridgeContent] of Object.entries(malformedBridgeCases)) { test(`doctor warns instead of reporting a healthy Claude bridge with ${label} managed markers`, (t) => { const output = runDoctor(t, `doctor-${label}`, bridgeContent); - assert.match(output, /\[warn\] Claude Code bridge/); - assert.match(output, /managed bridge markers are malformed/i); - assert.doesNotMatch(output, /\[ok\] Claude Code bridge/); + assert.match(output, /\[warn\] Claude Code\n/); + assert.match(output, /connection markers are damaged/i); + assert.doesNotMatch(output, /\[ok\] Claude Code\n/); }); } @@ -120,9 +120,9 @@ test("doctor ignores a correct target outside a valid managed block for another "" ].join("\n")); - assert.match(output, /\[warn\] Claude Code bridge/); - assert.match(output, /bridge points to a different AIOS folder/i); - assert.doesNotMatch(output, /\[ok\] Claude Code bridge/); + assert.match(output, /\[warn\] Claude Code\n/); + assert.match(output, /connected to a different AIOS folder/i); + assert.doesNotMatch(output, /\[ok\] Claude Code\n/); }); test("doctor ignores an expected-path comment inside a valid block with the wrong pointer", (t) => { @@ -134,9 +134,9 @@ test("doctor ignores an expected-path comment inside a valid block with the wron "" ].join("\n")); - assert.match(output, /\[warn\] Claude Code bridge/); - assert.match(output, /bridge points to a different AIOS folder/i); - assert.doesNotMatch(output, /\[ok\] Claude Code bridge/); + assert.match(output, /\[warn\] Claude Code\n/); + assert.match(output, /connected to a different AIOS folder/i); + assert.doesNotMatch(output, /\[ok\] Claude Code\n/); }); // A bridge was validated by comparing the pointer line against a path doctor @@ -157,11 +157,11 @@ const managedPointerBridge = (target) => [ test("doctor warns when the bridge points at this AIOS folder but its entrypoint is gone", (t) => { const output = runDoctor(t, "doctor-missing-entrypoint", managedPointerBridge); - assert.match(output, /\[warn\] Claude Code bridge/); + assert.match(output, /\[warn\] Claude Code\n/); assert.match(output, /AGENTS\.md/); - assert.doesNotMatch(output, /\[ok\] Claude Code bridge/); + assert.doesNotMatch(output, /\[ok\] Claude Code\n/); // The pointer is correct — repointing is not the remedy and cannot help. - assert.doesNotMatch(output, /bridge points to a different AIOS folder/i); + assert.doesNotMatch(output, /connection points to a different AIOS folder/i); }); // The bridge an older release wrote is still ours and still names this folder, @@ -196,14 +196,14 @@ for (const [index, retiredPointer] of bridgePointer("/placeholder").retired.entr { entrypoint: true } ); - assert.match(output, /\[warn\] Claude Code bridge/); - assert.doesNotMatch(output, /\[ok\] Claude Code bridge/); + assert.match(output, /\[warn\] Claude Code\n/); + assert.doesNotMatch(output, /\[ok\] Claude Code\n/); // The remedy has to name the command that rewrites the block. It is the only // one that does, and the user has no other signal that anything is stale. assert.match(output, /dotaios activate/); // Not "points to a different AIOS folder": the folder is right, the spelling // is old. Saying it is wrong sends the user to --overwrite for no reason. - assert.doesNotMatch(output, /bridge points to a different AIOS folder/i); + assert.doesNotMatch(output, /connection points to a different AIOS folder/i); }); } @@ -220,7 +220,7 @@ test("only the @ import is described as loading the whole folder", (t) => { olderBridgeWith((target) => `DotAIOS entrypoint (read this file first): ${path.join(target, "AGENTS.md")}`), { entrypoint: true } ); - assert.match(prose, /\[warn\] Claude Code bridge/); + assert.match(prose, /\[warn\] Claude Code\n/); assert.doesNotMatch(prose, /loads your whole AIOS folder/); }); @@ -239,8 +239,8 @@ test("doctor warns about the bridge when the whole AIOS folder was removed", (t) expectStatus: 1 }); - assert.match(output, /\[warn\] Claude Code bridge/); - assert.doesNotMatch(output, /\[ok\] Claude Code bridge/); + assert.match(output, /\[warn\] Claude Code\n/); + assert.doesNotMatch(output, /\[ok\] Claude Code\n/); }); // Nothing else in the suite asserts a GREEN bridge; every other bridge @@ -249,8 +249,8 @@ test("doctor warns about the bridge when the whole AIOS folder was removed", (t) test("doctor reports a healthy Claude bridge when the entrypoint really is there", (t) => { const output = runDoctor(t, "doctor-healthy-bridge", managedPointerBridge, { entrypoint: true }); - assert.match(output, /\[ok\] Claude Code bridge/); - assert.doesNotMatch(output, /\[warn\] Claude Code bridge/); + assert.match(output, /\[ok\] Claude Code\n/); + assert.doesNotMatch(output, /\[warn\] Claude Code\n/); }); // A marker on disk means init never finished, so the scaffold is still partial. @@ -274,7 +274,7 @@ test("doctor names the non-destructive remedy for an unmanaged bridge file", (t) entrypoint: true }); - assert.match(output, /\[warn\] Claude Code bridge/); + assert.match(output, /\[warn\] Claude Code\n/); assert.match(output, /activate --merge/); assert.doesNotMatch(output, /activate --overwrite/); }); diff --git a/tests/cli/search-safety.test.mjs b/tests/cli/search-safety.test.mjs index 2c5692ce..cefd4c6a 100644 --- a/tests/cli/search-safety.test.mjs +++ b/tests/cli/search-safety.test.mjs @@ -72,6 +72,101 @@ test("CLI search refuses a linked aios.json before authorizing an external vault assert.deepEqual(snapshotTree(tempRoot), before); }); +test("CLI search qualifies result counts and no-results on stdout when a scope is omitted", () => { + const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "dotaios-cli-search-partial-")); + const aiosPath = path.join(tempRoot, "aios"); + const vaultPath = path.join(tempRoot, "external-vault"); + try { + const initialized = run(["init", "--path", aiosPath, "--yes"]); + assert.equal(initialized.status, 0, initialized.stderr); + fs.mkdirSync(vaultPath); + fs.writeFileSync(path.join(aiosPath, "context", "work.md"), "# Work\n\nCLI_PARTIAL_SEARCH_CANARY\n"); + fs.writeFileSync(path.join(vaultPath, "oversized.md"), Buffer.alloc((4 * 1024 * 1024) + 1, 0x61)); + const configPath = path.join(aiosPath, "aios.json"); + const config = JSON.parse(fs.readFileSync(configPath, "utf8")); + fs.writeFileSync(configPath, `${JSON.stringify({ ...config, vault_path: vaultPath }, null, 2)}\n`); + + const result = spawnSync(process.execPath, [ + cli, + "search", + "CLI_PARTIAL_SEARCH_CANARY", + "--scope", + "all", + "--path", + aiosPath + ], { cwd: repoRoot, encoding: "utf8" }); + const empty = spawnSync(process.execPath, [ + cli, + "search", + "CLI_PARTIAL_NO_MATCH", + "--scope", + "all", + "--path", + aiosPath + ], { cwd: repoRoot, encoding: "utf8" }); + + assert.equal(result.status, 2); + assert.equal(empty.status, 2); + assert.match(result.stdout, /CLI_PARTIAL_SEARCH_CANARY/); + assert.match(result.stdout, /(?:partial|incomplete).*vault|vault.*(?:partial|incomplete)/i); + assert.match(empty.stdout, /no results.*(?:partial|incomplete).*vault|(?:partial|incomplete).*vault.*no results/is); + assert.doesNotMatch(result.stderr, /CLI_PARTIAL_SEARCH_CANARY/); + assert.match(result.stderr, /incomplete.*vault.*oversized file/is); + assert.match(empty.stderr, /incomplete.*vault.*oversized file/is); + assert.doesNotMatch(`${result.stdout}\n${result.stderr}\n${empty.stdout}\n${empty.stderr}`, escaped(tempRoot)); + } finally { + fs.rmSync(tempRoot, { recursive: true, force: true }); + } +}); + +test("CLI update appears once in search while two intentional saves remain two", () => { + const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "dotaios-cli-update-search-")); + const aiosPath = path.join(tempRoot, "aios"); + const note = "CLI_UPDATE_IDENTITY_CANARY"; + try { + run(["init", "--path", aiosPath, "--yes"]); + + run(["update", note, "--path", aiosPath]); + const once = run(["search", note, "--scope", "memory", "--path", aiosPath]); + assert.match(once.stdout, /1 result\(s\) found\./); + + run(["update", note, "--path", aiosPath]); + const twice = run(["search", note, "--scope", "memory", "--path", aiosPath]); + assert.match(twice.stdout, /2 result\(s\) found\./); + } finally { + fs.rmSync(tempRoot, { recursive: true, force: true }); + } +}); + +test("CLI search matches either representation before collapsing an update pair", () => { + const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "dotaios-cli-update-pair-fields-")); + const aiosPath = path.join(tempRoot, "aios"); + const note = "IDENTITY_QUERY_CANARY"; + try { + run(["init", "--path", aiosPath, "--yes"]); + run(["update", note, "--path", aiosPath]); + + const signalPath = path.join( + aiosPath, + "memory", + "signals", + fs.readdirSync(path.join(aiosPath, "memory", "signals")).find((name) => name.endsWith(".jsonl")) + ); + const signal = JSON.parse(fs.readFileSync(signalPath, "utf8").trim()); + const event = JSON.parse(fs.readFileSync(path.join(aiosPath, "memory", "events.jsonl"), "utf8").trim()); + assert.equal(signal.record_id, event.record_id, "the fixture must be one paired update operation"); + signal.ts = "2040-01-02T03:04:05.678Z"; + assert.notEqual(signal.ts, event.ts); + fs.writeFileSync(signalPath, `${JSON.stringify(signal)}\n`); + + const result = run(["search", signal.ts, "--scope", "memory", "--path", aiosPath]); + assert.match(result.stdout, /1 result\(s\) found\./); + assert.match(result.stdout, /2040-01-02T03:04:05\.678Z/); + } finally { + fs.rmSync(tempRoot, { recursive: true, force: true }); + } +}); + test("CLI project search keeps corpus selection separate from session attribution", () => { const fixture = createProjectSourceRetrievalFixture(); try { diff --git a/tests/cli/setup.test.mjs b/tests/cli/setup.test.mjs index 455afe9b..a8f05f6d 100644 --- a/tests/cli/setup.test.mjs +++ b/tests/cli/setup.test.mjs @@ -27,6 +27,7 @@ test("setup --dry-run previews concrete actions without DotAIOS-managed changes" path.resolve(repoRoot, "packages/cli/src/index.mjs"), "setup", "--dry-run", + "--verbose", "--path", aiosPath, "--home", homePath ], { @@ -81,7 +82,7 @@ test("setup --dry-run promises every skill-link directory the real run creates", const target = ["--path", aiosPath, "--home", homePath]; try { - const preview = spawnSync(process.execPath, [cli, "setup", "--dry-run", ...target], { + const preview = spawnSync(process.execPath, [cli, "setup", "--dry-run", "--verbose", ...target], { encoding: "utf8", env }); @@ -122,6 +123,7 @@ test("setup --dry-run reports an unmanaged bridge collision without changing it" path.resolve(repoRoot, "packages/cli/src/index.mjs"), "setup", "--dry-run", + "--verbose", "--path", aiosPath, "--home", homePath ], { @@ -154,6 +156,7 @@ test("setup --dry-run preserves a bridge whose managed markers are reversed", () path.resolve(repoRoot, "packages/cli/src/index.mjs"), "setup", "--dry-run", + "--verbose", "--path", aiosPath, "--home", homePath ], { @@ -192,6 +195,7 @@ test("setup --dry-run preserves a bridge with duplicate managed markers", () => path.resolve(repoRoot, "packages/cli/src/index.mjs"), "setup", "--dry-run", + "--verbose", "--path", aiosPath, "--home", homePath ], { @@ -397,6 +401,10 @@ test("non-interactive setup does not download the optional web browsing engine b env: { ...process.env, HOME: processHomePath, PATH: "/usr/bin:/bin" } }); assert.equal(result.status, 0, result.stderr); + assert.ok( + result.stdout.includes(` 2. Open the ${aiosPath} folder or make it your working directory.`), + `setup guidance must name the resolved --path target, got:\n${result.stdout}` + ); assert.match(result.stdout, /Web browsing engine: not installed.*plain fetch remains available/); assert.equal(fsSync.existsSync(path.join(homePath, ".dotaios", "bin", "lightpanda")), false); assert.doesNotMatch(result.stdout, /All set\./); diff --git a/tests/core/evidence-reader.test.mjs b/tests/core/evidence-reader.test.mjs index 5bb8dc25..6abd0ad2 100644 --- a/tests/core/evidence-reader.test.mjs +++ b/tests/core/evidence-reader.test.mjs @@ -5,12 +5,49 @@ import path from "node:path"; import test from "node:test"; import assert from "node:assert/strict"; -import { createEvidenceReader } from "../../packages/core/src/evidence-reader.mjs"; +import { createEvidenceReader, EvidenceReadError } from "../../packages/core/src/evidence-reader.mjs"; +import { readContainedSnapshotFile } from "../../packages/core/src/contained-read.mjs"; function tmpDir() { return fs.mkdtempSync(path.join(os.tmpdir(), "dotaios-evidence-reader-")); } +test("snapshot file reads fail closed before open when no no-follow capability exists", async () => { + const root = tmpDir(); + const filePath = path.join(root, "note.md"); + fs.writeFileSync(filePath, "bounded evidence\n"); + let openCalls = 0; + const filesystem = new Proxy(fsp, { + get(target, property) { + if (property === "open") { + return async (...args) => { + openCalls += 1; + return target.open(...args); + }; + } + const value = Reflect.get(target, property, target); + return typeof value === "function" ? value.bind(target) : value; + } + }); + + await assert.rejects( + () => readContainedSnapshotFile(root, filePath, { + filesystem, + parentPath: root, + parentSnapshot: { stats: fs.lstatSync(root, { bigint: true }), ancestors: [] }, + expectedSnapshot: { + type: "regular-file", + stats: fs.lstatSync(filePath), + ancestors: [] + }, + noFollowFlag: 0, + encoding: "utf8" + }), + (error) => error?.code === "DOTAIOS_BOUNDED_FILE_READ_UNAVAILABLE" + ); + assert.equal(openCalls, 0, "an unsupported platform must be refused before pathname open"); +}); + test("evidence reader lists one directory deterministically without traversing it", async () => { const root = tmpDir(); const dir = path.join(root, "skills"); @@ -159,3 +196,989 @@ test("evidence reader rejects a real directory replacement after enumeration", a (error) => error?.code === "DOTAIOS_EVIDENCE_CHANGED" ); }); + +test("evidence reader maps a shallow and nested text corpus inside one validated transaction", async () => { + const root = tmpDir(); + const corpus = path.join(root, "vault"); + const nested = path.join(corpus, "nested"); + fs.mkdirSync(nested, { recursive: true }); + fs.writeFileSync(path.join(corpus, "alpha.md"), "# Alpha\n\nALPHA_CANARY\n"); + fs.writeFileSync(path.join(nested, "beta.md"), "# Beta\n\nBETA_CANARY\n"); + fs.writeFileSync(path.join(nested, "ignored.txt"), "IGNORED_CANARY\n"); + const reader = createEvidenceReader({ roots: [root] }); + + const observed = await reader.withTextCorpus( + root, + corpus, + { extensions: [".md"] }, + (transaction) => transaction.mapFiles(({ filePath, content }) => ({ filePath, content })) + ); + + assert.deepEqual(observed, [ + { filePath: path.join(corpus, "alpha.md"), content: "# Alpha\n\nALPHA_CANARY\n" }, + { filePath: path.join(nested, "beta.md"), content: "# Beta\n\nBETA_CANARY\n" } + ]); +}); + +test("evidence corpus transactions revalidate the authorized root even when the corpus is missing", async () => { + const parent = tmpDir(); + const root = path.join(parent, "authorized"); + const parked = path.join(parent, "authorized-parked"); + fs.mkdirSync(root); + const reader = createEvidenceReader({ roots: [root] }); + + await assert.rejects( + () => reader.withTextCorpus( + root, + path.join(root, "missing"), + { extensions: [".md"] }, + async (transaction) => { + assert.deepEqual(await transaction.mapFiles(({ content }) => content), []); + fs.renameSync(root, parked); + fs.mkdirSync(root); + return "must not escape"; + } + ), + (error) => error?.code === "DOTAIOS_EVIDENCE_CHANGED" + ); +}); + +test("evidence corpus revalidation accepts an unchanged symlinked authorized root", async (t) => { + if (process.platform === "win32") return t.skip("symlink creation requires elevated Windows privileges"); + const parent = tmpDir(); + const realRoot = path.join(parent, "real-authorized"); + const linkedRoot = path.join(parent, "linked-authorized"); + fs.mkdirSync(realRoot); + fs.writeFileSync(path.join(realRoot, "note.md"), "SYMLINKED_ROOT_CORPUS_CANARY\n"); + fs.symlinkSync(realRoot, linkedRoot, "dir"); + const reader = createEvidenceReader({ roots: [linkedRoot] }); + + const observed = await reader.withTextCorpus( + linkedRoot, + linkedRoot, + { extensions: [".md"] }, + (transaction) => transaction.mapFiles(({ content }) => content) + ); + + assert.deepEqual(observed, ["SYMLINKED_ROOT_CORPUS_CANARY\n"]); +}); + +test("evidence corpus transactions revalidate the nearest observed ancestor of a nested missing corpus", async () => { + const root = tmpDir(); + const existing = path.join(root, "existing"); + const missing = path.join(existing, "future", "corpus"); + fs.mkdirSync(existing); + const reader = createEvidenceReader({ roots: [root] }); + + await assert.rejects( + () => reader.withTextCorpus( + root, + missing, + { extensions: [".md"] }, + async (transaction) => { + assert.deepEqual(await transaction.mapFiles(({ content }) => content), []); + fs.mkdirSync(missing, { recursive: true }); + fs.writeFileSync(path.join(missing, "inserted.md"), "# Inserted\n"); + return "must not escape"; + } + ), + (error) => error?.code === "DOTAIOS_EVIDENCE_CHANGED" + ); +}); + +test("evidence corpus transactions support an external authorized root and selector predicate", async () => { + const parent = tmpDir(); + const aiosRoot = path.join(parent, "aios"); + const externalRoot = path.join(parent, "external-vault"); + fs.mkdirSync(aiosRoot); + fs.mkdirSync(path.join(externalRoot, "plugin"), { recursive: true }); + fs.writeFileSync(path.join(externalRoot, "plugin", "manifest.json"), '{"name":"selected"}\n'); + fs.writeFileSync(path.join(externalRoot, "plugin", "private.json"), '{"name":"private"}\n'); + fs.writeFileSync(path.join(externalRoot, "note.md"), "# Not selected\n"); + const reader = createEvidenceReader({ roots: [aiosRoot, externalRoot] }); + + const observed = await reader.withTextCorpus( + externalRoot, + externalRoot, + { includeFile: (filePath) => path.basename(filePath) === "manifest.json" }, + (transaction) => transaction.mapFiles((file) => file) + ); + + assert.deepEqual(observed, [{ + filePath: path.join(externalRoot, "plugin", "manifest.json"), + content: '{"name":"selected"}\n', + mtimeMs: fs.statSync(path.join(externalRoot, "plugin", "manifest.json")).mtimeMs + }]); +}); + +test("evidence corpus transactions preserve ordinary contained-read semantics for in-root hard links", async () => { + const root = tmpDir(); + const original = path.join(root, "original.txt"); + const linked = path.join(root, "linked.md"); + fs.writeFileSync(original, "# Shared inode\n\nHARD_LINK_CANARY\n"); + fs.linkSync(original, linked); + const ordinaryReader = createEvidenceReader({ roots: [root] }); + const expected = await ordinaryReader.readText(root, linked); + const transactionReader = createEvidenceReader({ roots: [root] }); + + const observed = await transactionReader.withTextCorpus( + root, + root, + { extensions: [".md"] }, + (transaction) => transaction.mapFiles(({ filePath, content }) => ({ filePath, content })) + ); + + assert.deepEqual(observed, [{ filePath: linked, content: expected }]); +}); + +test("evidence corpus transaction capabilities close before successful results escape", async () => { + const root = tmpDir(); + const source = path.join(root, "note.md"); + fs.writeFileSync(source, "# Note\n"); + let finalValidationArmed = false; + let releaseValidation; + let validationEntered; + const entered = new Promise((resolve) => { validationEntered = resolve; }); + const released = new Promise((resolve) => { releaseValidation = resolve; }); + const filesystem = Object.create(fsp); + filesystem.lstat = async (targetPath, options) => { + if ( + finalValidationArmed + && path.resolve(String(targetPath)) === path.resolve(root) + && options?.bigint === true + ) { + finalValidationArmed = false; + validationEntered(); + await released; + } + return fsp.lstat(targetPath, options); + }; + const reader = createEvidenceReader({ roots: [root], filesystem }); + let capturedTransaction; + let settled = false; + const pending = reader.withTextCorpus( + root, + root, + { extensions: [".md"] }, + async (transaction) => { + capturedTransaction = transaction; + const mapped = await transaction.mapFiles(({ content }) => content); + finalValidationArmed = true; + return mapped; + } + ); + pending.finally(() => { settled = true; }).catch(() => {}); + + await entered; + await new Promise((resolve) => setImmediate(resolve)); + assert.equal(settled, false); + releaseValidation(); + assert.deepEqual(await pending, ["# Note\n"]); + assert.throws( + () => capturedTransaction.mapFiles(({ content }) => content), + (error) => error?.code === "DOTAIOS_EVIDENCE_TRANSACTION_CLOSED" + ); +}); + +test("evidence corpus transactions preserve caller-work failures", async () => { + const root = tmpDir(); + fs.writeFileSync(path.join(root, "note.md"), "# Note\n"); + const reader = createEvidenceReader({ roots: [root] }); + const failure = new Error("ranking failed"); + + await assert.rejects( + () => reader.withTextCorpus( + root, + root, + { extensions: [".md"] }, + async (transaction) => { + await transaction.mapFiles(({ content }) => content); + throw failure; + } + ), + (error) => error === failure + ); +}); + +test("evidence corpus transactions reject root, ancestor, and enumerated-directory swaps", async (t) => { + for (const kind of ["root", "ancestor", "directory"]) { + await t.test(kind, async () => { + const parent = tmpDir(); + const root = path.join(parent, "authorized"); + const ancestor = path.join(root, "nested"); + const corpus = path.join(ancestor, "corpus"); + fs.mkdirSync(corpus, { recursive: true }); + fs.writeFileSync(path.join(corpus, "note.md"), "# Original\n"); + const target = { root, ancestor, directory: corpus }[kind]; + const parked = `${target}-parked`; + const reader = createEvidenceReader({ roots: [root] }); + + await assert.rejects( + () => reader.withTextCorpus( + root, + corpus, + { extensions: [".md"] }, + async (transaction) => { + await transaction.mapFiles(({ content }) => content); + fs.renameSync(target, parked); + fs.mkdirSync(target, { recursive: true }); + return `${kind} result must not escape`; + } + ), + (error) => error?.code === "DOTAIOS_EVIDENCE_CHANGED" + ); + }); + } +}); + +test("evidence corpus transactions reject a directory replacement at the enumeration-open barrier", async () => { + const root = tmpDir(); + const corpus = path.join(root, "corpus"); + const parked = path.join(root, "corpus-parked"); + fs.mkdirSync(corpus); + fs.writeFileSync(path.join(corpus, "note.md"), "# Original\n"); + let swapped = false; + const filesystem = Object.create(fsp); + filesystem.opendir = async (targetPath, options) => { + if (!swapped && path.resolve(String(targetPath)) === path.resolve(corpus)) { + swapped = true; + await fsp.rename(corpus, parked); + await fsp.mkdir(corpus); + await fsp.writeFile(path.join(corpus, "note.md"), "# Replacement\n"); + } + return fsp.opendir(targetPath, options); + }; + const reader = createEvidenceReader({ roots: [root], filesystem }); + + await assert.rejects( + () => reader.withTextCorpus( + root, + corpus, + { extensions: [".md"] }, + (transaction) => transaction.mapFiles(({ content }) => content) + ), + (error) => swapped && error?.code === "DOTAIOS_EVIDENCE_CHANGED" + ); +}); + +test("evidence corpus transactions bind every opened file to its enumerated parent identity", async () => { + const root = tmpDir(); + const corpus = path.join(root, "corpus"); + const parked = path.join(root, "corpus-parked"); + const source = path.join(corpus, "note.md"); + fs.mkdirSync(corpus); + fs.writeFileSync(source, "# Original\n\nORIGINAL_CANARY\n"); + let swapped = false; + const filesystem = Object.create(fsp); + filesystem.open = async (targetPath, flags) => { + if (!swapped && path.resolve(String(targetPath)) === path.resolve(source)) { + swapped = true; + await fsp.rename(corpus, parked); + await fsp.mkdir(corpus); + await fsp.writeFile(source, "# Replacement\n\nREPLACEMENT_CANARY\n"); + } + return fsp.open(targetPath, flags); + }; + const reader = createEvidenceReader({ roots: [root], filesystem }); + + await assert.rejects( + () => reader.withTextCorpus( + root, + corpus, + { extensions: [".md"] }, + (transaction) => transaction.mapFiles(({ content }) => content) + ), + (error) => swapped && error?.code === "DOTAIOS_EVIDENCE_CHANGED" + ); +}); + +test("evidence corpus transactions reject a final-component file swap before bytes are read", async () => { + const root = tmpDir(); + const source = path.join(root, "note.md"); + const parked = path.join(root, "note-parked.md"); + fs.writeFileSync(source, "# Original\n\nORIGINAL_CANARY\n"); + let swapped = false; + const filesystem = Object.create(fsp); + filesystem.open = async (targetPath, flags) => { + if (!swapped && path.resolve(String(targetPath)) === path.resolve(source)) { + swapped = true; + await fsp.rename(source, parked); + await fsp.writeFile(source, "# Replacement\n\nREPLACEMENT_CANARY\n"); + } + return fsp.open(targetPath, flags); + }; + const reader = createEvidenceReader({ roots: [root], filesystem }); + + await assert.rejects( + () => reader.withTextCorpus( + root, + root, + { extensions: [".md"] }, + (transaction) => transaction.mapFiles(({ content }) => content) + ), + (error) => swapped && error?.code === "DOTAIOS_EVIDENCE_CHANGED" + ); +}); + +test("evidence corpus transactions reject in-place file mutation during a handle read", async () => { + const root = tmpDir(); + const source = path.join(root, "note.md"); + fs.writeFileSync(source, "# Original\n\nORIGINAL_CANARY\n"); + let mutated = false; + const filesystem = Object.create(fsp); + filesystem.open = async (targetPath, ...args) => { + const handle = await fsp.open(targetPath, ...args); + if (path.resolve(String(targetPath)) !== path.resolve(source)) return handle; + return new Proxy(handle, { + get(target, property) { + if (property === "read") { + return async (...readArgs) => { + const result = await target.read(...readArgs); + if (!mutated && result.bytesRead > 0) { + mutated = true; + await fsp.writeFile(source, "# Mutated!\n\nMUTATION_CANARY\n"); + } + return result; + }; + } + const value = Reflect.get(target, property, target); + return typeof value === "function" ? value.bind(target) : value; + } + }); + }; + const reader = createEvidenceReader({ roots: [root], filesystem }); + + await assert.rejects( + () => reader.withTextCorpus( + root, + root, + { extensions: [".md"] }, + (transaction) => transaction.mapFiles(({ content }) => content) + ), + (error) => mutated && error?.code === "DOTAIOS_EVIDENCE_CHANGED" + ); +}); + +test("evidence corpus transactions catch a synchronized swap and restore spanning final validation", async () => { + const root = tmpDir(); + const corpus = path.join(root, "corpus"); + const parked = path.join(root, "corpus-parked"); + fs.mkdirSync(corpus); + fs.writeFileSync(path.join(corpus, "note.md"), "# Stable bytes\n"); + let armed = false; + let intercepted = false; + const filesystem = Object.create(fsp); + filesystem.lstat = async (targetPath, options) => { + if ( + armed + && !intercepted + && path.resolve(String(targetPath)) === path.resolve(corpus) + && options?.bigint === true + ) { + intercepted = true; + await fsp.rename(corpus, parked); + await fsp.mkdir(corpus); + const replacement = await fsp.lstat(corpus, options); + await fsp.rmdir(corpus); + await fsp.rename(parked, corpus); + return replacement; + } + return fsp.lstat(targetPath, options); + }; + const reader = createEvidenceReader({ roots: [root], filesystem }); + + await assert.rejects( + () => reader.withTextCorpus( + root, + corpus, + { extensions: [".md"] }, + async (transaction) => { + await transaction.mapFiles(({ content }) => content); + armed = true; + return "must not escape"; + } + ), + (error) => intercepted && error?.code === "DOTAIOS_EVIDENCE_CHANGED" + ); + // Portable Node can detect swaps that span an observation barrier. As in the + // repository threat model, this does not claim immunity to an entirely + // unobserved same-user ABA completed between barriers. + assert.equal(fs.readFileSync(path.join(corpus, "note.md"), "utf8"), "# Stable bytes\n"); +}); + +test("evidence corpus transactions reject eligible symlinks and non-regular files", { + skip: process.platform === "win32" ? "mkfifo is unavailable on Windows" : false +}, async (t) => { + await t.test("symbolic link", async () => { + const root = tmpDir(); + const outside = path.join(root, "outside.txt"); + fs.writeFileSync(outside, "OUTSIDE_CANARY\n"); + fs.symlinkSync(outside, path.join(root, "linked.md")); + const reader = createEvidenceReader({ roots: [root] }); + await assert.rejects( + () => reader.withTextCorpus( + root, + root, + { extensions: [".md"] }, + (transaction) => transaction.mapFiles(({ content }) => content) + ), + (error) => error?.code === "DOTAIOS_EVIDENCE_PATH_UNSAFE" + ); + }); + + await t.test("fifo", async () => { + const root = tmpDir(); + const fifoPath = path.join(root, "blocked.md"); + const { spawnSync } = await import("node:child_process"); + assert.equal(spawnSync("mkfifo", [fifoPath]).status, 0); + const reader = createEvidenceReader({ roots: [root] }); + await assert.rejects( + () => reader.withTextCorpus( + root, + root, + { extensions: [".md"] }, + (transaction) => transaction.mapFiles(({ content }) => content) + ), + (error) => error?.code === "DOTAIOS_EVIDENCE_NOT_REGULAR_FILE" + ); + }); +}); + +test("evidence corpus transactions reject invalid UTF-8 without changing source bytes", async () => { + const root = tmpDir(); + const source = path.join(root, "invalid.md"); + const bytes = Buffer.from([0x23, 0x20, 0x58, 0x0a, 0xff, 0x0a]); + fs.writeFileSync(source, bytes); + const reader = createEvidenceReader({ roots: [root] }); + + await assert.rejects( + () => reader.withTextCorpus( + root, + root, + { extensions: [".md"] }, + (transaction) => transaction.mapFiles(({ content }) => content) + ), + (error) => error?.code === "DOTAIOS_EVIDENCE_INVALID_UTF8" + ); + assert.deepEqual(fs.readFileSync(source), bytes); +}); + +test("evidence corpus transactions retain every configured collection and byte ceiling", async (t) => { + const cases = [ + { + name: "file count", + limits: { maxFiles: 1, maxBytes: 100, maxEntries: 10, maxFileBytes: 100, maxDirectoryEntries: 10 }, + files: [["one.md", "one\n"], ["two.md", "two\n"]], + code: "DOTAIOS_EVIDENCE_BUDGET_EXCEEDED" + }, + { + name: "aggregate bytes", + limits: { maxFiles: 2, maxBytes: 5, maxEntries: 10, maxFileBytes: 100, maxDirectoryEntries: 10 }, + files: [["one.md", "one\n"], ["two.md", "two\n"]], + code: "DOTAIOS_EVIDENCE_BUDGET_EXCEEDED" + }, + { + name: "per-file bytes", + limits: { maxFiles: 1, maxBytes: 100, maxEntries: 10, maxFileBytes: 3, maxDirectoryEntries: 10 }, + files: [["one.md", "one\n"]], + code: "DOTAIOS_EVIDENCE_FILE_TOO_LARGE" + }, + { + name: "aggregate entries", + limits: { maxFiles: 2, maxBytes: 100, maxEntries: 1, maxFileBytes: 100, maxDirectoryEntries: 10 }, + files: [["one.md", "one\n"], ["two.md", "two\n"]], + code: "DOTAIOS_EVIDENCE_BUDGET_EXCEEDED" + }, + { + name: "directory entries", + limits: { maxFiles: 2, maxBytes: 100, maxEntries: 10, maxFileBytes: 100, maxDirectoryEntries: 1 }, + files: [["one.md", "one\n"], ["two.md", "two\n"]], + code: "DOTAIOS_EVIDENCE_DIRECTORY_TOO_LARGE" + } + ]; + + for (const fixture of cases) { + await t.test(fixture.name, async () => { + const root = tmpDir(); + for (const [name, content] of fixture.files) fs.writeFileSync(path.join(root, name), content); + const reader = createEvidenceReader({ roots: [root], limits: fixture.limits }); + await assert.rejects( + () => reader.withTextCorpus( + root, + root, + { extensions: [".md"] }, + (transaction) => transaction.mapFiles(({ content }) => content) + ), + (error) => error?.code === fixture.code + ); + }); + } +}); + +test("scope preflight protects later scopes deterministically before redistributing capacity", async () => { + const root = tmpDir(); + const counts = { large: 5, small: 1, later: 1 }; + for (const [scope, count] of Object.entries(counts)) { + const directory = path.join(root, scope); + fs.mkdirSync(directory); + for (let index = 0; index < count; index += 1) { + fs.writeFileSync(path.join(directory, `${index}.md`), `${scope}-${index}\n`); + } + } + const reader = createEvidenceReader({ + roots: [root], + limits: { maxBytes: 1024, maxFiles: 6, maxEntries: 100, maxFileBytes: 100, maxDirectoryEntries: 100 } + }); + + const result = await reader.withScopePreflight( + ["large", "small", "later"], + async (scope, scopeReader) => { + const prepared = await scopeReader.prepareTextCorpus( + root, + path.join(root, scope), + { extensions: [".md"] } + ); + if (scope === "large") await new Promise((resolve) => setTimeout(resolve, 10)); + return prepared; + }, + (_scope, _scopeReader, prepared) => prepared, + (_scope, scopeReader, prepared) => scopeReader.withPreparedTextCorpus( + prepared, + (transaction) => transaction.mapFiles(({ content }) => content) + ), + (transaction) => ({ + admitted: ["large", "small", "later"].filter((scope) => transaction.has(scope)), + omissions: transaction.omissions + }) + ); + + assert.deepEqual(result.admitted, ["small", "later"]); + assert.equal(result.omissions[0].scope, "large"); + assert.equal(result.omissions[0].reason, "file_count_exceeded"); +}); + +test("scope preflight admits and revalidates a corpus through a symlinked authorized root", async (t) => { + if (process.platform === "win32") return t.skip("symlink creation requires elevated Windows privileges"); + const parent = tmpDir(); + const realRoot = path.join(parent, "real-authorized"); + const linkedRoot = path.join(parent, "linked-authorized"); + const corpus = path.join(linkedRoot, "context"); + fs.mkdirSync(path.join(realRoot, "context"), { recursive: true }); + fs.writeFileSync(path.join(realRoot, "context", "note.md"), "SYMLINKED_ROOT_PREFLIGHT_CANARY\n"); + fs.symlinkSync(realRoot, linkedRoot, "dir"); + const reader = createEvidenceReader({ roots: [linkedRoot] }); + + const result = await reader.withScopePreflight( + ["context"], + (_scope, scopeReader) => scopeReader.prepareTextCorpus( + linkedRoot, + corpus, + { extensions: [".md"] } + ), + (_scope, _scopeReader, prepared) => prepared, + (_scope, scopeReader, prepared) => scopeReader.withPreparedTextCorpus( + prepared, + (transaction) => transaction.mapFiles(({ content }) => content) + ), + (transaction) => transaction.get("context") + ); + + assert.deepEqual(result, ["SYMLINKED_ROOT_PREFLIGHT_CANARY\n"]); +}); + +test("scope preflight protects later scopes before metadata and JSONL discovery spend entries", async (t) => { + await t.test("metadata inspection", async () => { + const root = tmpDir(); + for (const [scope, count] of [["large", 4], ["later", 1]]) { + const directory = path.join(root, scope); + fs.mkdirSync(directory); + for (let index = 0; index < count; index += 1) { + fs.writeFileSync(path.join(directory, `${index}.md`), `${scope}-${index}\n`); + } + } + const reader = createEvidenceReader({ + roots: [root], + limits: { maxBytes: 1024, maxFiles: 20, maxEntries: 4, maxFileBytes: 100, maxDirectoryEntries: 100 } + }); + + const admitted = await reader.withScopePreflight( + ["large", "later"], + (scope, scopeReader) => scopeReader.prepareTextCorpus( + root, + path.join(root, scope), + { extensions: [".md"] } + ), + (_scope, _scopeReader, prepared) => prepared, + (_scope, scopeReader, prepared) => scopeReader.withPreparedTextCorpus( + prepared, + (transaction) => transaction.mapFiles(({ content }) => content) + ), + (transaction) => ["large", "later"].filter((scope) => transaction.has(scope)) + ); + + assert.deepEqual(admitted, ["later"]); + }); + + await t.test("JSONL discovery", async () => { + const root = tmpDir(); + fs.writeFileSync( + path.join(root, "large.jsonl"), + Array.from({ length: 4 }, (_, index) => JSON.stringify({ index })).join("\n") + "\n" + ); + fs.writeFileSync(path.join(root, "later.jsonl"), '{"index":0}\n'); + const reader = createEvidenceReader({ + roots: [root], + limits: { maxBytes: 1024, maxFiles: 20, maxEntries: 4, maxFileBytes: 100, maxDirectoryEntries: 100 } + }); + + const admitted = await reader.withScopePreflight( + ["large", "later"], + (scope, scopeReader) => scopeReader.prepareJsonlMetadata( + root, + path.join(root, `${scope}.jsonl`) + ), + (_scope, scopeReader, prepared) => scopeReader.materializePreparedJsonl(prepared), + (_scope, scopeReader, prepared) => scopeReader.readPreparedJsonl(prepared), + (transaction) => ["large", "later"].filter((scope) => transaction.has(scope)) + ); + + assert.deepEqual(admitted, ["later"]); + }); +}); + +test("scope preflight revalidates a partially enumerated omitted directory before publishing results", async () => { + const root = tmpDir(); + const oversized = path.join(root, "oversized"); + const safe = path.join(root, "safe"); + fs.mkdirSync(oversized); + fs.mkdirSync(safe); + fs.writeFileSync(path.join(oversized, "one.md"), "one\n"); + fs.writeFileSync(path.join(oversized, "two.md"), "two\n"); + fs.writeFileSync(path.join(safe, "safe.md"), "safe\n"); + const reader = createEvidenceReader({ + roots: [root], + limits: { maxBytes: 100, maxFiles: 10, maxEntries: 10, maxFileBytes: 100, maxDirectoryEntries: 1 } + }); + + await assert.rejects( + () => reader.withScopePreflight( + ["oversized", "safe"], + (scope, scopeReader) => scopeReader.prepareTextCorpus( + root, + path.join(root, scope), + { extensions: [".md"] } + ), + (_scope, _scopeReader, prepared) => prepared, + (_scope, scopeReader, prepared) => scopeReader.withPreparedTextCorpus( + prepared, + (transaction) => transaction.mapFiles(({ content }) => content) + ), + (transaction) => { + assert.equal(transaction.has("safe"), true); + assert.equal(transaction.has("oversized"), false); + fs.writeFileSync(path.join(oversized, "late.md"), "late\n"); + return "must not escape"; + } + ), + (error) => error?.code === "DOTAIOS_EVIDENCE_CHANGED" + ); +}); + +test("scope preflight final-revalidates every inspected file before publishing", async (t) => { + await t.test("admitted corpus file", async () => { + const root = tmpDir(); + const corpus = path.join(root, "corpus"); + const filePath = path.join(corpus, "note.md"); + fs.mkdirSync(corpus); + fs.writeFileSync(filePath, "ORIGINAL\n"); + const reader = createEvidenceReader({ roots: [root] }); + + await assert.rejects( + () => reader.withScopePreflight( + ["corpus"], + (_scope, scopeReader) => scopeReader.prepareTextCorpus( + root, + corpus, + { extensions: [".md"] } + ), + (_scope, _scopeReader, prepared) => prepared, + (_scope, scopeReader, prepared) => scopeReader.withPreparedTextCorpus( + prepared, + (transaction) => transaction.mapFiles(({ content }) => content) + ), + (transaction) => { + assert.deepEqual(transaction.get("corpus"), ["ORIGINAL\n"]); + fs.writeFileSync(filePath, "CHANGED!\n"); + return "must not escape"; + } + ), + (error) => error?.code === "DOTAIOS_EVIDENCE_CHANGED" + ); + }); + + await t.test("omission evidence", async () => { + const root = tmpDir(); + const filePath = path.join(root, "oversized.jsonl"); + fs.writeFileSync(filePath, "xx"); + const reader = createEvidenceReader({ + roots: [root], + limits: { maxBytes: 100, maxFiles: 10, maxEntries: 10, maxFileBytes: 1, maxDirectoryEntries: 10 } + }); + + await assert.rejects( + () => reader.withScopePreflight( + ["oversized"], + (_scope, scopeReader) => scopeReader.prepareJsonlMetadata(root, filePath), + (_scope, _scopeReader, prepared) => prepared, + () => "must not execute", + (transaction) => { + assert.equal(transaction.has("oversized"), false); + fs.writeFileSync(filePath, "yy"); + return "must not escape"; + } + ), + (error) => error?.code === "DOTAIOS_EVIDENCE_CHANGED" + ); + }); +}); + +test("scope preflight closes retained phase readers and prepared capabilities on every exit", async (t) => { + async function exercise({ failureStage = null }) { + const root = tmpDir(); + fs.writeFileSync(path.join(root, "note.md"), "retained\n"); + const reader = createEvidenceReader({ roots: [root] }); + let retainedReader; + let retainedPrepared; + const operation = reader.withScopePreflight( + ["scope"], + async (_scope, scopeReader) => { + retainedReader = scopeReader; + retainedPrepared = await scopeReader.prepareTextCorpus( + root, + root, + { extensions: [".md"] } + ); + return retainedPrepared; + }, + (_scope, _scopeReader, prepared) => { + if (failureStage === "discovery") throw new Error("discovery failed"); + return prepared; + }, + (_scope, scopeReader, prepared) => { + if (failureStage === "execution") throw new Error("execution failed"); + return scopeReader.withPreparedTextCorpus( + prepared, + (transaction) => transaction.mapFiles(({ content }) => content) + ); + }, + (transaction) => { + if (failureStage === "callback") throw new Error("callback failed"); + return transaction.get("scope"); + } + ); + + if (failureStage) await assert.rejects(() => operation); + else assert.deepEqual(await operation, ["retained\n"]); + assert.throws( + () => retainedReader.withPreparedTextCorpus( + retainedPrepared, + (transaction) => transaction.mapFiles(({ content }) => content) + ), + (error) => error?.code === "DOTAIOS_EVIDENCE_TRANSACTION_CLOSED" + ); + } + + await t.test("success", () => exercise({})); + await t.test("discovery failure", () => exercise({ failureStage: "discovery" })); + await t.test("execution failure", () => exercise({ failureStage: "execution" })); + await t.test("callback failure", () => exercise({ failureStage: "callback" })); +}); + +test("scope preflight caps omissions exactly once at 32 plus the full aggregate remainder", async () => { + const root = tmpDir(); + const scopes = Array.from({ length: 40 }, (_, index) => `scope-${String(index).padStart(2, "0")}`); + for (const scope of scopes) { + const directory = path.join(root, scope); + fs.mkdirSync(directory); + fs.writeFileSync(path.join(directory, "oversized.md"), "xx"); + } + const reader = createEvidenceReader({ + roots: [root], + limits: { maxBytes: 100, maxFiles: 100, maxEntries: 100, maxFileBytes: 1, maxDirectoryEntries: 10 } + }); + + const omissions = await reader.withScopePreflight( + scopes, + (scope, scopeReader) => scopeReader.prepareTextCorpus( + root, + path.join(root, scope), + { extensions: [".md"] } + ), + (_scope, _scopeReader, prepared) => prepared, + (_scope, scopeReader, prepared) => scopeReader.withPreparedTextCorpus( + prepared, + (transaction) => transaction.mapFiles(({ content }) => content) + ), + (transaction) => transaction.omissions + ); + + assert.equal(omissions.length, 33); + assert.equal(omissions[31].scope, "scope-31"); + assert.deepEqual(omissions[32], { + scope: "all", + reason: "omissions_truncated", + observed: { files: 0, bytes: 0, entries: 8 }, + inspection: "not_searched", + recovery: { + code: "narrow_scope", + message: "Search one logical scope at a time to inspect every omission." + } + }); + assert.equal(Object.isFrozen(omissions), true); + assert.equal(Object.isFrozen(omissions[32]), true); +}); + +test("scope preflight converts only skippable execution ceilings into whole-scope omissions", async (t) => { + await t.test("preserves successful scopes and reports a bounded omission", async () => { + const root = tmpDir(); + const reader = createEvidenceReader({ roots: [root] }); + + const result = await reader.withScopePreflight( + ["oversized", "safe"], + () => null, + (_scope, _scopeReader, prepared) => prepared, + (scope) => { + if (scope === "oversized") { + throw new EvidenceReadError("DOTAIOS_EVIDENCE_FILE_COUNT_EXCEEDED"); + } + return "safe-result"; + }, + (transaction) => ({ + safe: transaction.get("safe"), + oversized: transaction.has("oversized"), + omissions: transaction.omissions + }) + ); + + assert.equal(result.safe, "safe-result"); + assert.equal(result.oversized, false); + assert.equal(result.omissions.length, 1); + assert.equal(result.omissions[0].scope, "oversized"); + assert.equal(result.omissions[0].reason, "file_count_exceeded"); + assert.equal(Object.isFrozen(result.omissions), true); + }); + + await t.test("keeps integrity failures request-fatal", async () => { + const root = tmpDir(); + const reader = createEvidenceReader({ roots: [root] }); + + await assert.rejects( + () => reader.withScopePreflight( + ["changed", "safe"], + () => null, + (_scope, _scopeReader, prepared) => prepared, + (scope) => { + if (scope === "changed") throw new EvidenceReadError("DOTAIOS_EVIDENCE_CHANGED"); + return "must not escape"; + }, + () => "must not publish" + ), + (error) => error?.code === "DOTAIOS_EVIDENCE_CHANGED" + ); + }); + + await t.test("retains final validation for successful scopes", async () => { + const root = tmpDir(); + const safeFile = path.join(root, "safe.md"); + fs.writeFileSync(safeFile, "safe\n"); + const reader = createEvidenceReader({ roots: [root] }); + + await assert.rejects( + () => reader.withScopePreflight( + ["oversized", "safe"], + (scope, scopeReader) => scope === "safe" + ? scopeReader.prepareTextCorpus(root, root, { extensions: [".md"] }) + : null, + (_scope, _scopeReader, prepared) => prepared, + (scope, scopeReader, prepared) => { + if (scope === "oversized") { + throw new EvidenceReadError("DOTAIOS_EVIDENCE_FILE_COUNT_EXCEEDED"); + } + return scopeReader.withPreparedTextCorpus( + prepared, + (transaction) => transaction.mapFiles(({ content }) => content) + ); + }, + (transaction) => { + assert.deepEqual(transaction.get("safe"), ["safe\n"]); + assert.equal(transaction.has("oversized"), false); + fs.writeFileSync(safeFile, "late\n"); + return "must not escape"; + } + ), + (error) => error?.code === "DOTAIOS_EVIDENCE_CHANGED" + ); + }); +}); + +test("scope preflight fails closed when one scope observes conflicting generations of the same directory", async () => { + const root = tmpDir(); + fs.writeFileSync(path.join(root, "events.jsonl"), '{"summary":"safe"}\n'); + const reader = createEvidenceReader({ roots: [root] }); + + await assert.rejects( + () => reader.withScopePreflight( + ["memory"], + (_scope, scopeReader) => scopeReader.prepareJsonlMetadata( + root, + path.join(root, "events.jsonl") + ), + async (_scope, scopeReader, prepared) => { + await scopeReader.materializePreparedJsonl(prepared); + fs.writeFileSync(path.join(root, "late.md"), "late\n"); + return scopeReader.materializePreparedJsonl(prepared); + }, + () => "must not execute", + () => "must not escape" + ), + (error) => error?.code === "DOTAIOS_EVIDENCE_CHANGED" + ); +}); + +test("evidence corpus file-operation growth stays constant per accepted file at fixed topology", async () => { + async function measure(fileCount) { + const root = tmpDir(); + for (let index = 0; index < fileCount; index += 1) { + fs.writeFileSync(path.join(root, `${index}.md`), `# ${index}\n`); + } + const operations = { lstat: 0, realpath: 0, open: 0 }; + const filesystem = new Proxy(fsp, { + get(target, property) { + if (Object.hasOwn(operations, property)) { + return async (...args) => { + operations[property] += 1; + return fsp[property](...args); + }; + } + const value = Reflect.get(target, property, target); + return typeof value === "function" ? value.bind(target) : value; + } + }); + const reader = createEvidenceReader({ roots: [root], filesystem }); + const observed = await reader.withTextCorpus( + root, + root, + { extensions: [".md"] }, + (transaction) => transaction.mapFiles(({ content }) => content) + ); + assert.equal(observed.length, fileCount); + assert.equal(operations.open, fileCount); + return operations; + } + + const one = await measure(1); + const eight = await measure(8); + assert.ok(eight.lstat - one.lstat <= 4 * 7, JSON.stringify({ one, eight })); + assert.ok(eight.realpath - one.realpath <= 2 * 7, JSON.stringify({ one, eight })); + assert.equal(eight.open - one.open, 7); +}); diff --git a/tests/core/memory-safety.test.mjs b/tests/core/memory-safety.test.mjs index 3f018798..b0ce0625 100644 --- a/tests/core/memory-safety.test.mjs +++ b/tests/core/memory-safety.test.mjs @@ -1,7 +1,9 @@ import fs from "node:fs"; import fsp from "node:fs/promises"; +import { spawnSync } from "node:child_process"; import os from "node:os"; import path from "node:path"; +import { performance } from "node:perf_hooks"; import test from "node:test"; import assert from "node:assert/strict"; import { @@ -14,6 +16,8 @@ import { } from "../../packages/core/src/memory.mjs"; import { searchMemoryDir } from "../../packages/core/src/search.mjs"; +const EVENT_COMPACTION_PENDING_MAGIC = "#!dotaios-event-compaction/v1"; + function tmpDir() { return fs.mkdtempSync(path.join(os.tmpdir(), "dotaios-memsafe-test-")); } @@ -32,29 +36,77 @@ function readLines(filePath) { return fs.readFileSync(filePath, "utf8").split("\n").filter((l) => l.trim()); } +function legacyRotationFixture() { + const dir = tmpDir(); + const eventsPath = path.join(dir, "events.jsonl"); + const archivePath = path.join(dir, "events-archive.jsonl"); + const legacy = Array.from({ length: 3 }, (_, index) => JSON.stringify({ + ts: `2026-01-01T00:00:0${index}.000Z`, + type: "legacy-rotation", + id: `legacy-${index}`, + summary: "x".repeat(720_000) + })); + const pending = [ + JSON.stringify({ ts: "2026-01-01T00:00:03.000Z", type: "legacy-rotation", id: "pending-0" }), + JSON.stringify({ ts: "2026-01-01T00:00:04.000Z", type: "legacy-rotation", id: "pending-1" }) + ]; + const live = JSON.stringify({ ts: "2026-01-01T00:00:05.000Z", type: "legacy-rotation", id: "live" }); + fs.writeFileSync(archivePath, `${legacy.join("\n")}\n`, { mode: 0o600 }); + fs.writeFileSync(eventsPath, `${[...pending, live].join("\n")}\n`, { mode: 0o600 }); + return { dir, eventsPath, archivePath, expectedArchive: [...legacy, ...pending], live }; +} + +function failAfterArchiveBoundary(method, matches) { + let interrupted = false; + return { + filesystem: { + ...fsp, + async [method](...args) { + const result = await fsp[method](...args); + if (!interrupted && matches(...args)) { + interrupted = true; + throw new Error(`injected-crash:${method}`); + } + return result; + } + }, + interrupted: () => interrupted + }; +} + +function readEventsArchiveLines(dir) { + return fs.readdirSync(dir) + .filter((name) => /^events-archive(?:\.\d{6})?\.jsonl$/.test(name)) + .sort((left, right) => { + if (left === "events-archive.jsonl") return 1; + if (right === "events-archive.jsonl") return -1; + return left.localeCompare(right); + }) + .flatMap((name) => readLines(path.join(dir, name))); +} + // Injectable fs double: throws once at the Nth call of a method, otherwise // delegates to the real promises fs. Simulates a crash at an exact step. function faultFs(failures) { const counts = {}; + const fired = new Set(); const wrap = (name) => async (...args) => { counts[name] = (counts[name] || 0) + 1; counts.total = (counts.total || 0) + 1; if (failures[name] && counts[name] === failures[name]) { + fired.add(name); throw new Error(`injected-crash:${name}:${counts[name]}`); } return fsp[name](...args); }; - const filesystem = { - readFile: wrap("readFile"), - writeFile: wrap("writeFile"), - appendFile: wrap("appendFile"), - rename: wrap("rename"), - unlink: wrap("unlink"), - readdir: wrap("readdir"), - stat: wrap("stat"), - mkdir: wrap("mkdir") - }; - return { filesystem, counts }; + const filesystem = { ...fsp }; + for (const name of [ + "appendFile", "chmod", "link", "lstat", "mkdir", "open", "readFile", + "readdir", "rename", "rm", "stat", "unlink", "writeFile" + ]) { + filesystem[name] = wrap(name); + } + return { filesystem, counts, fired }; } // --- Defect 1: crash-safe compaction --- @@ -63,8 +115,8 @@ test("compaction survives a crash at any single step: no event lost, none duplic const scenarios = [ { writeFile: 1 }, { writeFile: 2 }, - { appendFile: 1 }, { rename: 1 }, + { rename: 2 }, { unlink: 1 } ]; for (const failure of scenarios) { @@ -81,6 +133,12 @@ test("compaction survives a crash at any single step: no event lost, none duplic assert.match(String(error.message), /injected-crash/, `unexpected error for ${JSON.stringify(failure)}: ${error.message}`); } assert.ok((fault.counts.total || 0) > 0, "compactEvents must honor the injectable filesystem so crashes are testable"); + for (const method of Object.keys(failure)) { + assert.ok( + fault.fired.has(method), + `configured fault never fired for ${JSON.stringify(failure)}` + ); + } // Mid-crash, every event must survive in SOME durable file: the events // log, the archive, or the staged pending batch. @@ -108,6 +166,806 @@ test("compaction survives a crash at any single step: no event lost, none duplic } }); +test("compaction syncs the parent directory before flushing the pending archive", async () => { + const dir = tmpDir(); + const eventsPath = path.join(dir, "events.jsonl"); + const pendingPath = path.join(dir, "events-archive.jsonl.pending"); + seedEvents(eventsPath, 3); + const calls = []; + const filesystem = { + ...fsp, + async rename(source, destination) { + await fsp.rename(source, destination); + if (destination === eventsPath) calls.push("live-rename"); + }, + async open(filePath, flags, ...rest) { + const handle = await fsp.open(filePath, flags, ...rest); + if (filePath === pendingPath && typeof flags === "number") calls.push("pending-read"); + if (filePath !== dir || flags !== "r") return handle; + return new Proxy(handle, { + get(target, name) { + if (name === "sync") { + return async (...args) => { + calls.push("directory-sync"); + return target.sync(...args); + }; + } + const value = Reflect.get(target, name, target); + return typeof value === "function" ? value.bind(target) : value; + } + }); + } + }; + + await compactEvents(eventsPath, 1, { filesystem }); + + const renameIndex = calls.indexOf("live-rename"); + const syncIndex = calls.indexOf("directory-sync", renameIndex + 1); + const pendingReadIndex = calls.indexOf("pending-read"); + assert.ok(renameIndex >= 0, `missing live rename in ${JSON.stringify(calls)}`); + assert.ok(syncIndex > renameIndex, `directory sync must follow live rename: ${JSON.stringify(calls)}`); + assert.ok(pendingReadIndex > syncIndex, `pending flush must follow the durability barrier: ${JSON.stringify(calls)}`); +}); + +test("rotating compaction recovers every shard publication boundary without loss or duplication", async () => { + const scenarios = [ + { link: 2 }, + { rename: 2 }, + { rename: 3 }, + { unlink: 2 }, + { unlink: 3 } + ]; + for (const failure of scenarios) { + const dir = tmpDir(); + const eventsPath = path.join(dir, "events.jsonl"); + const archivePath = path.join(dir, "events-archive.jsonl"); + const original = Array.from({ length: 4 }, (_, index) => JSON.stringify({ + ts: `2026-01-01T00:00:0${index}.000Z`, + type: "rotation-crash", + id: `event-${index}`, + summary: index < 3 ? "x".repeat(720_000) : "kept" + })); + fs.writeFileSync(eventsPath, `${original.join("\n")}\n`, { mode: 0o600 }); + + const fault = faultFs(failure); + await assert.rejects( + compactEvents(eventsPath, 1, { filesystem: fault.filesystem }), + /injected-crash/ + ); + + const durablePaths = fs.readdirSync(dir) + .filter((name) => name === "events.jsonl" + || name === "events-archive.jsonl" + || /^events-archive\.\d{6}\.jsonl$/.test(name) + || name === "events-archive.jsonl.pending") + .map((name) => path.join(dir, name)); + const midState = durablePaths.flatMap(readLines); + for (const line of original) { + assert.ok(midState.includes(line), `event lost after shard crash ${JSON.stringify(failure)}`); + } + + await compactEvents(eventsPath, 1); + const settledPaths = fs.readdirSync(dir) + .filter((name) => name === "events.jsonl" + || name === "events-archive.jsonl" + || /^events-archive\.\d{6}\.jsonl$/.test(name)) + .sort() + .map((name) => path.join(dir, name)); + const settled = settledPaths.flatMap(readLines); + assert.deepEqual([...settled].sort(), [...original].sort()); + } +}); + +test("compaction recovery preserves byte-identical events as separate records", async () => { + const dir = tmpDir(); + const eventsPath = path.join(dir, "events.jsonl"); + const event = JSON.stringify({ + ts: "2026-01-01T00:00:00.000Z", + type: "intentional-repeat", + summary: "same bytes, three separate saves" + }); + fs.writeFileSync(eventsPath, `${event}\n${event}\n${event}\n`, { mode: 0o600 }); + + const fault = faultFs({ link: 2 }); + await assert.rejects( + compactEvents(eventsPath, 1, { filesystem: fault.filesystem }), + /injected-crash:link:2/ + ); + + await compactEvents(eventsPath, 1); + + const settled = [ + ...readEventsArchiveLines(dir), + ...readLines(eventsPath) + ]; + assert.equal(settled.length, 3, "three intentional saves must survive recovery as three records"); + assert.deepEqual(settled, [event, event, event]); +}); + +test("compaction recovery rejects a truncated transaction batch", async () => { + const dir = tmpDir(); + const eventsPath = path.join(dir, "events.jsonl"); + const archivePath = path.join(dir, "events-archive.jsonl"); + seedEvents(eventsPath, 3); + + const fault = faultFs({ link: 2 }); + await assert.rejects(compactEvents(eventsPath, 1, { filesystem: fault.filesystem })); + const staged = readLines(`${archivePath}.pending`); + fs.writeFileSync(`${archivePath}.pending`, `${staged.slice(0, 2).join("\n")}\n`, { mode: 0o600 }); + + await assert.rejects( + compactEvents(eventsPath, 1), + (error) => error?.code === "DOTAIOS_ARCHIVE_STATE_INVALID" + ); + assert.equal(fs.existsSync(`${archivePath}.pending`), true); +}); + +test("marked pending artifacts with corrupt, missing, or unknown contracts fail closed", async (t) => { + const cases = [ + { + name: "corrupt envelope", + header: '{"artifact_contract":"dotaios-event-compaction/v999","phase":"ready"' + }, + { + name: "missing contract", + header: JSON.stringify({ + phase: "ready", + beforeHash: "0".repeat(64), + afterHash: "1".repeat(64), + pendingHash: "2".repeat(64), + pendingRecords: 1, + archiveRecordsBefore: 1, + archiveChainBefore: "3".repeat(64), + archiveChainAfter: "4".repeat(64) + }) + }, + { + name: "unknown contract", + header: JSON.stringify({ + artifact_contract: "dotaios-event-compaction/v999", + phase: "ready" + }) + }, + { + name: "unknown marker version", + marker: "#!dotaios-event-compaction/v999", + header: JSON.stringify({ artifact_contract: "dotaios-event-compaction/v1" }) + }, + { + name: "one-byte marker corruption", + marker: "#!dotaios-event-compactioo/v1", + header: JSON.stringify({ artifact_contract: "dotaios-event-compaction/v1" }) + } + ]; + + for (const fixture of cases) { + await t.test(fixture.name, async () => { + const dir = tmpDir(); + const eventsPath = path.join(dir, "events.jsonl"); + const archivePath = path.join(dir, "events-archive.jsonl"); + const pendingPath = `${archivePath}.pending`; + const archiveBefore = '{"type":"existing-archive"}\n'; + const pendingBefore = `${fixture.marker || EVENT_COMPACTION_PENDING_MAGIC}\n${fixture.header}\n{"type":"staged-payload"}\n`; + seedEvents(eventsPath, 2); + fs.writeFileSync(archivePath, archiveBefore, { mode: 0o600 }); + fs.writeFileSync(pendingPath, pendingBefore, { mode: 0o600 }); + + await assert.rejects( + compactEvents(eventsPath, 1), + (error) => error?.code === "DOTAIOS_ARCHIVE_STATE_INVALID" + ); + + assert.equal(fs.readFileSync(archivePath, "utf8"), archiveBefore, + "invalid authority cannot mutate the archive"); + assert.equal(fs.readFileSync(pendingPath, "utf8"), pendingBefore, + "invalid authority remains available for inspection"); + assert.equal(readLines(eventsPath).length, 2, + "invalid authority cannot replace live memory"); + }); + } +}); + +test("a raw legacy pending JSONL batch remains recoverable", async () => { + const dir = tmpDir(); + const eventsPath = path.join(dir, "events.jsonl"); + const archivePath = path.join(dir, "events-archive.jsonl"); + const legacy = JSON.stringify({ + ts: "2025-12-31T23:59:59.000Z", + type: "legacy-pending", + artifact_contract: "dotaios-event-compaction/v1", + phase: "ready", + pendingRecords: 99 + }); + const live = JSON.stringify({ + ts: "2026-01-01T00:00:00.000Z", + type: "live" + }); + fs.writeFileSync(eventsPath, `${live}\n`, { mode: 0o600 }); + fs.writeFileSync(`${archivePath}.pending`, `${legacy}\n`, { mode: 0o600 }); + + await compactEvents(eventsPath, 10); + + assert.deepEqual(readEventsArchiveLines(dir), [legacy]); + assert.deepEqual(readLines(eventsPath), [live]); + assert.equal(fs.existsSync(`${archivePath}.pending`), false); +}); + +test("unmarked legacy pending accepts only JSON object records", async (t) => { + for (const pendingBefore of ["null\n", "42\n", '"text"\n', "[1,2,3]\n"]) { + await t.test(pendingBefore.trim(), async () => { + const dir = tmpDir(); + const eventsPath = path.join(dir, "events.jsonl"); + const archivePath = path.join(dir, "events-archive.jsonl"); + const pendingPath = `${archivePath}.pending`; + const liveBefore = '{"type":"live"}\n'; + const archiveBefore = '{"type":"existing-archive"}\n'; + fs.writeFileSync(eventsPath, liveBefore, { mode: 0o600 }); + fs.writeFileSync(archivePath, archiveBefore, { mode: 0o600 }); + fs.writeFileSync(pendingPath, pendingBefore, { mode: 0o600 }); + + await assert.rejects( + compactEvents(eventsPath, 10), + (error) => error?.code === "DOTAIOS_ARCHIVE_STATE_INVALID" + ); + + assert.equal(fs.readFileSync(eventsPath, "utf8"), liveBefore); + assert.equal(fs.readFileSync(archivePath, "utf8"), archiveBefore); + assert.equal(fs.readFileSync(pendingPath, "utf8"), pendingBefore); + }); + } +}); + +test("a crash after pending publication cannot route a new batch through legacy recovery", async () => { + const dir = tmpDir(); + const eventsPath = path.join(dir, "events.jsonl"); + const archivePath = path.join(dir, "events-archive.jsonl"); + const records = [1, 2, 3].map((n) => + `{ "ts": "2026-01-01T00:00:0${n}.000Z", "type": "manual", "n": ${n} }` + ); + fs.writeFileSync(eventsPath, `${records.join("\n")}\n`, { mode: 0o600 }); + let interrupted = false; + const filesystem = { + ...fsp, + async link(source, destination) { + const result = await fsp.link(source, destination); + if (!interrupted && destination === `${archivePath}.pending`) { + interrupted = true; + throw new Error("injected-crash:pending-published"); + } + return result; + } + }; + + await assert.rejects( + compactEvents(eventsPath, 1, { filesystem }), + /injected-crash:pending-published/ + ); + assert.equal(interrupted, true); + assert.equal(fs.existsSync(`${archivePath}.pending`), true); + const preparedHeader = JSON.parse(readLines(`${archivePath}.pending`)[1]); + assert.equal(preparedHeader.artifact_contract, "dotaios-event-compaction/v1"); + assert.equal(preparedHeader.phase, "prepared", + "every new pending batch must atomically carry prepared transaction authority"); + assert.equal(readLines(eventsPath).length, 3, "prepared evidence never authorizes the live rename"); + + await compactEvents(eventsPath, 1); + + const settled = [...readEventsArchiveLines(dir), ...readLines(eventsPath)]; + assert.equal(settled.length, 3, "the retry must settle the original three records exactly once"); + assert.deepEqual(settled.map((line) => JSON.parse(line).n), [1, 2, 3]); +}); + +test("event append recovers a prepared compaction before writing the new record", async () => { + const dir = tmpDir(); + const eventsPath = path.join(dir, "events.jsonl"); + const archivePath = path.join(dir, "events-archive.jsonl"); + const original = seedEvents(eventsPath, 3); + const appended = { + ts: "2026-01-01T00:00:09.000Z", + type: "post-crash-append", + summary: "saved after prepared publication" + }; + let interrupted = false; + const filesystem = { + ...fsp, + async link(source, destination) { + const result = await fsp.link(source, destination); + if (!interrupted && destination === `${archivePath}.pending`) { + interrupted = true; + throw new Error("injected-crash:prepared-published"); + } + return result; + } + }; + + await assert.rejects( + compactEvents(eventsPath, 1, { filesystem }), + /injected-crash:prepared-published/ + ); + + await appendEventRecord(eventsPath, appended); + assert.equal(fs.existsSync(`${archivePath}.pending`), false, + "the writer lock recovers prepared evidence before appending"); + await compactEvents(eventsPath, 1); + + assert.deepEqual( + [...readEventsArchiveLines(dir), ...readLines(eventsPath)], + [...original, JSON.stringify(appended)] + ); +}); + +test("prepared recovery preserves complete records appended by an older writer", async () => { + const dir = tmpDir(); + const eventsPath = path.join(dir, "events.jsonl"); + const archivePath = path.join(dir, "events-archive.jsonl"); + const original = seedEvents(eventsPath, 3); + const suffix = JSON.stringify({ + ts: "2026-01-01T00:00:09.000Z", + type: "older-writer-suffix", + summary: "identical complete append" + }); + let interrupted = false; + const filesystem = { + ...fsp, + async link(source, destination) { + const result = await fsp.link(source, destination); + if (!interrupted && destination === `${archivePath}.pending`) { + interrupted = true; + throw new Error("injected-crash:prepared-published"); + } + return result; + } + }; + + await assert.rejects( + compactEvents(eventsPath, 1, { filesystem }), + /injected-crash:prepared-published/ + ); + fs.appendFileSync(eventsPath, `${suffix}\n${suffix}\n`); + + await compactEvents(eventsPath, 1); + + assert.deepEqual( + [...readEventsArchiveLines(dir), ...readLines(eventsPath)], + [...original, suffix, suffix], + "suffix recovery preserves byte-identical complete appends as distinct records" + ); +}); + +test("prepared recovery scales linearly with a large live prefix", async () => { + const measureRecovery = async (count) => { + const dir = tmpDir(); + const eventsPath = path.join(dir, "events.jsonl"); + const archivePath = path.join(dir, "events-archive.jsonl"); + seedEvents(eventsPath, count); + const suffix = JSON.stringify({ + ts: "2026-01-01T00:01:00.000Z", + type: "older-writer-suffix" + }); + let interrupted = false; + const filesystem = { + ...fsp, + async link(source, destination) { + const result = await fsp.link(source, destination); + if (!interrupted && destination === `${archivePath}.pending`) { + interrupted = true; + throw new Error("injected-crash:prepared-published"); + } + return result; + } + }; + + await assert.rejects( + compactEvents(eventsPath, 1, { filesystem }), + /injected-crash:prepared-published/ + ); + assert.equal(interrupted, true, "the prepared-publication crash must fire"); + fs.appendFileSync(eventsPath, `${suffix}\n`); + const started = performance.now(); + const result = await compactEvents(eventsPath, count + 10); + const elapsedMs = performance.now() - started; + + assert.deepEqual(result, { archived: 0, kept: count + 1 }); + assert.equal(fs.existsSync(`${archivePath}.pending`), false); + assert.equal(readLines(eventsPath).length, count + 1); + return elapsedMs; + }; + + const sampleMinimum = async (count) => { + const samples = []; + for (let sample = 0; sample < 3; sample += 1) samples.push(await measureRecovery(count)); + return Math.min(...samples); + }; + const smallCount = 1_250; + const largeCount = 5_000; + const smallMs = await sampleMinimum(smallCount); + const largeMs = await sampleMinimum(largeCount); + const linearGrowth = largeCount / smallCount; + const allowedMs = smallMs * linearGrowth * 2 + 25; + + assert.ok( + largeMs <= allowedMs, + `4x recovery grew from ${smallMs.toFixed(1)}ms to ${largeMs.toFixed(1)}ms; allowed ${allowedMs.toFixed(1)}ms` + ); +}); + +test("prepared recovery rejects an incomplete appended suffix without consuming authority", async () => { + const dir = tmpDir(); + const eventsPath = path.join(dir, "events.jsonl"); + const archivePath = path.join(dir, "events-archive.jsonl"); + seedEvents(eventsPath, 3); + let interrupted = false; + const filesystem = { + ...fsp, + async link(source, destination) { + const result = await fsp.link(source, destination); + if (!interrupted && destination === `${archivePath}.pending`) { + interrupted = true; + throw new Error("injected-crash:prepared-published"); + } + return result; + } + }; + + await assert.rejects(compactEvents(eventsPath, 1, { filesystem })); + fs.appendFileSync(eventsPath, '{"type":"partial"'); + const liveBefore = fs.readFileSync(eventsPath, "utf8"); + const pendingBefore = fs.readFileSync(`${archivePath}.pending`, "utf8"); + + await assert.rejects( + compactEvents(eventsPath, 1), + (error) => error?.code === "DOTAIOS_ARCHIVE_STATE_INVALID" + ); + assert.equal(fs.readFileSync(eventsPath, "utf8"), liveBefore); + assert.equal(fs.readFileSync(`${archivePath}.pending`, "utf8"), pendingBefore); + assert.equal(fs.existsSync(archivePath), false); +}); + +test("compaction recovery never mistakes an unrelated archive append for pending progress", async () => { + const dir = tmpDir(); + const eventsPath = path.join(dir, "events.jsonl"); + const archivePath = path.join(dir, "events-archive.jsonl"); + const original = seedEvents(eventsPath, 3); + const unrelated = JSON.stringify({ + ts: "2026-01-01T00:00:09.000Z", + type: "unrelated", + summary: "separate archive writer" + }); + + const fault = faultFs({ link: 2 }); + await assert.rejects( + compactEvents(eventsPath, 1, { filesystem: fault.filesystem }), + /injected-crash:link:2/ + ); + fs.appendFileSync(archivePath, `${unrelated}\n`, { mode: 0o600 }); + + await assert.rejects( + compactEvents(eventsPath, 1), + (error) => error?.code === "DOTAIOS_ARCHIVE_STATE_INVALID" + ); + + const durable = [ + ...readEventsArchiveLines(dir), + ...readLines(eventsPath), + ...readLines(`${archivePath}.pending`) + ]; + for (const line of original) { + assert.ok(durable.includes(line), `unrelated mutation must not erase pending original: ${line}`); + } + assert.ok(durable.includes(unrelated)); + assert.equal(fs.existsSync(`${archivePath}.pending`), true, + "unrecognized archive mutation preserves recovery evidence for inspection"); +}); + +test("a crash after ready publication still cannot replace live memory until retry", async () => { + const dir = tmpDir(); + const eventsPath = path.join(dir, "events.jsonl"); + const archivePath = path.join(dir, "events-archive.jsonl"); + const original = seedEvents(eventsPath, 3); + let interrupted = false; + const filesystem = { + ...fsp, + async rename(source, destination) { + const result = await fsp.rename(source, destination); + if (!interrupted && destination === `${archivePath}.pending`) { + interrupted = true; + throw new Error("injected-crash:ready-published"); + } + return result; + } + }; + + await assert.rejects( + compactEvents(eventsPath, 1, { filesystem }), + /injected-crash:ready-published/ + ); + assert.equal(interrupted, true); + assert.deepEqual(readLines(eventsPath), original, "ready evidence still precedes the live commit point"); + assert.equal(JSON.parse(readLines(`${archivePath}.pending`)[1]).phase, "ready"); + + await compactEvents(eventsPath, 1); + + const settled = [...readEventsArchiveLines(dir), ...readLines(eventsPath)]; + assert.deepEqual(settled, original); +}); + +test("event append recovers a ready compaction before writing the new record", async () => { + const dir = tmpDir(); + const eventsPath = path.join(dir, "events.jsonl"); + const archivePath = path.join(dir, "events-archive.jsonl"); + const original = seedEvents(eventsPath, 3); + const appended = { + ts: "2026-01-01T00:00:09.000Z", + type: "post-crash-append", + summary: "saved after ready publication" + }; + let interrupted = false; + const filesystem = { + ...fsp, + async rename(source, destination) { + const result = await fsp.rename(source, destination); + if (!interrupted && destination === `${archivePath}.pending`) { + interrupted = true; + throw new Error("injected-crash:ready-published"); + } + return result; + } + }; + + await assert.rejects( + compactEvents(eventsPath, 1, { filesystem }), + /injected-crash:ready-published/ + ); + + await appendEventRecord(eventsPath, appended); + assert.equal(fs.existsSync(`${archivePath}.pending`), false, + "the writer lock recovers ready evidence before appending"); + await compactEvents(eventsPath, 1); + + assert.deepEqual( + [...readEventsArchiveLines(dir), ...readLines(eventsPath)], + [...original, JSON.stringify(appended)] + ); +}); + +test("ready recovery preserves complete records appended after the live commit", async () => { + const dir = tmpDir(); + const eventsPath = path.join(dir, "events.jsonl"); + const archivePath = path.join(dir, "events-archive.jsonl"); + const original = seedEvents(eventsPath, 3); + const suffix = JSON.stringify({ + ts: "2026-01-01T00:00:09.000Z", + type: "older-writer-suffix", + summary: "identical complete append after live commit" + }); + const fault = faultFs({ link: 2 }); + + await assert.rejects( + compactEvents(eventsPath, 1, { filesystem: fault.filesystem }), + /injected-crash:link:2/ + ); + assert.equal(JSON.parse(readLines(`${archivePath}.pending`)[1]).phase, "ready"); + assert.equal(readLines(eventsPath).length, 1, "the live replacement committed before this crash"); + fs.appendFileSync(eventsPath, `${suffix}\n${suffix}\n`); + + await compactEvents(eventsPath, 1); + + assert.deepEqual( + [...readEventsArchiveLines(dir), ...readLines(eventsPath)], + [...original, suffix, suffix], + "ready recovery archives the pending prefix and keeps both appended records" + ); +}); + +test("legacy active normalization recovers marker, shard, and active transitions exactly once", async (t) => { + const boundaries = [ + { + name: "marker publication", + method: "link", + matches: (source, destination, archivePath) => destination === `${archivePath}.rotation`, + shardExists: false + }, + { + name: "shard publication", + method: "link", + matches: (source, destination, archivePath) => destination === archivePath.replace(/\.jsonl$/, ".000001.jsonl"), + shardExists: true + }, + { + name: "active replacement", + method: "rename", + matches: (source, destination, archivePath) => destination === archivePath, + shardExists: true + } + ]; + for (const boundary of boundaries) { + await t.test(boundary.name, async () => { + const { dir, eventsPath, archivePath, expectedArchive, live } = legacyRotationFixture(); + const fault = failAfterArchiveBoundary( + boundary.method, + (source, destination) => boundary.matches(source, destination, archivePath) + ); + + await assert.rejects( + compactEvents(eventsPath, 1, { filesystem: fault.filesystem }), + /injected-crash/ + ); + + assert.equal(fault.interrupted(), true, `${boundary.name} must be reached`); + assert.equal(fs.existsSync(`${archivePath}.rotation`), true, "the recovery marker remains authoritative"); + assert.equal( + fs.existsSync(archivePath.replace(/\.jsonl$/, ".000001.jsonl")), + boundary.shardExists, + "the interrupted boundary must leave the expected shard state" + ); + + await compactEvents(eventsPath, 1); + + assert.deepEqual(readEventsArchiveLines(dir), expectedArchive); + assert.deepEqual(readLines(eventsPath), [live]); + assert.equal(fs.existsSync(`${archivePath}.rotation`), false, "recovery consumes its marker"); + assert.equal(fs.existsSync(`${archivePath}.pending`), false, "recovery consumes the staged batch"); + }); + } +}); + +test("rotation recovers real process death inside exclusive archive publication", async (t) => { + if (process.platform === "win32") return; + for (const scenario of [ + { name: "rotation marker", suffix: ".rotation" }, + { name: "immutable shard", suffix: ".000001.jsonl" } + ]) { + await t.test(scenario.name, async () => { + const dir = tmpDir(); + const eventsPath = path.join(dir, "events.jsonl"); + const archivePath = path.join(dir, "events-archive.jsonl"); + const targetPath = scenario.suffix === ".rotation" + ? `${archivePath}.rotation` + : archivePath.replace(/\.jsonl$/, scenario.suffix); + const markerPath = `${archivePath}.rotation`; + const original = Array.from({ length: 4 }, (_, index) => JSON.stringify({ + ts: `2026-01-01T00:00:0${index}.000Z`, + type: "hard-kill-rotation", + id: `event-${index}`, + summary: index < 3 ? "x".repeat(720_000) : "kept" + })); + fs.writeFileSync(eventsPath, `${original.join("\n")}\n`, { mode: 0o600 }); + const memoryModule = new URL("../../packages/core/src/memory.mjs", import.meta.url).href; + const child = spawnSync(process.execPath, ["--input-type=module", "-e", ` + import fs from "node:fs/promises"; + import { compactEvents } from ${JSON.stringify(memoryModule)}; + const filesystem = { + ...fs, + async link(source, destination) { + await fs.link(source, destination); + if (destination.endsWith(process.env.DOTAIOS_TEST_KILL_SUFFIX)) { + process.kill(process.pid, "SIGKILL"); + } + } + }; + await compactEvents(process.env.DOTAIOS_TEST_EVENTS_PATH, 1, { filesystem }); + `], { + env: { + ...process.env, + DOTAIOS_TEST_EVENTS_PATH: eventsPath, + DOTAIOS_TEST_KILL_SUFFIX: scenario.suffix + }, + encoding: "utf8" + }); + + assert.equal(child.signal, "SIGKILL", child.stderr || child.stdout); + const targetStats = fs.lstatSync(targetPath); + assert.equal(targetStats.nlink, 2, "the killed publisher must leave the linked temporary behind"); + const temporaryPrefix = `.${path.basename(targetPath)}.`; + assert.equal( + fs.readdirSync(dir).filter((name) => name.startsWith(temporaryPrefix) && name.endsWith(".tmp")).length, + 1, + "the fixture must exercise the real link-before-unlink crash window" + ); + + await compactEvents(eventsPath, 1); + + assert.deepEqual( + readEventsArchiveLines(dir).map((line) => JSON.parse(line).id), + ["event-0", "event-1", "event-2"] + ); + assert.deepEqual(readLines(eventsPath).map((line) => JSON.parse(line).id), ["event-3"]); + assert.equal(fs.existsSync(markerPath), false); + assert.equal( + fs.readdirSync(dir).some((name) => name.startsWith(temporaryPrefix) && name.endsWith(".tmp")), + false + ); + }); + } +}); + +test("markerless ordinary rotation overlap fails closed for explicit legacy recovery", async () => { + const dir = tmpDir(); + const eventsPath = path.join(dir, "events.jsonl"); + const archivePath = path.join(dir, "events-archive.jsonl"); + const shardPath = path.join(dir, "events-archive.000001.jsonl"); + const pendingPath = `${archivePath}.pending`; + const overlapped = [ + JSON.stringify({ ts: "2026-01-01T00:00:00.000Z", type: "legacy-overlap", id: "old-0" }), + JSON.stringify({ ts: "2026-01-01T00:00:01.000Z", type: "legacy-overlap", id: "old-1" }) + ]; + const pending = JSON.stringify({ + ts: "2026-01-01T00:00:02.000Z", + type: "legacy-overlap", + id: "pending" + }); + const live = JSON.stringify({ ts: "2026-01-01T00:00:03.000Z", type: "live", id: "live" }); + const overlapContent = `${overlapped.join("\n")}\n`; + fs.writeFileSync(archivePath, overlapContent, { mode: 0o600 }); + fs.writeFileSync(shardPath, overlapContent, { mode: 0o600 }); + fs.writeFileSync(pendingPath, `${pending}\n`, { mode: 0o600 }); + fs.writeFileSync(eventsPath, `${live}\n`, { mode: 0o600 }); + const before = Object.fromEntries( + [archivePath, shardPath, pendingPath, eventsPath] + .map((filePath) => [filePath, fs.readFileSync(filePath)]) + ); + + await assert.rejects( + () => compactEvents(eventsPath, 1), + (error) => { + assert.equal(error?.code, "DOTAIOS_ARCHIVE_LEGACY_RECOVERY_REQUIRED"); + assert.match(error.message, /legacy rotation recovery/i); + assert.deepEqual(error.diagnostic, { + kind: "markerless-rotation-overlap", + archive: "events-archive.jsonl", + shard: "events-archive.000001.jsonl", + action: "preserve-and-inspect" + }); + return true; + } + ); + + for (const [filePath, content] of Object.entries(before)) { + assert.deepEqual(fs.readFileSync(filePath), content, `${path.basename(filePath)} must remain authoritative`); + } + assert.equal(fs.existsSync(path.join(dir, "events-archive.000002.jsonl")), false); + assert.equal(fs.existsSync(`${archivePath}.rotation`), false); + assert.equal(fs.existsSync(`${archivePath}.rotation-format`), false); +}); + +test("markerless oversized normalization overlap fails before publishing another shard", async () => { + const dir = tmpDir(); + const eventsPath = path.join(dir, "events.jsonl"); + const archivePath = path.join(dir, "events-archive.jsonl"); + const shardPath = path.join(dir, "events-archive.000001.jsonl"); + const pendingPath = `${archivePath}.pending`; + const legacy = Array.from({ length: 3 }, (_, index) => JSON.stringify({ + ts: `2026-01-01T00:00:0${index}.000Z`, + type: "legacy-overlap", + id: `old-${index}`, + summary: "x".repeat(720_000) + })); + const pending = JSON.stringify({ + ts: "2026-01-01T00:00:03.000Z", + type: "legacy-overlap", + id: "pending" + }); + const live = JSON.stringify({ ts: "2026-01-01T00:00:04.000Z", type: "live", id: "live" }); + const activeContent = `${legacy.join("\n")}\n`; + const shardContent = `${legacy.slice(0, 2).join("\n")}\n`; + fs.writeFileSync(archivePath, activeContent, { mode: 0o600 }); + fs.writeFileSync(shardPath, shardContent, { mode: 0o600 }); + fs.writeFileSync(pendingPath, `${pending}\n`, { mode: 0o600 }); + fs.writeFileSync(eventsPath, `${live}\n`, { mode: 0o600 }); + + await assert.rejects( + () => compactEvents(eventsPath, 1), + (error) => error?.code === "DOTAIOS_ARCHIVE_LEGACY_RECOVERY_REQUIRED" + ); + + assert.equal(fs.readFileSync(archivePath, "utf8"), activeContent); + assert.equal(fs.readFileSync(shardPath, "utf8"), shardContent); + assert.equal(fs.readFileSync(pendingPath, "utf8"), `${pending}\n`); + assert.equal(fs.readFileSync(eventsPath, "utf8"), `${live}\n`); + assert.equal(fs.existsSync(path.join(dir, "events-archive.000002.jsonl")), false); + assert.equal(fs.existsSync(`${archivePath}.rotation-format`), false); +}); + test("compaction re-run on an already-compacted file is a no-op", async () => { const dir = tmpDir(); const eventsPath = path.join(dir, "events.jsonl"); @@ -158,7 +1016,10 @@ test("event append waits for compaction and the appended record survives replace const filesystem = { ...fsp, async rename(source, destination) { - if (source === `${eventsPath}.tmp` && destination === eventsPath) { + const isReplacement = path.dirname(source) === path.dirname(eventsPath) + && path.basename(source).startsWith(`.${path.basename(eventsPath)}.`) + && path.basename(source).endsWith(".tmp"); + if (isReplacement && destination === eventsPath) { renameStarted(); await allowRename; } @@ -446,10 +1307,10 @@ test("a crash while deleting a trimmed signal file loses no line and duplicates assert.deepEqual(fs.readdirSync(signalsDir), [], "the retry finishes the removal"); }); -test("a crash while appending to the signals archive keeps the source file", async () => { +test("a crash while publishing the signals archive keeps the source file", async () => { const { signalsDir, archivePath } = signalsFixture({ staleFiles: 2, linesPerFile: 2 }); - const { filesystem } = faultFs({ appendFile: 1 }); + const { filesystem } = faultFs({ link: 1 }); await assert.rejects(() => trimSignals(signalsDir, 30, { filesystem })); assert.equal(fs.readdirSync(signalsDir).length, 2, "nothing may be deleted before the archive holds it"); @@ -473,6 +1334,99 @@ test("a staged signals batch left behind by a crash is recovered, not lost", asy assert.equal(readLines(archivePath).length, 2, "recovery must not duplicate the batch"); }); +test("archive maintenance rejects an in-place pending mutation without deleting authority", async () => { + const { signalsDir, archivePath, names } = signalsFixture({ staleFiles: 1, linesPerFile: 2 }); + const pendingPath = `${archivePath}.pending`; + const sourcePath = path.join(signalsDir, names[0]); + let mutated = false; + const filesystem = { + ...fsp, + async open(filePath, flags, ...rest) { + const handle = await fsp.open(filePath, flags, ...rest); + if (filePath !== pendingPath || typeof flags !== "number") return handle; + return new Proxy(handle, { + get(target, name) { + if (name === "readFile") { + return async (...args) => { + const content = await target.readFile(...args); + if (!mutated) { + await fsp.appendFile(filePath, '{"type":"same-inode-mutation"}\n'); + mutated = true; + } + return content; + }; + } + const value = Reflect.get(target, name, target); + return typeof value === "function" ? value.bind(target) : value; + } + }); + } + }; + + await assert.rejects( + () => trimSignals(signalsDir, 30, { filesystem }), + (error) => error?.code === "DOTAIOS_ARCHIVE_STATE_INVALID" + ); + + assert.equal(mutated, true, "the test must mutate the already-open pending inode"); + assert.equal(fs.existsSync(pendingPath), true, "the pending batch remains recovery authority"); + assert.equal(fs.existsSync(sourcePath), true, "the source remains authoritative until publication succeeds"); + assert.equal(fs.existsSync(archivePath), false, "no torn archive generation is published"); +}); + +test("legacy archive mode repair never follows a substituted pathname", async () => { + if (process.platform === "win32") return; + const dir = tmpDir(); + const eventsPath = path.join(dir, "events.jsonl"); + const archivePath = path.join(dir, "events-archive.jsonl"); + const displacedPath = path.join(dir, "events-archive.displaced.jsonl"); + const outsidePath = path.join(dir, "outside.jsonl"); + fs.writeFileSync(archivePath, '{"type":"legacy"}\n', { mode: 0o644 }); + fs.chmodSync(archivePath, 0o644); + fs.writeFileSync(outsidePath, '{"type":"outside"}\n', { mode: 0o644 }); + fs.chmodSync(outsidePath, 0o644); + seedEvents(eventsPath, 2); + let substituted = false; + const substitutePath = () => { + if (substituted) return; + fs.renameSync(archivePath, displacedPath); + fs.symlinkSync(outsidePath, archivePath); + substituted = true; + }; + const filesystem = { + ...fsp, + async chmod(filePath, mode) { + if (filePath === archivePath) substitutePath(); + return fsp.chmod(filePath, mode); + }, + async open(filePath, flags, ...rest) { + const handle = await fsp.open(filePath, flags, ...rest); + if (filePath !== archivePath || typeof flags !== "number") return handle; + return new Proxy(handle, { + get(target, name) { + if (name === "chmod") { + return async (...args) => { + substitutePath(); + return target.chmod(...args); + }; + } + const value = Reflect.get(target, name, target); + return typeof value === "function" ? value.bind(target) : value; + } + }); + } + }; + + await assert.rejects( + () => compactEvents(eventsPath, 1, { filesystem }), + (error) => error?.code === "DOTAIOS_OWNED_STATE_INVALID" + ); + + assert.equal(substituted, true, "the pathname must be swapped at the repair boundary"); + assert.equal(fs.statSync(outsidePath).mode & 0o777, 0o644, "repair must not chmod the symlink target"); + assert.equal(fs.existsSync(`${archivePath}.pending`), true, "the pending batch remains authoritative"); +}); + test("trimSignals skips when another process holds the archive lock", async () => { const { signalsDir, archivePath } = signalsFixture({ staleFiles: 1, linesPerFile: 1 }); fs.writeFileSync(`${archivePath}.lock`, JSON.stringify({ pid: 1, ts: Date.now() })); diff --git a/tests/core/memory.test.mjs b/tests/core/memory.test.mjs index 02fa2ca4..f2bbfc6e 100644 --- a/tests/core/memory.test.mjs +++ b/tests/core/memory.test.mjs @@ -21,6 +21,31 @@ function tmpDir() { return fs.mkdtempSync(path.join(os.tmpdir(), "dotaios-mem-test-")); } +const ARCHIVE_ROTATE_BYTES = 2 * 1024 * 1024; +const ARCHIVE_LINE_MAX_BYTES = 4 * 1024 * 1024; + +function archiveEvent(index, payloadBytes) { + return { + ts: `2026-05-${String(index + 1).padStart(2, "0")}T12:00:00.000Z`, + type: "archive-fixture", + summary: `archive-${index}-${"x".repeat(payloadBytes)}` + }; +} + +function readEventsArchiveGeneration(dir) { + return fs.readdirSync(dir) + .filter((name) => /^events-archive(?:\.\d{6})?\.jsonl$/.test(name)) + .sort((left, right) => { + if (left === "events-archive.jsonl") return 1; + if (right === "events-archive.jsonl") return -1; + return left.localeCompare(right); + }) + .flatMap((name) => fs.readFileSync(path.join(dir, name), "utf8") + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line))); +} + test("parseJsonlLine parses valid JSON and skips blanks", () => { assert.deepEqual(parseJsonlLine('{"a":1}'), { a: 1 }); assert.equal(parseJsonlLine(""), null); @@ -132,6 +157,182 @@ test("compactEvents archives old entries and keeps recent", async () => { assert.equal(archived[0].i, 0); }); +test("compactEvents rotates complete JSONL lines into immutable numbered shards", async () => { + const dir = tmpDir(); + const eventsPath = path.join(dir, "events.jsonl"); + const events = [ + archiveEvent(0, 720_000), + archiveEvent(1, 720_000), + archiveEvent(2, 720_000), + archiveEvent(3, 16) + ]; + fs.writeFileSync(eventsPath, events.map(formatJsonlEntry).join(""), { mode: 0o600 }); + + await compactEvents(eventsPath, 1); + + const shardPath = path.join(dir, "events-archive.000001.jsonl"); + const activePath = path.join(dir, "events-archive.jsonl"); + assert.equal(fs.existsSync(shardPath), true); + assert.equal(fs.existsSync(activePath), true); + assert.ok(fs.statSync(shardPath).size <= ARCHIVE_ROTATE_BYTES); + assert.ok(fs.statSync(activePath).size <= ARCHIVE_ROTATE_BYTES); + assert.equal(fs.statSync(shardPath).mode & 0o777, 0o600); + assert.equal(fs.statSync(activePath).mode & 0o777, 0o600); + const archived = [...await readJsonl(shardPath), ...await readJsonl(activePath)]; + assert.deepEqual(archived.map(({ summary }) => summary.slice(0, 9)), ["archive-0", "archive-1", "archive-2"]); +}); + +test("legacy over-target active archive normalizes before compacting a pending batch", async () => { + const dir = tmpDir(); + const eventsPath = path.join(dir, "events.jsonl"); + const archivePath = path.join(dir, "events-archive.jsonl"); + const legacy = [archiveEvent(0, 720_000), archiveEvent(1, 720_000), archiveEvent(2, 720_000)]; + const pending = [archiveEvent(3, 16), archiveEvent(4, 16)]; + const live = archiveEvent(5, 16); + fs.writeFileSync(archivePath, legacy.map(formatJsonlEntry).join(""), { mode: 0o600 }); + fs.writeFileSync(eventsPath, [...pending, live].map(formatJsonlEntry).join(""), { mode: 0o600 }); + + const result = await compactEvents(eventsPath, 1); + + assert.deepEqual(result, { archived: 2, kept: 1 }); + assert.equal(fs.existsSync(path.join(dir, "events-archive.000001.jsonl")), true); + assert.deepEqual(readEventsArchiveGeneration(dir), [...legacy, ...pending]); + assert.deepEqual(await readJsonl(eventsPath), [live]); +}); + +test("normal rotations preserve legitimate byte-identical event records", async () => { + const dir = tmpDir(); + const eventsPath = path.join(dir, "events.jsonl"); + const repeated = { + ts: "2026-05-01T12:00:00.000Z", + type: "repeated-event", + summary: "x".repeat(1_100_000) + }; + const firstTail = { ts: "2026-05-02T12:00:00.000Z", type: "tail", id: 1 }; + const secondTail = { ts: "2026-05-03T12:00:00.000Z", type: "tail", id: 2 }; + fs.writeFileSync( + eventsPath, + `${formatJsonlEntry(repeated).repeat(3)}${formatJsonlEntry(firstTail)}`, + { mode: 0o600 } + ); + + await compactEvents(eventsPath, 1); + fs.appendFileSync(eventsPath, formatJsonlEntry(secondTail)); + await compactEvents(eventsPath, 1); + + const stored = readEventsArchiveGeneration(dir); + stored.push(...await readJsonl(eventsPath)); + + assert.deepEqual( + stored.map((entry) => entry.type === "repeated-event" ? "repeated" : `tail-${entry.id}`), + ["repeated", "repeated", "repeated", "tail-1", "tail-2"] + ); +}); + +test("one valid line above the rotation target occupies its own shard", async () => { + const dir = tmpDir(); + const eventsPath = path.join(dir, "events.jsonl"); + const huge = archiveEvent(0, ARCHIVE_ROTATE_BYTES + 4096); + const kept = archiveEvent(1, 16); + fs.writeFileSync(eventsPath, `${formatJsonlEntry(huge)}${formatJsonlEntry(kept)}`, { mode: 0o600 }); + + await compactEvents(eventsPath, 1); + + const shardPath = path.join(dir, "events-archive.000001.jsonl"); + assert.deepEqual((await readJsonl(shardPath)).map(({ summary }) => summary.slice(0, 9)), ["archive-0"]); + assert.ok(fs.statSync(shardPath).size > ARCHIVE_ROTATE_BYTES); + assert.ok(fs.statSync(shardPath).size <= ARCHIVE_LINE_MAX_BYTES); + assert.deepEqual(await readJsonl(path.join(dir, "events-archive.jsonl")), []); +}); + +test("rotation never overwrites a preexisting immutable shard", async () => { + const dir = tmpDir(); + const eventsPath = path.join(dir, "events.jsonl"); + const firstShard = path.join(dir, "events-archive.000001.jsonl"); + const preserved = formatJsonlEntry({ ts: "2026-04-01T00:00:00.000Z", type: "preserved-shard" }); + fs.writeFileSync(firstShard, preserved, { mode: 0o600 }); + fs.writeFileSync(eventsPath, [ + archiveEvent(0, 1_100_000), + archiveEvent(1, 1_100_000), + archiveEvent(2, 16) + ].map(formatJsonlEntry).join(""), { mode: 0o600 }); + + await compactEvents(eventsPath, 1); + + assert.equal(fs.readFileSync(firstShard, "utf8"), preserved); + assert.equal(fs.existsSync(path.join(dir, "events-archive.000002.jsonl")), true); +}); + +test("an archive line above the read ceiling fails before signal source removal", async () => { + const dir = tmpDir(); + const signalsDir = path.join(dir, "signals"); + fs.mkdirSync(signalsDir); + const oldDate = "2020-01-01"; + const sourcePath = path.join(signalsDir, `${oldDate}.jsonl`); + fs.writeFileSync(sourcePath, formatJsonlEntry({ + type: "oversized", + summary: "x".repeat(ARCHIVE_LINE_MAX_BYTES) + })); + + await assert.rejects( + () => trimSignals(signalsDir, 30), + (error) => error?.code === "DOTAIOS_ARCHIVE_LINE_TOO_LARGE" + ); + assert.equal(fs.existsSync(sourcePath), true); + assert.equal(fs.existsSync(path.join(dir, "signals-archive.jsonl")), false); + assert.equal(fs.existsSync(path.join(dir, "signals-archive.000001.jsonl")), false); +}); + +test("archive maintenance narrows an eligible legacy active file from 0644 to 0600", async () => { + if (process.platform === "win32") return; + const dir = tmpDir(); + const eventsPath = path.join(dir, "events.jsonl"); + const archivePath = path.join(dir, "events-archive.jsonl"); + fs.writeFileSync(archivePath, formatJsonlEntry(archiveEvent(0, 16)), { mode: 0o644 }); + fs.chmodSync(archivePath, 0o644); + fs.writeFileSync( + eventsPath, + `${formatJsonlEntry(archiveEvent(1, 16))}${formatJsonlEntry(archiveEvent(2, 16))}`, + { mode: 0o600 } + ); + + await compactEvents(eventsPath, 1); + + assert.equal(fs.statSync(archivePath).mode & 0o777, 0o600); + assert.deepEqual((await readJsonl(archivePath)).map(({ summary }) => summary.slice(0, 9)), [ + "archive-0", + "archive-1" + ]); +}); + +test("unsafe archive identities and modes fail before trimmed signal deletion", async (t) => { + if (process.platform === "win32") return; + for (const fixture of ["hard-link-active", "linked-shard", "writable-active"]) { + await t.test(fixture, async () => { + const dir = tmpDir(); + const signalsDir = path.join(dir, "signals"); + const sourcePath = path.join(signalsDir, "2020-01-01.jsonl"); + const archivePath = path.join(dir, "signals-archive.jsonl"); + const outsidePath = path.join(dir, "outside.jsonl"); + fs.mkdirSync(signalsDir); + fs.writeFileSync(sourcePath, '{"type":"must-survive"}\n'); + fs.writeFileSync(outsidePath, '{"type":"outside"}\n', { mode: 0o600 }); + if (fixture === "hard-link-active") fs.linkSync(outsidePath, archivePath); + if (fixture === "linked-shard") fs.symlinkSync(outsidePath, path.join(dir, "signals-archive.000001.jsonl")); + if (fixture === "writable-active") { + fs.writeFileSync(archivePath, '{"type":"legacy"}\n', { mode: 0o666 }); + fs.chmodSync(archivePath, 0o666); + } + const outsideBefore = fs.readFileSync(outsidePath); + + await assert.rejects(() => trimSignals(signalsDir, 30)); + + assert.equal(fs.existsSync(sourcePath), true); + assert.deepEqual(fs.readFileSync(outsidePath), outsideBefore); + }); + } +}); + test("trimSignals removes files older than retention period", async () => { const dir = tmpDir(); const signalsDir = path.join(dir, "signals"); @@ -152,6 +353,22 @@ test("trimSignals removes files older than retention period", async () => { assert.deepEqual(archived, [{ type: "old" }], "a trimmed signal is moved to the archive, never dropped"); }); +test("concurrent signal maintenance converges without loss or duplicate archive lines", async () => { + const dir = tmpDir(); + const signalsDir = path.join(dir, "signals"); + fs.mkdirSync(signalsDir); + const sourcePath = path.join(signalsDir, "2020-01-01.jsonl"); + fs.writeFileSync(sourcePath, '{"type":"one"}\n{"type":"two"}\n'); + + await Promise.all([trimSignals(signalsDir, 30), trimSignals(signalsDir, 30)]); + + assert.equal(fs.existsSync(sourcePath), false); + assert.deepEqual(await readJsonl(path.join(dir, "signals-archive.jsonl")), [ + { type: "one" }, + { type: "two" } + ]); +}); + test("searchMemory returns matches by timestamp across events archives and signals", async () => { const dir = tmpDir(); const memoryDir = path.join(dir, "memory"); @@ -177,3 +394,97 @@ test("searchMemory returns matches by timestamp across events archives and signa "2026-05-02T12:00:00.000Z" ]); }); + +test("searchMemory discovers numbered shards in numeric order and deduplicates retry overlap", async () => { + const dir = tmpDir(); + const memoryDir = path.join(dir, "memory"); + fs.mkdirSync(memoryDir); + const older = { ts: "2026-05-01T12:00:00.000Z", type: "note", summary: "sharded needle older" }; + const overlap = { ts: "2026-05-02T12:00:00.000Z", type: "note", summary: "sharded needle overlap" }; + const active = { ts: "2026-05-03T12:00:00.000Z", type: "note", summary: "sharded needle active" }; + fs.writeFileSync(path.join(memoryDir, "events-archive.000002.jsonl"), formatJsonlEntry(overlap), { mode: 0o600 }); + fs.writeFileSync(path.join(memoryDir, "events-archive.000001.jsonl"), formatJsonlEntry(older), { mode: 0o600 }); + fs.writeFileSync( + path.join(memoryDir, "events-archive.jsonl"), + `${formatJsonlEntry(overlap)}${formatJsonlEntry(active)}`, + { mode: 0o600 } + ); + + const results = await searchMemory(memoryDir, "sharded needle"); + + assert.deepEqual(results.map(({ summary }) => summary), [ + "sharded needle active", + "sharded needle overlap", + "sharded needle older" + ]); + assert.equal(results.filter(({ summary }) => summary.endsWith("overlap")).length, 1); + assert.match(results.find(({ summary }) => summary.endsWith("older")).source, /000001/); +}); + +test("searchMemory deduplicates newest shard, active archive, and live retry provenance as a multiset", async () => { + const dir = tmpDir(); + const memoryDir = path.join(dir, "memory"); + const signalsDir = path.join(memoryDir, "signals"); + fs.mkdirSync(signalsDir, { recursive: true }); + const overlap = { + ts: "2026-05-07T12:00:00.000Z", + type: "signal", + summary: "three generation retry needle" + }; + const serialized = formatJsonlEntry(overlap); + fs.writeFileSync(path.join(memoryDir, "signals-archive.000001.jsonl"), `${serialized}${serialized}`, { mode: 0o600 }); + fs.writeFileSync(path.join(memoryDir, "signals-archive.jsonl"), serialized, { mode: 0o600 }); + fs.writeFileSync(path.join(signalsDir, "2026-05-07.jsonl"), `${serialized}${serialized}`); + + const results = await searchMemory(memoryDir, "three generation retry needle", { limit: 10 }); + + assert.equal(results.length, 2, "retry copies collapse to the largest provenance multiplicity"); + assert.ok(results.every(({ source }) => source === "memory/signals/2026-05-07.jsonl")); +}); + +test("searchMemory preserves legitimate identical records outside retry overlap", async () => { + const dir = tmpDir(); + const memoryDir = path.join(dir, "memory"); + const signalsDir = path.join(memoryDir, "signals"); + fs.mkdirSync(signalsDir, { recursive: true }); + const duplicate = { + ts: "2026-05-06T12:00:00.000Z", + type: "note", + summary: "legitimate duplicate needle" + }; + const shardDuplicate = { + ts: "2026-05-05T12:00:00.000Z", + type: "note", + summary: "legitimate duplicate needle across immutable shards" + }; + const serialized = formatJsonlEntry(duplicate); + const serializedShard = formatJsonlEntry(shardDuplicate); + + fs.writeFileSync(path.join(memoryDir, "events.jsonl"), `${serialized}${serialized}`); + fs.writeFileSync(path.join(memoryDir, "events-archive.jsonl"), serialized, { mode: 0o600 }); + fs.writeFileSync(path.join(memoryDir, "events-archive.000001.jsonl"), serializedShard, { mode: 0o600 }); + fs.writeFileSync(path.join(memoryDir, "events-archive.000002.jsonl"), serializedShard, { mode: 0o600 }); + fs.writeFileSync(path.join(memoryDir, "signals-archive.jsonl"), serialized, { mode: 0o600 }); + fs.writeFileSync(path.join(signalsDir, "laptop-2026-05-06.jsonl"), serialized); + fs.writeFileSync(path.join(signalsDir, "mini-2026-05-06.jsonl"), serialized); + + const results = await searchMemory(memoryDir, "legitimate duplicate needle", { limit: 10 }); + + assert.equal(results.length, 6, "one event retry copy is suppressed without collapsing canonical duplicates"); + assert.equal(results.filter(({ source }) => source === "memory/events.jsonl").length, 2); + assert.deepEqual( + results.filter(({ source }) => source.includes("events-archive.00000")).map(({ source }) => source).sort(), + [ + "memory/events-archive.000001.jsonl", + "memory/events-archive.000002.jsonl" + ] + ); + assert.deepEqual( + results.filter(({ source }) => source.startsWith("memory/signals/")).map(({ source }) => source).sort(), + [ + "memory/signals/laptop-2026-05-06.jsonl", + "memory/signals/mini-2026-05-06.jsonl" + ] + ); + assert.equal(results.some(({ source }) => source === "memory/signals-archive.jsonl"), false); +}); diff --git a/tests/core/owned-state.test.mjs b/tests/core/owned-state.test.mjs new file mode 100644 index 00000000..1a87d62b --- /dev/null +++ b/tests/core/owned-state.test.mjs @@ -0,0 +1,124 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import assert from "node:assert/strict"; +import { + assertOwnedFileStats, + recoverOwnedFileExclusivePublication +} from "../../packages/core/src/owned-state.mjs"; + +function fileStats({ nlink, uid = -1, mode = 0 } = {}) { + return { + nlink, + uid, + mode, + isFile: () => true, + isSymbolicLink: () => false + }; +} + +test("Windows owned files require exactly one link without relying on POSIX ownership metadata", () => { + assert.doesNotThrow(() => assertOwnedFileStats( + fileStats({ nlink: 1 }), + 0o600, + { platform: "win32" } + )); +}); + +test("Windows owned files reject two or more links", () => { + for (const nlink of [2, 3]) { + assert.throws( + () => assertOwnedFileStats(fileStats({ nlink }), 0o600, { platform: "win32" }), + (error) => error?.code === "DOTAIOS_OWNED_STATE_INVALID" + ); + } +}); + +test("POSIX owned files accept bigint filesystem stats", () => { + assert.doesNotThrow(() => assertOwnedFileStats( + fileStats({ + nlink: 1n, + uid: BigInt(typeof process.getuid === "function" ? process.getuid() : -1), + mode: 0o100600n + }), + 0o600, + { platform: "linux" } + )); +}); + +test("Windows recovery accepts exactly the two links created by exclusive publication", async (t) => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "dotaios-owned-state-test-")); + t.after(() => fs.rmSync(directory, { recursive: true, force: true })); + const target = path.join(directory, "receipt.json"); + const temporary = path.join(directory, ".receipt.json.00000000-0000-4000-8000-000000000001.tmp"); + fs.writeFileSync(target, "owned bytes\n", { mode: 0o644 }); + fs.chmodSync(target, 0o644); + fs.linkSync(target, temporary); + + assert.equal(await recoverOwnedFileExclusivePublication(target, { platform: "win32" }), true); + assert.equal(fs.existsSync(temporary), false); + assert.equal(fs.readFileSync(target, "utf8"), "owned bytes\n"); + assert.equal(fs.lstatSync(target).nlink, 1); +}); + +test("Windows recovery rejects an ordinary one-link owned file", async (t) => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "dotaios-owned-state-test-")); + t.after(() => fs.rmSync(directory, { recursive: true, force: true })); + const target = path.join(directory, "receipt.json"); + fs.writeFileSync(target, "ordinary bytes\n", { mode: 0o644 }); + fs.chmodSync(target, 0o644); + + assert.equal(await recoverOwnedFileExclusivePublication(target, { platform: "win32" }), false); + assert.equal(fs.readFileSync(target, "utf8"), "ordinary bytes\n"); + assert.equal(fs.lstatSync(target).nlink, 1); +}); + +test("Windows recovery rejects excess links without unlinking the publication temporary", async (t) => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "dotaios-owned-state-test-")); + t.after(() => fs.rmSync(directory, { recursive: true, force: true })); + const target = path.join(directory, "receipt.json"); + const temporary = path.join(directory, ".receipt.json.00000000-0000-4000-8000-000000000002.tmp"); + const excess = path.join(directory, "foreign-hardlink"); + fs.writeFileSync(target, "owned bytes\n", { mode: 0o644 }); + fs.chmodSync(target, 0o644); + fs.linkSync(target, temporary); + fs.linkSync(target, excess); + + assert.equal(await recoverOwnedFileExclusivePublication(target, { platform: "win32" }), false); + assert.equal(fs.existsSync(temporary), true); + assert.equal(fs.existsSync(excess), true); + assert.equal(fs.lstatSync(target).nlink, 3); +}); + +test("Windows recovery leaves two UUID-shaped publication temporaries untouched", async (t) => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "dotaios-owned-state-test-")); + t.after(() => fs.rmSync(directory, { recursive: true, force: true })); + const target = path.join(directory, "receipt.json"); + const first = path.join(directory, ".receipt.json.00000000-0000-4000-8000-000000000003.tmp"); + const second = path.join(directory, ".receipt.json.00000000-0000-4000-8000-000000000004.tmp"); + fs.writeFileSync(target, "owned bytes\n", { mode: 0o644 }); + fs.linkSync(target, first); + fs.linkSync(target, second); + + assert.equal(await recoverOwnedFileExclusivePublication(target, { platform: "win32" }), false); + assert.equal(fs.existsSync(first), true); + assert.equal(fs.existsSync(second), true); + assert.equal(fs.lstatSync(target).nlink, 3); +}); + +test("Windows recovery ignores a non-UUID publication temporary", async (t) => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "dotaios-owned-state-test-")); + t.after(() => fs.rmSync(directory, { recursive: true, force: true })); + const target = path.join(directory, "receipt.json"); + const temporary = path.join(directory, ".receipt.json.00000000-0000-4000-8000-000000000005.tmp"); + const nonUuid = path.join(directory, ".receipt.json.not-a-uuid.tmp"); + fs.writeFileSync(target, "owned bytes\n", { mode: 0o644 }); + fs.linkSync(target, temporary); + fs.writeFileSync(nonUuid, "unrelated bytes\n"); + + assert.equal(await recoverOwnedFileExclusivePublication(target, { platform: "win32" }), true); + assert.equal(fs.existsSync(temporary), false); + assert.equal(fs.readFileSync(nonUuid, "utf8"), "unrelated bytes\n"); + assert.equal(fs.lstatSync(target).nlink, 1); +}); diff --git a/tests/core/search-ranking.test.mjs b/tests/core/search-ranking.test.mjs index e8bd7228..ec9077e6 100644 --- a/tests/core/search-ranking.test.mjs +++ b/tests/core/search-ranking.test.mjs @@ -3,7 +3,8 @@ import os from "node:os"; import path from "node:path"; import test from "node:test"; import assert from "node:assert/strict"; -import { searchMemoryDir, searchMarkdownDir } from "../../packages/core/src/search.mjs"; +import { createEvidenceReader } from "../../packages/core/src/evidence-reader.mjs"; +import { searchAios, searchMemoryDir, searchMarkdownDir } from "../../packages/core/src/search.mjs"; function tmpDir() { return fs.mkdtempSync(path.join(os.tmpdir(), "dotaios-rank-test-")); @@ -21,6 +22,66 @@ function writeEvents(memoryDir, entries) { ); } +function genericContainedCorpusReader(roots) { + const baseReader = createEvidenceReader({ roots: Array.isArray(roots) ? roots : [roots] }); + const genericScopeReader = (scopeReader) => ({ + ...scopeReader, + async prepareTextCorpus(transactionRoot, directoryPath, options) { + return Object.freeze({ + kind: "generic-contained-corpus", + root: transactionRoot, + options: Object.freeze({ ...options }), + files: await scopeReader.listFiles(transactionRoot, directoryPath, options) + }); + }, + async withPreparedTextCorpus(prepared, callback) { + if (prepared?.kind !== "generic-contained-corpus") { + return scopeReader.withPreparedTextCorpus(prepared, callback); + } + return callback(Object.freeze({ + async mapFiles(mapper) { + return Promise.all(prepared.files.map(async (filePath) => { + const observed = await scopeReader.readText(prepared.root, filePath, { returnSnapshot: true }); + return mapper(Object.freeze({ + filePath, + content: observed.content, + mtimeMs: observed.stats.mtimeMs + })); + })); + } + })); + } + }); + const reader = { + ...baseReader, + async withTextCorpus(transactionRoot, directoryPath, options, callback) { + const files = await baseReader.listFiles(transactionRoot, directoryPath, options); + return callback(Object.freeze({ + async mapFiles(mapper) { + return Promise.all(files.map(async (filePath) => { + const observed = await baseReader.readText(transactionRoot, filePath, { returnSnapshot: true }); + return mapper(Object.freeze({ + filePath, + content: observed.content, + mtimeMs: observed.stats.mtimeMs + })); + })); + } + })); + }, + async withScopePreflight(scopes, inspect, discover, execute, callback) { + return baseReader.withScopePreflight( + scopes, + (scope, scopeReader) => inspect(scope, genericScopeReader(scopeReader)), + (scope, scopeReader, prepared) => discover(scope, genericScopeReader(scopeReader), prepared), + (scope, scopeReader, prepared) => execute(scope, genericScopeReader(scopeReader), prepared), + callback + ); + } + }; + return reader; +} + // (a) Recency decay: a fresh hit must beat a stale hit of comparable lexical // relevance — and raw term frequency must not drown recency. @@ -140,3 +201,87 @@ test("ranking is deterministic and leaks no internal fields", async () => { assert.ok(!("__rank" in result), "internal rank must not leak into consumer shape"); } }); + +test("bulk search is exactly equal to the generic contained-read oracle across canonical scopes", async () => { + const root = tmpDir(); + const externalVault = tmpDir(); + const write = (relativePath, content) => { + const filePath = path.join(root, relativePath); + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync(filePath, content); + return filePath; + }; + + const old = new Date(Date.now() - 90 * 86_400_000); + const stale = write("context/a-stale.md", "# Launch plan\n\nExact launch plan body.\n"); + fs.utimesSync(stale, old, old); + write("context/b-punctuation.md", "# Notes\n\nThe launch, plan uses punctuation.\n"); + write("context/c-lines.md", "# Notes\n\nlaunch\nplan on the next line\n"); + write("context/d-description.md", "---\ndescription: Launch plan in frontmatter\n---\n# Other\n"); + write("context/launch-plan/e-path.md", "# Other\n\nlaunch plan in a boosted path\n"); + write("context/f-inflection.md", "# Delivery\n\nThe team launched plans yesterday.\n"); + write("context/g-substring.md", "# Substring\n\nA prelaunch planner keeps the substring behavior.\n"); + write("context/z-tie.md", "# Notes\n\nExact launch plan body.\n"); + write("context/.env.md", "launch plan secret\n"); + + write("memory/events.jsonl", `${JSON.stringify({ ts: isoAgo(1), type: "note", summary: "launch plan stream" })}\n`); + write("memory/daily/2026-08-13.md", "# Daily\n\nlaunch plan daily note\n"); + write("memory/inbox/capture.md", "# Inbox\n\nlaunch plan inbox note\n"); + write("plugins/demo/manifest.json", "{\n \"description\": \"launch plan plugin\"\n}\n"); + write("plugins/demo/package.json", "{\n \"description\": \"launch plan must stay omitted\"\n}\n"); + write("projects/acme/README.md", "---\nid: project-acme-001\nproject: acme\n---\n# Acme\n\nlaunch plan selected project\n"); + write("projects/other/README.md", "---\nid: project-other-002\nproject: other\n---\n# Other\n\nlaunch plan unselected project\n"); + fs.writeFileSync(path.join(externalVault, "external.md"), "# External\n\nlaunch plan external vault\n"); + + const safeReader = createEvidenceReader({ roots: [root, externalVault] }); + const genericReader = genericContainedCorpusReader([root, externalVault]); + const requests = [ + { aiosPath: root, vaultPath: externalVault, query: "launch plan", scope: "all", projectSelector: "acme" }, + { aiosPath: root, vaultPath: externalVault, query: "launch plan", scope: "context" }, + { aiosPath: root, vaultPath: externalVault, query: "launch plan", scope: "memory" }, + { aiosPath: root, vaultPath: externalVault, query: "launch plan", scope: "plugins" }, + { aiosPath: root, vaultPath: externalVault, query: "launch plan", scope: "projects", projectSelector: "project-acme-001" }, + { aiosPath: root, vaultPath: externalVault, query: "launch plan", scope: "vault" } + ]; + + for (const request of requests) { + const safe = await searchAios({ ...request, evidenceReader: safeReader }); + const generic = await searchAios({ ...request, evidenceReader: genericReader }); + assert.deepEqual(safe, generic, `safe transaction changed ${request.scope} output`); + } + + const context = await searchMarkdownDir(path.join(root, "context"), "launch plan", { + sourcePrefix: "context", + reader: createEvidenceReader({ roots: [root] }), + root + }); + assert.deepEqual(context.map(({ file }) => file), [ + "d-description.md", + "launch-plan/e-path.md", + "g-substring.md", + "z-tie.md", + "a-stale.md", + "b-punctuation.md", + "c-lines.md", + "f-inflection.md" + ]); + assert.deepEqual(context.find(({ file }) => file === "c-lines.md").matches, [ + { line: 2, lineEnd: 4, content: "launch / plan on the next line", match: "partial", area: "body" }, + { line: 3, lineEnd: 5, content: "launch / plan on the next line", match: "partial", area: "body" } + ]); + + const all = await searchAios({ + aiosPath: root, + vaultPath: externalVault, + query: "launch plan", + projectSelector: "acme", + evidenceReader: createEvidenceReader({ roots: [root, externalVault] }) + }); + const serialized = JSON.stringify(all); + assert.match(serialized, /memory\/daily\/2026-08-13\.md/); + assert.match(serialized, /memory\/inbox\/capture\.md/); + assert.match(serialized, /plugins\/demo\/manifest\.json/); + assert.match(serialized, /projects\/acme\/README\.md/); + assert.match(serialized, /vault\/external\.md/); + assert.doesNotMatch(serialized, /package\.json|projects\/other|\.env\.md/); +}); diff --git a/tests/core/search-safety.test.mjs b/tests/core/search-safety.test.mjs index 5fe07f77..89284d77 100644 --- a/tests/core/search-safety.test.mjs +++ b/tests/core/search-safety.test.mjs @@ -103,13 +103,182 @@ test("search rejects invalid UTF-8 without replacing bytes", async () => { assert.deepEqual(fs.readFileSync(filePath), bytes); }); +test("memory search rejects a rotation that changes the observed archive generation", async () => { + const root = tmpDir(); + const memoryDir = path.join(root, "memory"); + fs.mkdirSync(memoryDir); + fs.writeFileSync( + path.join(memoryDir, "events-archive.000001.jsonl"), + `${JSON.stringify({ ts: "2026-05-01T12:00:00.000Z", type: "note", summary: "ARCHIVE_RACE_CANARY" })}\n`, + { mode: 0o600 } + ); + fs.writeFileSync(path.join(memoryDir, "events-archive.jsonl"), "", { mode: 0o600 }); + const baseReader = createEvidenceReader({ roots: [root] }); + let oldGenerationResult = null; + const reader = { + ...baseReader, + withScopePreflight(scopes, inspect, discover, execute, callback) { + return baseReader.withScopePreflight(scopes, inspect, discover, execute, async (transaction) => { + const result = await callback(transaction); + oldGenerationResult = result; + fs.writeFileSync( + path.join(memoryDir, "events-archive.000002.jsonl"), + `${JSON.stringify({ ts: "2026-05-02T12:00:00.000Z", type: "note", summary: "late shard" })}\n`, + { mode: 0o600 } + ); + return result; + }); + } + }; + + await assert.rejects( + () => searchAios({ aiosPath: root, query: "ARCHIVE_RACE_CANARY", scope: "memory", evidenceReader: reader }), + (error) => error?.code === "DOTAIOS_EVIDENCE_CHANGED" + ); + assert.match(JSON.stringify(oldGenerationResult), /ARCHIVE_RACE_CANARY/); +}); + +test("memory search revalidates retained JSONL after ranking and before publishing", async () => { + const root = tmpDir(); + const memoryDir = path.join(root, "memory"); + const eventsPath = path.join(memoryDir, "events.jsonl"); + fs.mkdirSync(memoryDir); + fs.writeFileSync(eventsPath, '{"summary":"RETAINED_JSONL_CANARY"}\n'); + const baseReader = createEvidenceReader({ roots: [root] }); + const reader = { + ...baseReader, + withScopePreflight(scopes, inspect, discover, execute, callback) { + return baseReader.withScopePreflight(scopes, inspect, discover, execute, async (transaction) => { + const result = await callback(transaction); + fs.writeFileSync(eventsPath, '{"summary":"RETAINED_JSONL_CHANGED"}\n'); + return result; + }); + } + }; + + await assert.rejects( + () => searchAios({ + aiosPath: root, + query: "RETAINED_JSONL_CANARY", + scope: "memory", + evidenceReader: reader + }), + (error) => error?.code === "DOTAIOS_EVIDENCE_CHANGED" + ); +}); + +test("memory search fails closed on a linked numbered archive shard", async () => { + const parent = tmpDir(); + const root = path.join(parent, "aios"); + const memoryDir = path.join(root, "memory"); + const outside = path.join(parent, "outside.jsonl"); + fs.mkdirSync(memoryDir, { recursive: true }); + fs.writeFileSync(outside, '{"summary":"LINKED_ARCHIVE_CANARY"}\n'); + fs.symlinkSync(outside, path.join(memoryDir, "events-archive.000001.jsonl")); + + await assert.rejects( + () => searchAios({ aiosPath: root, query: "LINKED_ARCHIVE_CANARY", scope: "memory" }), + (error) => error?.code === "DOTAIOS_EVIDENCE_PATH_UNSAFE" + ); +}); + +test("search ranks inside the corpus transaction and publishes only after final validation", async () => { + const root = tmpDir(); + fs.writeFileSync(path.join(root, "b-body.md"), "# Body\n\nTRANSACTION_SEARCH_CANARY\n"); + fs.writeFileSync(path.join(root, "a-heading.md"), "# TRANSACTION_SEARCH_CANARY\n\nPlain body.\n"); + const baseReader = createEvidenceReader({ roots: [root] }); + let transactionCalls = 0; + let callbackResult; + let releaseFinalValidation; + const finalValidationGate = new Promise((resolve) => { + releaseFinalValidation = resolve; + }); + let reportCallbackComplete; + const callbackComplete = new Promise((resolve) => { + reportCallbackComplete = resolve; + }); + const reader = { + ...baseReader, + async withTextCorpus(transactionRoot, directoryPath, options, callback) { + transactionCalls += 1; + return baseReader.withTextCorpus(transactionRoot, directoryPath, options, async (transaction) => { + callbackResult = await callback(transaction); + reportCallbackComplete(); + await finalValidationGate; + return callbackResult; + }); + } + }; + + const pending = searchMarkdownDir(root, "TRANSACTION_SEARCH_CANARY", { reader, root }); + const firstCompleted = await Promise.race([ + callbackComplete.then(() => "callback"), + pending.then(() => "search") + ]); + + assert.equal(firstCompleted, "callback", "ranking must complete inside the transaction callback"); + assert.deepEqual(callbackResult.map(({ file }) => file), ["a-heading.md", "b-body.md"]); + let published = false; + pending.then(() => { + published = true; + }); + await Promise.resolve(); + assert.equal(published, false, "callback results must remain private until final validation completes"); + + releaseFinalValidation(); + const results = await pending; + + assert.deepEqual(results, callbackResult); + assert.equal(transactionCalls, 1); +}); + +test("search rejects ranked results when final corpus validation observes a changed generation", async () => { + const root = tmpDir(); + fs.writeFileSync(path.join(root, "note.md"), "# Existing\n\nFINAL_VALIDATION_CANARY\n"); + const baseReader = createEvidenceReader({ roots: [root] }); + let rankedInsideCallback = null; + const reader = { + ...baseReader, + withTextCorpus(transactionRoot, directoryPath, options, callback) { + return baseReader.withTextCorpus(transactionRoot, directoryPath, options, async (transaction) => { + rankedInsideCallback = await callback(transaction); + fs.writeFileSync(path.join(root, "late.md"), "# Late generation\n"); + return rankedInsideCallback; + }); + } + }; + + await assert.rejects( + () => searchMarkdownDir(root, "FINAL_VALIDATION_CANARY", { reader, root }), + (error) => error?.code === "DOTAIOS_EVIDENCE_CHANGED" + ); + assert.deepEqual(rankedInsideCallback.map(({ file }) => file), ["note.md"]); +}); + +test("request-scoped search observes added, modified, and deleted files on the next request", async () => { + const root = tmpDir(); + const firstPath = path.join(root, "first.md"); + const secondPath = path.join(root, "second.md"); + fs.writeFileSync(firstPath, "# First\n\nNEXT_REQUEST_CANARY\n"); + const reader = createEvidenceReader({ roots: [root] }); + const search = () => searchMarkdownDir(root, "NEXT_REQUEST_CANARY", { reader, root }); + + assert.deepEqual((await search()).map(({ file }) => file), ["first.md"]); + fs.writeFileSync(secondPath, "# Second\n\nNEXT_REQUEST_CANARY\n"); + assert.deepEqual((await search()).map(({ file }) => file), ["first.md", "second.md"]); + fs.writeFileSync(firstPath, "# First\n\nChanged content.\n"); + assert.deepEqual((await search()).map(({ file }) => file), ["second.md"]); + fs.unlinkSync(secondPath); + assert.deepEqual(await search(), []); +}); + // This asserted the budget by its number — "the 513th file" — so it passed only // while the default was the bounded projection's 512, which made every search on // a real folder fail closed. The property worth keeping is that an exhausted // budget still refuses rather than quietly returning a partial corpus; the // number it happens to be set to is not that property. The default's real size // is covered in tests/core/search_corpus_scale.test.mjs. -test("search fails closed when its read budget is genuinely exhausted", async () => { +test("scope search reports its whole corpus omitted when its read budget is exhausted", async () => { const root = tmpDir(); const context = path.join(root, "context"); fs.mkdirSync(context); @@ -121,10 +290,282 @@ test("search fails closed when its read budget is genuinely exhausted", async () limits: { maxBytes: 4096, maxFiles: 2, maxEntries: 4, maxFileBytes: 4096 } }); - await assert.rejects( - () => searchAios({ aiosPath: root, query: "needle", scope: "context", evidenceReader: reader }), - (error) => error?.code === "DOTAIOS_EVIDENCE_BUDGET_EXCEEDED" - ); + const groups = await searchAios({ aiosPath: root, query: "needle", scope: "context", evidenceReader: reader }); + + assert.deepEqual([...groups], []); + assert.equal(groups.omissions.length, 1); + assert.equal(groups.omissions[0].scope, "context"); + assert.ok(["file_count_exceeded", "entry_count_exceeded"].includes(groups.omissions[0].reason)); +}); + +test("all-scope search returns unaffected results with a frozen path-free ceiling omission", async () => { + const parent = tmpDir(); + const root = path.join(parent, "aios"); + const vault = path.join(parent, "vault"); + fs.mkdirSync(path.join(root, "context"), { recursive: true }); + fs.mkdirSync(vault); + fs.writeFileSync(path.join(root, "context", "safe.md"), "# Safe\n\nSAFE_PARTIAL_SEARCH_CANARY\n"); + fs.writeFileSync(path.join(vault, "oversized.md"), "x".repeat(65)); + const reader = createEvidenceReader({ + roots: [root, vault], + limits: { maxBytes: 1024, maxFiles: 100, maxEntries: 100, maxFileBytes: 64, maxDirectoryEntries: 100 } + }); + + const groups = await searchAios({ + aiosPath: root, + vaultPath: vault, + query: "SAFE_PARTIAL_SEARCH_CANARY", + scope: "all", + evidenceReader: reader + }); + + assert.match(JSON.stringify(groups), /SAFE_PARTIAL_SEARCH_CANARY/); + assert.deepEqual(groups.omissions, [{ + scope: "vault", + reason: "file_too_large", + observed: { files: 0, bytes: 0, entries: 1 }, + inspection: "not_searched", + recovery: { + code: "split_or_move_file", + message: "Split the oversized file, or move it outside this search scope." + } + }]); + assert.equal(Object.isFrozen(groups.omissions), true); + assert.equal(Object.isFrozen(groups.omissions[0]), true); + assert.equal(Object.isFrozen(groups.omissions[0].observed), true); + assert.equal(Object.isFrozen(groups.omissions[0].recovery), true); + assert.doesNotMatch(JSON.stringify(groups.omissions), new RegExp(parent.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"))); +}); + +test("all-scope preflight keeps aggregate physical work inside one request budget", async () => { + const root = tmpDir(); + const context = path.join(root, "context"); + const vault = path.join(root, "vault"); + fs.mkdirSync(context); + fs.mkdirSync(vault); + for (const directory of [context, vault]) { + fs.writeFileSync(path.join(directory, "one.md"), "needle!\n"); + fs.writeFileSync(path.join(directory, "two.md"), "needle!\n"); + } + const measuredDirectories = new Set([path.resolve(context), path.resolve(vault)]); + const limits = { + maxBytes: 16, + maxFiles: 2, + maxEntries: 2, + maxFileBytes: 8, + maxDirectoryEntries: 10 + }; + + for (const delayedDirectory of measuredDirectories) { + const physical = { bytes: 0, files: 0, entries: 0 }; + const filesystem = new Proxy(fsp, { + get(target, property) { + if (property === "open") { + return async (filePath, ...args) => { + const resolved = path.resolve(String(filePath)); + if (path.dirname(resolved) === delayedDirectory) { + await new Promise((resolve) => setTimeout(resolve, 5)); + } + const handle = await fsp.open(filePath, ...args); + if (!measuredDirectories.has(path.dirname(resolved))) return handle; + physical.files += 1; + return Object.create(handle, { + read: { value: async (...readArgs) => { + const result = await handle.read(...readArgs); + physical.bytes += result.bytesRead; + return result; + } } + }); + }; + } + if (property === "opendir") { + return async (directoryPath, ...args) => { + const directory = await fsp.opendir(directoryPath, ...args); + if (!measuredDirectories.has(path.resolve(String(directoryPath)))) return directory; + return new Proxy(directory, { + get(dirTarget, dirProperty) { + if (dirProperty === "read") { + return async (...readArgs) => { + const entry = await directory.read(...readArgs); + if (entry) physical.entries += 1; + return entry; + }; + } + const value = Reflect.get(dirTarget, dirProperty, dirTarget); + return typeof value === "function" ? value.bind(dirTarget) : value; + } + }); + }; + } + const value = Reflect.get(target, property, target); + return typeof value === "function" ? value.bind(target) : value; + } + }); + + const groups = await searchAios({ + aiosPath: root, + query: "needle", + evidenceReader: createEvidenceReader({ roots: [root], filesystem, limits }) + }); + + assert.ok(groups.some(({ results }) => results.length > 0)); + assert.ok(physical.files > 0 && physical.bytes > 0, JSON.stringify({ delayedDirectory, physical })); + assert.ok(physical.bytes <= limits.maxBytes, JSON.stringify({ delayedDirectory, physical })); + assert.ok(physical.files <= limits.maxFiles, JSON.stringify({ delayedDirectory, physical })); + assert.ok(physical.entries <= limits.maxEntries, JSON.stringify({ delayedDirectory, physical })); + } +}); + +test("session discovery retains catalog and body bytes without rereading before ranking", async () => { + const root = tmpDir(); + const sessionsDir = path.join(root, "memory", "sessions"); + const datedDir = path.join(sessionsDir, "2026-08-13"); + const bodyPath = path.join(datedDir, "session.md"); + const indexPath = path.join(sessionsDir, "index.jsonl"); + fs.mkdirSync(datedDir, { recursive: true }); + fs.writeFileSync(bodyPath, "---\ntitle: Session\n---\n\nSESSION_DISCOVERY_CANARY\n"); + fs.writeFileSync(indexPath, `${JSON.stringify({ + session_id: "session-001", + agent: "codex", + captured_at: "2026-08-13T10:00:00.000Z", + title: "Unrelated title", + path: "memory/sessions/2026-08-13/session.md" + })}\n`); + const opens = new Map(); + const filesystem = Object.create(fsp); + filesystem.open = async (filePath, ...args) => { + const resolved = path.resolve(String(filePath)); + opens.set(resolved, (opens.get(resolved) || 0) + 1); + return fsp.open(filePath, ...args); + }; + + const [group] = await searchAios({ + aiosPath: root, + query: "SESSION_DISCOVERY_CANARY", + scope: "sessions", + evidenceReader: createEvidenceReader({ roots: [root], filesystem }) + }); + + assert.equal(group.results[0].session_id, "session-001"); + assert.equal(opens.get(path.resolve(indexPath)), 1); + assert.equal(opens.get(path.resolve(bodyPath)), 1); +}); + +test("session body planning mirrors reverse query limit behavior and charges unique paths", async (t) => { + await t.test("a title hit does not plan an unnecessary body", async () => { + const root = tmpDir(); + const sessionsDir = path.join(root, "memory", "sessions"); + fs.mkdirSync(sessionsDir, { recursive: true }); + fs.writeFileSync(path.join(sessionsDir, "body.md"), "body does not match\n"); + fs.writeFileSync(path.join(sessionsDir, "index.jsonl"), `${JSON.stringify({ + session_id: "title-hit", + agent: "codex", + captured_at: "2026-08-13T10:00:00.000Z", + title: "EXACT_TITLE_CANARY", + path: "memory/sessions/body.md" + })}\n`); + + const [group] = await searchAios({ + aiosPath: root, + query: "EXACT_TITLE_CANARY", + scope: "sessions", + evidenceReader: createEvidenceReader({ + roots: [root], + limits: { maxBytes: 4096, maxFiles: 1, maxEntries: 10, maxFileBytes: 4096, maxDirectoryEntries: 10 } + }) + }); + + assert.equal(group.results[0].session_id, "title-hit"); + }); + + await t.test("duplicate index rows plan and read their shared body once", async () => { + const root = tmpDir(); + const sessionsDir = path.join(root, "memory", "sessions"); + fs.mkdirSync(sessionsDir, { recursive: true }); + const bodyPath = path.join(sessionsDir, "body.md"); + fs.writeFileSync(bodyPath, "UNIQUE_BODY_CANARY\n"); + const rows = ["older", "newer"].map((sessionId, index) => JSON.stringify({ + session_id: sessionId, + agent: "codex", + captured_at: `2026-08-13T1${index}:00:00.000Z`, + title: "unrelated", + path: "memory/sessions/body.md" + })); + fs.writeFileSync(path.join(sessionsDir, "index.jsonl"), `${rows.join("\n")}\n`); + const opens = new Map(); + const filesystem = Object.create(fsp); + filesystem.open = async (filePath, ...args) => { + const resolved = path.resolve(String(filePath)); + opens.set(resolved, (opens.get(resolved) || 0) + 1); + return fsp.open(filePath, ...args); + }; + + const [group] = await searchAios({ + aiosPath: root, + query: "UNIQUE_BODY_CANARY", + scope: "sessions", + limit: 1, + evidenceReader: createEvidenceReader({ + roots: [root], + filesystem, + limits: { maxBytes: 4096, maxFiles: 2, maxEntries: 10, maxFileBytes: 4096, maxDirectoryEntries: 10 } + }) + }); + + assert.equal(group.results[0].session_id, "newer"); + assert.equal(opens.get(path.resolve(bodyPath)), 1); + }); +}); + +test("scope search distinguishes every skippable ceiling without catching integrity failures", async (t) => { + const cases = [ + { + name: "directory entries", + reason: "directory_entries_exceeded", + limits: { maxBytes: 1024, maxFiles: 10, maxEntries: 10, maxFileBytes: 1024, maxDirectoryEntries: 1 }, + files: [["one.md", "one\n"], ["two.md", "two\n"]] + }, + { + name: "aggregate bytes", + reason: "aggregate_bytes_exceeded", + limits: { maxBytes: 7, maxFiles: 10, maxEntries: 10, maxFileBytes: 10, maxDirectoryEntries: 10 }, + files: [["one.md", "one\n"], ["two.md", "two\n"]] + }, + { + name: "file count", + reason: "file_count_exceeded", + limits: { maxBytes: 1024, maxFiles: 1, maxEntries: 10, maxFileBytes: 1024, maxDirectoryEntries: 10 }, + files: [["one.md", "one\n"], ["two.md", "two\n"]] + }, + { + name: "entry count", + reason: "entry_count_exceeded", + limits: { maxBytes: 1024, maxFiles: 10, maxEntries: 1, maxFileBytes: 1024, maxDirectoryEntries: 10 }, + files: [["one.md", "one\n"], ["two.md", "two\n"]] + } + ]; + + for (const fixture of cases) { + await t.test(fixture.name, async () => { + const root = tmpDir(); + const context = path.join(root, "context"); + fs.mkdirSync(context); + for (const [name, content] of fixture.files) fs.writeFileSync(path.join(context, name), content); + const groups = await searchAios({ + aiosPath: root, + query: "missing", + scope: "context", + evidenceReader: createEvidenceReader({ roots: [root], limits: fixture.limits }) + }); + + assert.deepEqual([...groups], []); + assert.equal(groups.omissions[0].reason, fixture.reason); + assert.equal( + groups.omissions[0].inspection, + fixture.reason === "directory_entries_exceeded" ? "partially_enumerated" : "not_searched" + ); + }); + } }); test("search rejects a special eligible file before opening it", { diff --git a/tests/core/search_benchmark_manifest.test.mjs b/tests/core/search_benchmark_manifest.test.mjs new file mode 100644 index 00000000..1810e43a --- /dev/null +++ b/tests/core/search_benchmark_manifest.test.mjs @@ -0,0 +1,509 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { spawnSync } from "node:child_process"; +import test from "node:test"; +import assert from "node:assert/strict"; +import { fileURLToPath } from "node:url"; + +import { + assertExactResults, + createBenchmarkReport, + generateFixture, + loadManifest, + manifestReceipt, + runPublicSearchBenchmarkSample, + runSafeCorpusReadBenchmarkSample, + assertPublicSearchOperationGate, + runUnsafeBenchmarkOnlyRawSearchSample +} from "../../scripts/bench-search.mjs"; + +const repoRoot = fileURLToPath(new URL("../..", import.meta.url)); +const manifestPath = path.join(repoRoot, "benchmarks", "search", "manifest.json"); +const benchmarkReceiptPaths = [ + path.join(repoRoot, "docs", "benchmarks", "2026-08-13-search-baseline.md"), + path.join(repoRoot, "docs", "benchmarks", "2026-08-13-search-optimized.md"), + path.join(repoRoot, "docs", "benchmarks", "2026-08-13-search-final.md") +]; +const finalReportsDirectory = path.join(repoRoot, "docs", "benchmarks", "reports"); + +function declaredManifestReceipt(receiptPath, receipt) { + const declarations = [...receipt.matchAll(/^\- Manifest SHA-256: `([^`]+)`\s*$/gmi)]; + assert.equal( + declarations.length, + 1, + `${path.relative(repoRoot, receiptPath)} must declare exactly one Manifest SHA-256.` + ); + + const receiptHash = declarations[0][1]; + assert.match( + receiptHash, + /^[a-f0-9]{64}$/, + `${path.relative(repoRoot, receiptPath)} must declare a lowercase SHA-256 digest.` + ); + return receiptHash; +} + +function expectedReportResults(query, selection, manifest) { + let indices; + if (query.expectation.kind === "none") { + indices = []; + } else if (query.expectation.kind === "fixed-indices") { + indices = query.expectation.fileIndices; + } else if (query.expectation.kind === "modulo") { + indices = []; + for ( + let index = query.expectation.remainder; + index < selection.fileCount; + index += query.expectation.modulo + ) indices.push(index); + } else { + throw new Error(`Unsupported query expectation kind: ${query.expectation.kind}.`); + } + const paths = indices.map((index) => { + const file = `note-${String(index).padStart(5, "0")}.md`; + if (selection.layout === "shallow") { + return `vault/bucket-${String(index % manifest.corpus.layouts.shallow.bucketCount).padStart(2, "0")}/${file}`; + } + const branching = manifest.corpus.layouts.nested.branchingFactor; + return `vault/branch-${String(Math.floor(index / (branching ** 2)) % branching).padStart(2, "0")}` + + `/branch-${String(Math.floor(index / branching) % branching).padStart(2, "0")}` + + `/branch-${String(index % branching).padStart(2, "0")}/${file}`; + }).sort(); + return paths.slice(0, query.expectation.resultLimit ?? paths.length); +} + +test("the same manifest and seed produce the same fixture inventory and controlled order", async (t) => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "dotaios-search-benchmark-test-")); + t.after(() => fs.rm(root, { recursive: true, force: true })); + const manifest = await loadManifest(manifestPath); + const selection = { + fileCount: Math.min(...manifest.corpus.fileCounts), + layout: "shallow", + distribution: "prose" + }; + + const first = await generateFixture({ manifest, destination: path.join(root, "first"), selection }); + const second = await generateFixture({ manifest, destination: path.join(root, "second"), selection }); + + assert.equal(first.inventorySha256, second.inventorySha256); + assert.deepEqual(first.controlledResults, second.controlledResults); + assert.deepEqual(first.controlledResults["low-hit"], second.controlledResults["low-hit"]); + assert.equal(manifestReceipt(manifest), first.manifestSha256); + assert.equal(first.fileCount, selection.fileCount); + assert.ok(first.controlledResults["low-hit"].length > 0); +}); + +test("the safe benchmark sample measures complete default all-scope searchAios", async (t) => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "dotaios-public-search-benchmark-")); + t.after(() => fs.rm(root, { recursive: true, force: true })); + const manifest = await loadManifest(manifestPath); + assert.deepEqual(manifest.corpus.fileCounts, [500, 2500, 10000], "the formal matrix stays fixed"); + const fixtureRoot = path.join(root, "fixture"); + const fixtureReceipt = await generateFixture({ + manifest, + destination: fixtureRoot, + selection: { fileCount: 500, layout: "shallow", distribution: "prose" } + }); + const query = manifest.queries.find(({ id }) => id === "low-hit"); + const highHitQuery = manifest.queries.find(({ id }) => id === "high-hit"); + + const sample = await runPublicSearchBenchmarkSample({ manifest, fixtureRoot, fixtureReceipt, query }); + const highHitSample = await runPublicSearchBenchmarkSample({ + manifest, + fixtureRoot, + fixtureReceipt, + query: highHitQuery + }); + const safeControl = await runSafeCorpusReadBenchmarkSample({ manifest, fixtureRoot, fixtureReceipt }); + + assert.deepEqual(sample.exactResults, fixtureReceipt.controlledResults[query.id]); + assert.ok(sample.exactResults.length > 0, "the controlled low-hit proof cannot be vacuous"); + assert.deepEqual(highHitSample.exactResults, fixtureReceipt.controlledResults[highHitQuery.id]); + assert.ok(highHitSample.exactResults.length > 0, "the controlled high-hit proof cannot be vacuous"); + assert.deepEqual(sample.surface, { + entryPoint: "searchAios", + requestedScope: "all", + completeness: "complete", + omissions: [], + returnedScopes: ["sessions", "context", "memory", "vault", "decisions", "skills", "references", "plugins"] + }); + assert.doesNotThrow(() => assertPublicSearchOperationGate( + [ + { id: query.id, warm: { operations: sample.operations } }, + { id: highHitQuery.id, warm: { operations: highHitSample.operations } } + ], + { warm: { operations: safeControl.operations } } + )); +}); + +test("new benchmark reports are v2 while checked-in v1 receipts remain historical", async () => { + const manifest = await loadManifest(manifestPath); + const operations = { lstat: 10, realpath: 2, open: 1 }; + const report = createBenchmarkReport({ + manifest, + fixtureReceipt: { + inventorySha256: "0".repeat(64), + selection: { fileCount: 500, layout: "shallow", distribution: "prose" } + }, + searches: [{ id: "low-hit", warm: { operations } }], + rawSearchControl: [], + rawReadControl: {}, + safeCorpusReadControl: { warm: { operations } } + }); + + assert.equal(report.schemaVersion, "dotaios-search-benchmark-result/v2"); + assert.deepEqual(report.searchSurface, { + entryPoint: "searchAios", + requestedScope: "all", + completeness: "complete" + }); + assert.equal(report.operationGate.passed, true); +}); + +test("the public operation gate rejects per-file containment multiplication", () => { + const safeControl = { warm: { operations: { lstat: 2_100, realpath: 200, open: 500 } } }; + assert.doesNotThrow(() => assertPublicSearchOperationGate( + [{ id: "optimized", warm: { operations: { lstat: 2_300, realpath: 210, open: 500 } } }], + safeControl + )); + assert.throws( + () => assertPublicSearchOperationGate( + [{ id: "regressed", warm: { operations: { lstat: 20_000, realpath: 9_000, open: 500 } } }], + safeControl + ), + /operation gate/i + ); +}); + +test("the public operation gate rejects malformed counters before arithmetic", () => { + const validSearches = [{ + id: "controlled", + warm: { operations: { lstat: 120, realpath: 30, open: 10 } } + }]; + const validControl = { warm: { operations: { lstat: 100, realpath: 20, open: 10 } } }; + const validAllowance = { lstat: 32, realpath: 16, open: 2 }; + const invalidValues = [undefined, Number.NaN, Number.POSITIVE_INFINITY, -1, 0.5, "1"]; + + for (const counter of ["lstat", "realpath", "open"]) { + for (const location of ["search", "control", "allowance"]) { + for (const invalid of invalidValues) { + const searches = structuredClone(validSearches); + const control = structuredClone(validControl); + const allowance = structuredClone(validAllowance); + const target = location === "search" + ? searches[0].warm.operations + : location === "control" + ? control.warm.operations + : allowance; + target[counter] = invalid; + assert.throws( + () => assertPublicSearchOperationGate(searches, control, allowance), + /non-negative safe integer/i, + `${location}.${counter} must reject ${String(invalid)}` + ); + } + + const searches = structuredClone(validSearches); + const control = structuredClone(validControl); + const allowance = structuredClone(validAllowance); + const target = location === "search" + ? searches[0].warm.operations + : location === "control" + ? control.warm.operations + : allowance; + delete target[counter]; + assert.throws( + () => assertPublicSearchOperationGate(searches, control, allowance), + /non-negative safe integer/i, + `${location}.${counter} must reject a missing value` + ); + } + } +}); + +test("any manifest corpus, query, or protocol change invalidates the receipt", async () => { + const manifest = await loadManifest(manifestPath); + const changedQuery = structuredClone(manifest); + changedQuery.queries[0].text += " changed"; + const changedProtocol = structuredClone(manifest); + changedProtocol.protocol.warmupSamples += 1; + const changedCorpus = structuredClone(manifest); + changedCorpus.corpus.generator.seed += 1; + + const original = manifestReceipt(manifest); + assert.notEqual(manifestReceipt(changedQuery), original); + assert.notEqual(manifestReceipt(changedProtocol), original); + assert.notEqual(manifestReceipt(changedCorpus), original); +}); + +test("checked-in benchmark receipts declare the current manifest receipt", async () => { + const expectedManifestReceipt = manifestReceipt(await loadManifest(manifestPath)); + + for (const receiptPath of benchmarkReceiptPaths) { + const receipt = await fs.readFile(receiptPath, "utf8"); + assert.equal(declaredManifestReceipt(receiptPath, receipt), expectedManifestReceipt, receiptPath); + } +}); + +test("final benchmark reports cover the frozen matrix and retain exact authority", async () => { + const manifest = await loadManifest(manifestPath); + const expectedManifestReceipt = manifestReceipt(manifest); + const expectedSelections = manifest.corpus.fileCounts.flatMap((fileCount) => + manifest.corpus.scenarioMatrix.map(({ layout, distribution }) => + `${fileCount}:${layout}:${distribution}` + ) + ); + const reportNames = (await fs.readdir(finalReportsDirectory)) + .filter((name) => name.startsWith("2026-08-13-") && name.endsWith(".report.json")) + .sort(); + assert.equal(reportNames.length, expectedSelections.length); + + const selections = []; + for (const reportName of reportNames) { + const report = JSON.parse(await fs.readFile(path.join(finalReportsDirectory, reportName), "utf8")); + assert.equal(report.schemaVersion, "dotaios-search-benchmark-result/v1"); + assert.equal(report.manifestSha256, expectedManifestReceipt); + assert.equal(Object.hasOwn(report.runtime, "hostname"), false); + assert.deepEqual(report.protocol, manifest.protocol); + assert.deepEqual(report.searches.map(({ id }) => id), manifest.queries.map(({ id }) => id)); + assert.equal(report.searches.every(({ warm }) => warm.samples === manifest.protocol.measuredSamples), true); + assert.equal(report.rawSearchControl.length, report.searches.length); + for (const result of report.searches) { + assert.equal( + report.rawSearchControl.find(({ id }) => id === result.id)?.outputSha256, + result.outputSha256, + `${reportName}:${result.id} must preserve exact safe/unsafe output parity.` + ); + } + selections.push( + `${report.selection.fileCount}:${report.selection.layout}:${report.selection.distribution}` + ); + } + + assert.deepEqual(selections.sort(), expectedSelections.sort()); +}); + +test("public v2 benchmark reports prove the exact six-cell product-search matrix", async () => { + const manifest = await loadManifest(manifestPath); + const expectedSelections = manifest.corpus.fileCounts.flatMap((fileCount) => + manifest.corpus.scenarioMatrix.map(({ layout, distribution }) => + `${fileCount}:${layout}:${distribution}` + ) + ).sort(); + const reportNames = (await fs.readdir(finalReportsDirectory)) + .filter((name) => name.startsWith("2026-08-14-public-") && name.endsWith(".report.json")) + .sort(); + assert.equal(reportNames.length, 6); + + const selections = []; + for (const reportName of reportNames) { + const report = JSON.parse(await fs.readFile(path.join(finalReportsDirectory, reportName), "utf8")); + assert.equal(report.schemaVersion, "dotaios-search-benchmark-result/v2"); + assert.equal(report.benchmarkId, manifest.benchmarkId); + assert.equal(report.manifestSha256, manifestReceipt(manifest)); + assert.deepEqual(report.protocol, manifest.protocol); + assert.deepEqual(report.searchSurface, { + entryPoint: "searchAios", + requestedScope: "all", + completeness: "complete" + }); + assert.deepEqual(report.searches.map(({ id }) => id), manifest.queries.map(({ id }) => id)); + assert.deepEqual(report.rawSearchControl.map(({ id }) => id), manifest.queries.map(({ id }) => id)); + for (const [index, query] of manifest.queries.entries()) { + const search = report.searches[index]; + const rawControl = report.rawSearchControl[index]; + const expected = expectedReportResults(query, report.selection, manifest); + assert.deepEqual(search.surface, { + entryPoint: "searchAios", + requestedScope: "all", + completeness: "complete", + omissions: [], + returnedScopes: ["sessions", "context", "memory", "vault", "decisions", "skills", "references", "plugins"] + }); + assert.deepEqual(search.exactResults, expected, `${reportName}:${query.id} controlled order`); + assert.deepEqual(rawControl.exactResults, expected, `${reportName}:${query.id} raw controlled order`); + assert.match(search.outputSha256, /^[a-f0-9]{64}$/); + assert.equal(rawControl.outputSha256, search.outputSha256, `${reportName}:${query.id} output hash`); + } + assert.equal(report.operationGate.passed, true); + const validatedGate = assertPublicSearchOperationGate( + report.searches, + report.safeCorpusReadControl, + report.operationGate.allowance + ); + assert.equal(report.operationGate.comparison, validatedGate.comparison); + selections.push( + `${report.selection.fileCount}:${report.selection.layout}:${report.selection.distribution}` + ); + } + + assert.deepEqual(selections.sort(), expectedSelections); +}); + +test("manifest validation rejects malformed harness sampling and scenario fields", async () => { + const manifest = await loadManifest(manifestPath); + for (const mutate of [ + (value) => { value.protocol.resultLimit = 0; }, + (value) => { value.protocol.rssPollIntervalMs = Number.MAX_SAFE_INTEGER + 1; }, + (value) => { value.corpus.scenarioMatrix = []; }, + (value) => { value.corpus.scenarioMatrix = [{ layout: "missing", distribution: "prose" }]; }, + (value) => { value.queries[1].expectation.fileIndices = null; }, + (value) => { value.queries[1].expectation.fileIndices = [3, -1]; }, + (value) => { value.queries[2].expectation.modulo = 0; }, + (value) => { value.queries[2].expectation.modulo = Number.POSITIVE_INFINITY; }, + (value) => { value.queries[2].expectation.remainder = -1; }, + (value) => { value.queries[2].expectation.remainder = value.queries[2].expectation.modulo; } + ]) { + const invalid = structuredClone(manifest); + mutate(invalid); + assert.throws(() => manifestReceipt(invalid)); + } +}); + +test("the public manifest receipt rejects unknown query expectation kinds", async () => { + const manifest = await loadManifest(manifestPath); + manifest.queries[0].expectation.kind = "future-kind"; + + assert.throws( + () => manifestReceipt(manifest), + /unsupported query expectation kind/i + ); +}); + +test("benchmark commands reserve exclusive output before reading or timing fixtures", async (t) => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "dotaios-search-benchmark-output-")); + t.after(() => fs.rm(root, { recursive: true, force: true })); + const occupied = path.join(root, "occupied.json"); + const missingReceipt = path.join(root, "missing-receipt.json"); + await fs.writeFile(occupied, "do not replace\n"); + + for (const command of ["run", "raw-search"]) { + const result = spawnSync(process.execPath, [ + path.join(repoRoot, "scripts", "bench-search.mjs"), + command, + "--fixture", path.join(root, "missing-fixture"), + "--receipt", missingReceipt, + "--output", occupied + ], { cwd: repoRoot, encoding: "utf8" }); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /EEXIST/); + assert.doesNotMatch(result.stderr, /missing-receipt/); + assert.equal(await fs.readFile(occupied, "utf8"), "do not replace\n"); + } +}); + +test("fixture generation rejects a repository destination without reserving its receipt", async (t) => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "dotaios-search-benchmark-preflight-")); + t.after(() => fs.rm(root, { recursive: true, force: true })); + const destination = path.join(repoRoot, `.benchmark-invalid-${process.pid}`); + const receiptPath = path.join(root, "invalid.receipt.json"); + + const result = spawnSync(process.execPath, [ + path.join(repoRoot, "scripts", "bench-search.mjs"), + "generate", + "--output", destination, + "--receipt", receiptPath, + "--count", "500", + "--layout", "shallow", + "--distribution", "prose" + ], { cwd: repoRoot, encoding: "utf8" }); + + assert.notEqual(result.status, 0); + assert.match(result.stderr, /outside the repository/i); + await assert.rejects(() => fs.lstat(receiptPath), { code: "ENOENT" }); + await assert.rejects(() => fs.lstat(destination), { code: "ENOENT" }); +}); + +test("fixture generation rejects an outside path that resolves back into the repository", async (t) => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "dotaios-search-benchmark-link-")); + t.after(() => fs.rm(root, { recursive: true, force: true })); + const repositoryLink = path.join(root, "repository"); + await fs.symlink(repoRoot, repositoryLink); + const destination = path.join(repositoryLink, `.benchmark-escape-${process.pid}`); + const manifest = await loadManifest(manifestPath); + + await assert.rejects( + () => generateFixture({ + manifest, + destination, + selection: { fileCount: 500, layout: "shallow", distribution: "prose" } + }), + /outside the repository/i + ); + await assert.rejects(() => fs.lstat(path.join(repoRoot, path.basename(destination))), { code: "ENOENT" }); +}); + +test("timing validation rejects empty and misordered controlled search output", () => { + const expected = ["vault/controlled-0001.md", "vault/controlled-0002.md"]; + + assert.throws( + () => assertExactResults([], expected, { queryId: "low-hit" }), + /empty controlled result/i + ); + assert.throws( + () => assertExactResults([...expected].reverse(), expected, { queryId: "low-hit" }), + /result mismatch/i + ); + assert.doesNotThrow(() => assertExactResults(expected, expected, { queryId: "low-hit" })); +}); + +test("the CLI harness exits nonzero before timing a mismatched controlled fixture", async (t) => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "dotaios-search-benchmark-invalid-")); + t.after(() => fs.rm(root, { recursive: true, force: true })); + const manifest = await loadManifest(manifestPath); + const fixtureRoot = path.join(root, "fixture"); + const receipt = await generateFixture({ + manifest, + destination: fixtureRoot, + selection: { fileCount: 500, layout: "shallow", distribution: "prose" } + }); + receipt.controlledResults["low-hit"].reverse(); + const receiptPath = path.join(root, "receipt.json"); + await fs.writeFile(receiptPath, `${JSON.stringify(receipt)}\n`); + + const result = spawnSync(process.execPath, [ + path.join(repoRoot, "scripts", "bench-search.mjs"), + "run", + "--fixture", fixtureRoot, + "--receipt", receiptPath + ], { cwd: repoRoot, encoding: "utf8" }); + + assert.notEqual(result.status, 0); + assert.match(result.stderr, /mismatch/i); +}); + +test("the unsafe benchmark-only raw search validates exact order before accepting a sample", async (t) => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "dotaios-raw-search-control-")); + t.after(() => fs.rm(root, { recursive: true, force: true })); + const manifest = await loadManifest(manifestPath); + const fixtureRoot = path.join(root, "fixture"); + const fixtureReceipt = await generateFixture({ + manifest, + destination: fixtureRoot, + selection: { fileCount: 500, layout: "shallow", distribution: "prose" } + }); + const query = manifest.queries.find(({ id }) => id === "low-hit"); + const expected = fixtureReceipt.controlledResults[query.id]; + + const sample = await runUnsafeBenchmarkOnlyRawSearchSample({ + manifest, + fixtureRoot, + fixtureReceipt, + query, + expectedResults: expected + }); + + assert.deepEqual(sample.exactResults, expected); + assert.deepEqual(sample.operations, { lstat: 0, realpath: 0, open: 500 }); + await assert.rejects( + () => runUnsafeBenchmarkOnlyRawSearchSample({ + manifest, + fixtureRoot, + fixtureReceipt, + query, + expectedResults: [...expected].reverse() + }), + /result mismatch/i + ); +}); diff --git a/tests/core/search_corpus_scale.test.mjs b/tests/core/search_corpus_scale.test.mjs index 62d0f706..d0ee6f08 100644 --- a/tests/core/search_corpus_scale.test.mjs +++ b/tests/core/search_corpus_scale.test.mjs @@ -5,6 +5,7 @@ import test from "node:test"; import assert from "node:assert/strict"; import { searchAios } from "../../packages/core/src/search.mjs"; import { createEvidenceReader, DEFAULT_EVIDENCE_READ_LIMITS } from "../../packages/core/src/evidence-reader.mjs"; +import { publicSearchOperationCeiling } from "../../scripts/bench-search.mjs"; // Search read through the same budget the bounded startup projection uses — // 512 files, 4096 entries, 16 MiB — and its error code still says so @@ -130,23 +131,91 @@ test("the shipped budget is a real ceiling, not an unbounded read", () => { assert.ok(DEFAULT_EVIDENCE_READ_LIMITS.maxEntries <= 5_000_000, "maxEntries must stay bounded"); }); -// The budget still exists, and hitting it is still an error. What changed is -// that the error has to be actionable: the old text named no limit, no cause, -// and no next step, so a person had no way to tell a real containment refusal -// from simply owning too many notes. -test("exhausting the read budget explains the limit and what to do", async (t) => { +// The budget still exists, but a search-isolatable ceiling now omits the whole +// logical corpus instead of returning partial-corpus ranking or failing a +// request that may have other unaffected scopes. +test("exhausting the read budget returns one actionable whole-scope omission", async (t) => { const root = await makeCorpus(t, { fillerFiles: 20 }); const reader = createEvidenceReader({ roots: [root], limits: { maxBytes: 512, maxFiles: 2, maxEntries: 2, maxFileBytes: 512 } }); - const error = await searchAios({ aiosPath: root, query: "peregrine routing", evidenceReader: reader }) - .then(() => null, (thrown) => thrown); + const groups = await searchAios({ aiosPath: root, query: "peregrine routing", evidenceReader: reader }); - assert.ok(error, "an exhausted budget still fails rather than silently returning a partial corpus"); - assert.match(error.message, /budget|limit/i, "the message must name the limit it hit"); - assert.match(error.message, /\b(move|split|less|fewer)\b/i, "the message must name something the person can do"); - assert.doesNotMatch(error.message, /scope|project/i, "no command-specific concepts: this error also reaches skills, activate, and MCP, which have neither"); - assert.doesNotMatch(error.message, new RegExp(root.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")), "errors stay path-free"); + assert.equal( + groups.some((group) => group.scope === groups.omissions[0].scope), + false, + "the omitted logical scope returns no partial-corpus ranking" + ); + assert.match(JSON.stringify(groups), /peregrine routing/, "unaffected admitted scopes still return valid results"); + assert.equal(groups.omissions.length, 1); + assert.ok(["file_count_exceeded", "entry_count_exceeded"].includes(groups.omissions[0].reason)); + assert.match(groups.omissions[0].recovery.message, /\b(move|archive|narrow)\b/i); + assert.doesNotMatch( + JSON.stringify(groups.omissions), + new RegExp(root.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")), + "omissions stay path-free" + ); +}); + +test("public search preflight amortizes containment work across a deep corpus", async (t) => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "dotaios-public-search-operations-")); + t.after(() => fs.rm(root, { recursive: true, force: true })); + const corpus = path.join(root, "vault", "deep", "nested", "notes"); + await fs.mkdir(corpus, { recursive: true }); + const fileCount = 64; + await Promise.all(Array.from({ length: fileCount }, (_, index) => + fs.writeFile(path.join(corpus, `${index}.md`), `# Note ${index}\n\npublic operation canary\n`) + )); + + const operations = { lstat: 0, realpath: 0, open: 0 }; + const filesystem = new Proxy(fs, { + get(target, property) { + if (Object.hasOwn(operations, property)) { + return async (...args) => { + operations[property] += 1; + return target[property](...args); + }; + } + const value = Reflect.get(target, property, target); + return typeof value === "function" ? value.bind(target) : value; + } + }); + const reader = createEvidenceReader({ roots: [root], filesystem }); + + const groups = await searchAios({ + aiosPath: root, + query: "public operation canary", + scope: "vault", + evidenceReader: reader + }); + + assert.equal(groups[0].results.length, 20, "the operation receipt must cover real ranked hits"); + assert.equal(operations.open, fileCount, "every accepted file remains handle-bound"); + const directoryCount = 5; // AIOS root plus vault/deep/nested/notes. + const ceiling = publicSearchOperationCeiling({ fileCount, directoryCount }); + assert.ok( + operations.lstat <= ceiling.lstat, + `public preflight repeated containment work per ancestor: ${JSON.stringify(operations)}` + ); + assert.ok( + operations.realpath <= ceiling.realpath, + `public preflight repeated canonicalization work per phase: ${JSON.stringify(operations)}` + ); +}); + +test("the public operation ceiling cannot hide ancestor work multiplied per file", () => { + const fileCount = 64; + const directoryCount = 5; + const ceiling = publicSearchOperationCeiling({ fileCount, directoryCount }); + const perFileAncestorWalk = { + lstat: fileCount * directoryCount, + realpath: fileCount * directoryCount, + open: fileCount + }; + + assert.ok(perFileAncestorWalk.lstat > ceiling.lstat); + assert.ok(perFileAncestorWalk.realpath > ceiling.realpath); + assert.equal(ceiling.open, fileCount); }); diff --git a/tests/core/working-context.test.mjs b/tests/core/working-context.test.mjs index 62a5bfe9..07e2e608 100644 --- a/tests/core/working-context.test.mjs +++ b/tests/core/working-context.test.mjs @@ -253,6 +253,23 @@ test("compact projection answers identity and priorities within the same budget" assert.equal(result.context.budget.used, result.rendered.length); }); +test("compact projection strips identity frontmatter without changing source files", async () => { + const aiosPath = tmpAios(); + fs.mkdirSync(path.join(aiosPath, "context"), { recursive: true }); + const identityPath = path.join(aiosPath, "context", "identity.md"); + const prioritiesPath = path.join(aiosPath, "context", "priorities.md"); + fs.writeFileSync(identityPath, "---\nsource: private-import\nkind: context\n---\n# Identity\n\nI lead the launch.\n"); + fs.writeFileSync(prioritiesPath, "---\nupdated_at: 2026-08-13\n---\n# Priorities\n\nShip the trust release.\n"); + const before = [fs.readFileSync(identityPath), fs.readFileSync(prioritiesPath)]; + + const result = await buildWorkingContext(aiosPath, {}, { clock: fixedClock }); + + assert.match(result.rendered, /I lead the launch/); + assert.match(result.rendered, /Ship the trust release/); + assert.doesNotMatch(result.rendered, /private-import|updated_at|kind: context|^---$/m); + assert.deepEqual([fs.readFileSync(identityPath), fs.readFileSync(prioritiesPath)], before); +}); + test("projection reads durable project metadata from the local README", async () => { const aiosPath = tmpAios(); fs.mkdirSync(path.join(aiosPath, "projects", "project-a"), { recursive: true }); diff --git a/tests/mcp/server.test.mjs b/tests/mcp/server.test.mjs index 2de2356f..7d9d0b6f 100644 --- a/tests/mcp/server.test.mjs +++ b/tests/mcp/server.test.mjs @@ -11,6 +11,7 @@ const repoRoot = path.resolve(new URL("../..", import.meta.url).pathname); const cli = path.join(repoRoot, "packages", "cli", "src", "index.mjs"); const server = path.join(repoRoot, "packages", "mcp", "src", "server.mjs"); const releaseVersion = JSON.parse(fs.readFileSync(path.join(repoRoot, "package.json"), "utf8")).version; +const SEARCH_RESULT_BUDGET_FLOOR = 3530; test("mcp exposes one bounded read-only DotAIOS gateway", () => { const { aiosPath } = setupAios(); @@ -55,7 +56,7 @@ test("mcp exposes one bounded read-only DotAIOS gateway", () => { jsonrpc: "2.0", id: 4, method: "tools/call", - params: { name: "search_aios", arguments: { query: "gateway", scope: "projects", project: "demo-id", budget: 1000 } }, + params: { name: "search_aios", arguments: { query: "gateway", scope: "projects", project: "demo-id", budget: SEARCH_RESULT_BUDGET_FLOOR } }, }, { jsonrpc: "2.0", @@ -100,6 +101,28 @@ test("mcp exposes one bounded read-only DotAIOS gateway", () => { assert.equal(fs.readFileSync(eventsPath, "utf8"), eventsBefore); }); +test("read_working_context preserves visible identity and priorities but omits frontmatter without mutation", () => { + const { aiosPath } = setupAios(); + const identityPath = path.join(aiosPath, "context", "identity.md"); + const prioritiesPath = path.join(aiosPath, "context", "priorities.md"); + fs.writeFileSync(identityPath, "---\nsource: private-import\nkind: context\n---\n# Identity\n\nI lead the launch.\n"); + fs.writeFileSync(prioritiesPath, "---\nupdated_at: 2026-08-13\n---\n# Priorities\n\nShip the trust release.\n"); + const before = [fs.readFileSync(identityPath), fs.readFileSync(prioritiesPath)]; + + const [response] = runMcp(aiosPath, [{ + jsonrpc: "2.0", + id: 1, + method: "tools/call", + params: { name: "read_working_context", arguments: { budget: 1000 } } + }]); + const markdown = JSON.parse(toolText(response)).markdown; + + assert.match(markdown, /I lead the launch/); + assert.match(markdown, /Ship the trust release/); + assert.doesNotMatch(markdown, /private-import|updated_at|kind: context|^---$/m); + assert.deepEqual([fs.readFileSync(identityPath), fs.readFileSync(prioritiesPath)], before); +}); + test("search_aios matches CLI project selection by slug and stable id without widening the tool allowlist", () => { const { aiosPath } = setupAios(); for (const [slug, id, canary] of [ @@ -119,13 +142,13 @@ test("search_aios matches CLI project selection by slug and stable id without wi jsonrpc: "2.0", id: 2, method: "tools/call", - params: { name: "search_aios", arguments: { query: "campaign assets", scope: "projects", project: "acme-campaign", budget: 2000 } } + params: { name: "search_aios", arguments: { query: "campaign assets", scope: "projects", project: "acme-campaign", budget: SEARCH_RESULT_BUDGET_FLOOR } } }, { jsonrpc: "2.0", id: 3, method: "tools/call", - params: { name: "search_aios", arguments: { query: "campaign assets", scope: "projects", project: "project-acme-001", budget: 2000 } } + params: { name: "search_aios", arguments: { query: "campaign assets", scope: "projects", project: "project-acme-001", budget: SEARCH_RESULT_BUDGET_FLOOR } } } ]); @@ -162,7 +185,7 @@ test("search_aios preserves the exact raw project selector like CLI and core sea query: "RAW_SELECTOR_MCP_CANARY", scope: "projects", project: " acme-campaign ", - budget: 2000, + budget: SEARCH_RESULT_BUDGET_FLOOR, }, }, }, @@ -176,7 +199,7 @@ test("search_aios preserves the exact raw project selector like CLI and core sea query: "RAW_SELECTOR_MCP_CANARY", scope: "projects", project: "acme-campaign", - budget: 2000, + budget: SEARCH_RESULT_BUDGET_FLOOR, }, }, }, @@ -218,7 +241,7 @@ test("search_aios refuses a selected catalog identity outside the selector contr query: "INVALID_ID_PRIVATE_CANARY", scope: "projects", project: "acme-campaign", - budget: 2000, + budget: SEARCH_RESULT_BUDGET_FLOOR, }, }, }]); @@ -236,7 +259,7 @@ test("mcp search budgets bound the exact serialized response at minimum, default `# Work\n\n${"bounded memory ".repeat(200)}\n`, ); const query = `bounded ${"context ".repeat(55)}`.slice(0, 500); - for (const requestedBudget of [256, undefined, 32000]) { + for (const requestedBudget of [SEARCH_RESULT_BUDGET_FLOOR, undefined, 32000]) { const argumentsValue = { query }; if (requestedBudget !== undefined) argumentsValue.budget = requestedBudget; const [response] = runMcp(aiosPath, [{ @@ -252,10 +275,54 @@ test("mcp search budgets bound the exact serialized response at minimum, default assert.ok(text.length <= expectedBudget); assert.equal(payload.budget.used, text.length); assert.equal(payload.budget.limit, expectedBudget); - if (requestedBudget === 256) assert.equal(payload.budget.truncated, true); + if (requestedBudget === SEARCH_RESULT_BUDGET_FLOOR) assert.equal(payload.budget.truncated, true); } }); +test("search_aios stabilizes budget metadata across the pretty-to-compact boundary", () => { + const { aiosPath } = setupAios(); + for (let index = 0; index < 20; index += 1) { + fs.writeFileSync( + path.join(aiosPath, "context", `boundary-${index}.md`), + `# Boundary ${index}\n\nserialization-boundary ${"x".repeat(193)}\n`, + ); + } + + const request = { + jsonrpc: "2.0", + id: 1, + method: "tools/call", + params: { + name: "search_aios", + arguments: { query: "serialization-boundary", scope: "context", limit: 20, budget: 32000 }, + }, + }; + const [fullResponse] = runMcp(aiosPath, [request]); + const fullPayload = JSON.parse(toolText(fullResponse)); + const boundarySeed = { + ...fullPayload, + budget: { ...fullPayload.budget, limit: 10000, used: 0 }, + }; + const boundaryBudget = JSON.stringify(boundarySeed, null, 2).length + 3; + + assert.ok(boundaryBudget >= 10000 && boundaryBudget <= 32000); + assert.ok(JSON.stringify(boundarySeed).length < 10000); + + request.id = 2; + request.params.arguments.budget = boundaryBudget; + const [boundaryResponse] = runMcp(aiosPath, [request]); + + assert.equal(boundaryResponse.error, undefined); + const text = toolText(boundaryResponse); + const payload = JSON.parse(text); + assert.equal(payload.results.length, 20); + assert.equal(payload.budget.limit, boundaryBudget); + assert.equal(payload.budget.used, text.length); + assert.equal(payload.budget.truncated, false); + assert.equal(text, JSON.stringify(payload)); + assert.notEqual(text, JSON.stringify(payload, null, 2)); +}); + test("mcp skill budgets bound every returned field at minimum, default, and maximum", () => { const { aiosPath } = setupAios(); const skillDir = path.join(aiosPath, "skills", "verbose"); @@ -293,7 +360,7 @@ test("mcp response budgets remain exact for astral Unicode inputs", () => { jsonrpc: "2.0", id: 1, method: "tools/call", - params: { name: "search_aios", arguments: { query: astral, scope: "context", budget: 256 } }, + params: { name: "search_aios", arguments: { query: astral, scope: "context", budget: SEARCH_RESULT_BUDGET_FLOOR } }, }, { jsonrpc: "2.0", @@ -303,14 +370,17 @@ test("mcp response budgets remain exact for astral Unicode inputs", () => { }, ]); - for (const response of responses) { + for (const [response, expectedBudget] of responses.map((response, index) => [ + response, + index === 0 ? SEARCH_RESULT_BUDGET_FLOOR : 256, + ])) { assert.equal(response.error, undefined); const text = toolText(response); const payload = JSON.parse(text); - assert.ok(text.length <= 256); - assert.equal(payload.budget.limit, 256); + assert.ok(text.length <= expectedBudget); + assert.equal(payload.budget.limit, expectedBudget); assert.equal(payload.budget.used, text.length); - assert.equal(payload.budget.truncated, true); + assert.equal(payload.budget.truncated, expectedBudget === 256); } }); @@ -550,7 +620,7 @@ test("search_aios fails closed on linked evidence without exposing a path", () = } }); -test("search_aios enforces its per-file source-work bound on JSONL", () => { +test("search_aios returns an incomplete successful envelope for a per-file ceiling", () => { const { aiosPath } = setupAios(); const eventsPath = path.join(aiosPath, "memory", "events.jsonl"); // Sized from the shipped limit, not a copy of it. Hardcoding 1 MiB meant this @@ -569,8 +639,14 @@ test("search_aios enforces its per-file source-work bound on JSONL", () => { }]); const [response] = result.stdout.split("\n").filter(Boolean).map((line) => JSON.parse(line)); - assert.equal(response.error.code, -32603); - assert.equal(response.error.message, "DotAIOS request failed safely."); + assert.equal(response.error, undefined); + assert.equal(response.result.isError, false); + const payload = JSON.parse(toolText(response)); + assert.equal(payload.complete, false); + assert.deepEqual(payload.results, []); + assert.equal(payload.omissions[0].scope, "memory"); + assert.equal(payload.omissions[0].reason, "file_too_large"); + assert.equal(payload.budget.truncated, false); assert.deepEqual(snapshotTree(aiosPath), before); assert.doesNotMatch( `${result.stdout}\n${result.stderr}`, @@ -578,6 +654,162 @@ test("search_aios enforces its per-file source-work bound on JSONL", () => { ); }); +test("search_aios succeeds at its advertised budget floor with one complete omission", () => { + const { aiosPath } = setupAios(); + fs.writeFileSync( + path.join(aiosPath, "memory", "events.jsonl"), + Buffer.alloc(DEFAULT_EVIDENCE_READ_LIMITS.maxFileBytes + 1, 0x61), + ); + + const [listed] = runMcp(aiosPath, [ + { jsonrpc: "2.0", id: 1, method: "tools/list" }, + ]); + const budgetSchema = listed.result.tools + .find((tool) => tool.name === "search_aios") + .inputSchema.properties.budget; + const [response] = runMcp(aiosPath, [{ + jsonrpc: "2.0", + id: 2, + method: "tools/call", + params: { + name: "search_aios", + arguments: { query: "missing", scope: "memory", budget: budgetSchema.minimum }, + }, + }]); + + assert.equal(budgetSchema.minimum, SEARCH_RESULT_BUDGET_FLOOR); + assert.equal(response.error, undefined); + assert.equal(response.result.isError, false); + const text = toolText(response); + const payload = JSON.parse(text); + assert.equal(payload.complete, false); + assert.deepEqual(payload.results, []); + assert.deepEqual(Object.keys(payload.omissions[0]), [ + "scope", + "reason", + "observed", + "inspection", + "recovery", + ]); + assert.equal(payload.omissions[0].reason, "file_too_large"); + assert.equal(payload.omissions[0].recovery.code, "split_or_move_file"); + assert.match(payload.omissions[0].recovery.message, /split|move/i); + assert.equal(payload.budget.limit, budgetSchema.minimum); + assert.equal(payload.budget.used, text.length); + assert.ok(text.length <= budgetSchema.minimum); +}); + +test("search_aios fits its maximum selectable omission set at the exact budget floor", () => { + const { aiosPath } = setupAios(); + const projectSlug = "ceiling-project"; + const projectPath = path.join(aiosPath, "projects", projectSlug); + fs.mkdirSync(projectPath, { recursive: true }); + fs.writeFileSync( + path.join(projectPath, "README.md"), + `---\nid: ceiling-project-id\nproject: ${projectSlug}\n---\n# Ceiling project\n`, + ); + for (const filePath of [ + path.join(aiosPath, "memory", "sessions", "index.jsonl"), + path.join(aiosPath, "context", "oversized.md"), + path.join(aiosPath, "memory", "events.jsonl"), + path.join(aiosPath, "vault", "oversized.md"), + path.join(projectPath, "oversized.md"), + path.join(aiosPath, "decisions", "oversized.md"), + path.join(aiosPath, "skills", "oversized", "SKILL.md"), + path.join(aiosPath, "references", "oversized.md"), + path.join(aiosPath, "plugins", "oversized", "manifest.json"), + ]) { + writeOversizedEvidenceFile(filePath); + } + + const responses = runMcp(aiosPath, [ + { + jsonrpc: "2.0", + id: 1, + method: "tools/call", + params: { + name: "search_aios", + arguments: { + query: "missing", + scope: "all", + project: projectSlug, + budget: SEARCH_RESULT_BUDGET_FLOOR - 1, + }, + }, + }, + { + jsonrpc: "2.0", + id: 2, + method: "tools/call", + params: { + name: "search_aios", + arguments: { + query: "missing", + scope: "all", + project: projectSlug, + budget: SEARCH_RESULT_BUDGET_FLOOR, + }, + }, + }, + ]); + + assert.equal(responses[0].error.code, -32602); + assert.match(responses[0].error.message, new RegExp(`${SEARCH_RESULT_BUDGET_FLOOR} to 32000`)); + assert.equal(responses[1].error, undefined); + assert.equal(responses[1].result.isError, false); + const text = toolText(responses[1]); + const payload = JSON.parse(text); + assert.equal(payload.complete, false); + assert.deepEqual(payload.results, []); + assert.deepEqual( + payload.omissions.map((omission) => omission.scope), + ["sessions", "context", "memory", "vault", "projects", "decisions", "skills", "references", "plugins"], + ); + assert.equal(payload.omissions.length, 9); + for (const omission of payload.omissions) { + assert.equal(omission.reason, "file_too_large"); + assert.deepEqual(Object.keys(omission.observed), ["files", "bytes", "entries"]); + assert.equal(omission.inspection, "not_searched"); + assert.equal(omission.recovery.code, "split_or_move_file"); + assert.match(omission.recovery.message, /split|move/i); + } + assert.equal(payload.budget.limit, SEARCH_RESULT_BUDGET_FLOOR); + assert.equal(payload.budget.used, text.length); + assert.ok(text.length <= SEARCH_RESULT_BUDGET_FLOOR); + assert.doesNotMatch(JSON.stringify(payload.omissions), new RegExp(aiosPath.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"))); +}); + +test("search_aios keeps completion metadata when result transport truncates", () => { + const { aiosPath } = setupAios(); + fs.writeFileSync( + path.join(aiosPath, "memory", "events.jsonl"), + Buffer.alloc(DEFAULT_EVIDENCE_READ_LIMITS.maxFileBytes + 1, 0x61) + ); + for (let index = 0; index < 8; index += 1) { + fs.writeFileSync( + path.join(aiosPath, "context", `partial-${index}.md`), + `# Work ${index}\n\n${"MCP_PARTIAL_TRANSPORT_CANARY ".repeat(100)}\n`, + ); + } + + const [response] = runMcp(aiosPath, [{ + jsonrpc: "2.0", + id: 1, + method: "tools/call", + params: { + name: "search_aios", + arguments: { query: "MCP_PARTIAL_TRANSPORT_CANARY", scope: "all", budget: SEARCH_RESULT_BUDGET_FLOOR } + } + }]); + const payload = JSON.parse(toolText(response)); + + assert.equal(response.result.isError, false); + assert.equal(payload.complete, false); + assert.equal(payload.budget.truncated, true); + assert.equal(payload.omissions[0].scope, "memory"); + assert.equal(payload.omissions[0].reason, "file_too_large"); +}); + test("search_aios rejects a session index path that escapes the AIOS root", () => { const { aiosPath, tempRoot } = setupAios(); const outsidePath = path.join(tempRoot, "outside-session.md"); @@ -1152,6 +1384,16 @@ function setupAios() { return { aiosPath, tempRoot }; } +function writeOversizedEvidenceFile(filePath) { + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + const descriptor = fs.openSync(filePath, "w"); + try { + fs.ftruncateSync(descriptor, DEFAULT_EVIDENCE_READ_LIMITS.maxFileBytes + 1); + } finally { + fs.closeSync(descriptor); + } +} + function runMcp(aiosPath, messages) { const result = runMcpResult(aiosPath, messages); if (result.status !== 0) throw new Error(`mcp failed\n${result.stdout}\n${result.stderr}`);