diff --git a/.github/workflows/create-tag.yml b/.github/workflows/create-tag.yml index f62b37799..7479da920 100644 --- a/.github/workflows/create-tag.yml +++ b/.github/workflows/create-tag.yml @@ -38,6 +38,7 @@ on: - memory - memory-consolidate - opencode + - openwiki - pi - provider-anthropic - provider-claude-code diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2c7839b90..ae2d68958 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -32,6 +32,7 @@ on: - 'memory/v*' - 'memory-consolidate/v*' - 'opencode/v*' + - 'openwiki/v*' - 'pi/v*' - 'provider-anthropic/v*' - 'provider-claude-code/v*' diff --git a/README.md b/README.md index 5bef39623..a6128c122 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,7 @@ npx skills add iii-hq/iii --all | [`browser`](browser/) | Rust | Interactive Chromium sessions over CDP with console/network capture, a11y-tree snapshots with actionable refs, viewable screenshots, and DevTools element picking for the console UI. | | [`worktree`](worktree/) | Rust | Git worktree lifecycle for parallel agents — `worktree::*` mint, claim, and track isolated worktrees per repo, emit six lifecycle trigger types, and land branches back through a per-repo FIFO queue (rebase, test gate, ff-only merge). | | [`github`](github/) | Rust | GitHub CLI (`gh`) as an iii worker — typed `github::pr/issue/repo/run/workflow/release/search::*` functions plus `github::exec` argv passthrough and `github::api` for any GitHub REST endpoint. | +| [`openwiki`](openwiki/) | Node | Source-grounded markdown wiki for any git repository — a lead agent plans the index and writer sub-agents store cited pages via `openwiki::write-page`, with router and heuristic fallback tiers, incremental refresh from git diffs on a per-wiki cron schedule, and a browser UI + JSON API under `/openwiki`. | ## SDK diff --git a/iii-permissions.yaml b/iii-permissions.yaml index 1f54fc6fc..262965ddb 100644 --- a/iii-permissions.yaml +++ b/iii-permissions.yaml @@ -167,6 +167,14 @@ rules: # github: internal hot-reload hook — invoked only by the engine's # configuration:updated trigger dispatch, never agent-callable. - '!github::on-config-change' + # openwiki: internal trigger targets. cron::refresh-due is the per-wiki cron + # target; on-turn-started/completed are harness trigger-bridge targets whose + # direct call could forge generation progress. The hot-reload hook follows + # the same pattern as the other on-config-change denies. + - '!openwiki::cron::refresh-due' + - '!openwiki::on-turn-started' + - '!openwiki::on-turn-completed' + - '!openwiki::on-config-change' # Read-only / introspection (extend below for your tools). - state::get diff --git a/openwiki/.gitignore b/openwiki/.gitignore new file mode 100644 index 000000000..3bdd52eb2 --- /dev/null +++ b/openwiki/.gitignore @@ -0,0 +1,3 @@ +node_modules/ +dist/ +.DS_Store diff --git a/openwiki/README.md b/openwiki/README.md new file mode 100644 index 000000000..f12075fdf --- /dev/null +++ b/openwiki/README.md @@ -0,0 +1,118 @@ +# openwiki + +

+ + + + OpenWiki browser UI + +

+ +Builds and maintains a source-grounded, interlinked markdown wiki for a code +repository, and serves a browser UI to read and search it. Point it at a git +repo: an agent reads the source, plans a hierarchical index, writes one cited +page per topic, and keeps the wiki current from git diffs on a per-wiki +schedule. Pages persist in iii-state; the engine serves the UI and JSON API +under `/openwiki`. + +## Install + +```bash +iii worker add openwiki +``` + +This pulls `state`, `cron`, and `llm-router` transitively. Add a model provider +through the console's onboarding (anthropic, openai, codex, ...) and pages are +model-written; the provider credential lives in the `llm-router` config, never +in this worker. + +For the best tier, agent-orchestrated pages written by one sub-agent per page +with line citations, add the harness stack as well: + +```bash +iii worker add harness +``` + +`harness` transitively pulls `session-manager`, `context-manager`, `shell` +(jailed git for clone/diff), and the model providers. openwiki degrades +gracefully when a worker is absent: + +| Present | Pages are | +|---|---| +| `harness` + a configured provider | agent-orchestrated, line-cited (best) | +| `llm-router` only | model-written from pre-selected files | +| neither | heuristic, built from file headers, always works | + +## Quickstart + +Open the browser UI on the engine's HTTP port: + +```text +http://localhost:3111/openwiki +``` + +Or drive it from the CLI: + +```bash +iii trigger openwiki::generate --json '{"repo_url":"https://github.com/owner/repo"}' +# -> { "wiki_id": "", "status": "started" } + +iii trigger openwiki::status --json '{"id":""}' # poll until phase = ready +iii trigger openwiki::page --json '{"id":"","slug":"overview"}' +iii trigger openwiki::search --json '{"id":"","q":"config"}' +``` + +`openwiki::refresh { id }` is incremental: it pulls the clone, diffs against +the recorded commit, and regenerates only the pages whose source changed. +`openwiki::set-schedule { id, schedule }` puts that refresh on a per-wiki +cadence (`off` | `3h` | `6h` | `12h` | `daily` | `weekly` | a cron string); a +content-hash gate keeps an unchanged repo from churning the wiki. + +The full function catalogue (generation, scoped source readers for writer +sub-agents, cited Q&A via `openwiki::ask`, Mermaid diagrams, `AGENTS.md` +export, lint) is one `iii worker info openwiki` away. HTTP triggers mirror the +read/generate functions under `/openwiki/api/*`, generation progress streams +live over SSE, and page citations deep-link to source at the pinned commit. + +openwiki also registers `openwiki::read-wiki-structure`, +`openwiki::read-wiki-contents`, and `openwiki::ask-question`, which the +[mcp](https://github.com/iii-hq/workers/tree/main/mcp) worker advertises to any +MCP client: + +```bash +iii worker add mcp +``` + +## How generation works + +1. Clone the repo (through the `shell` worker, with a local `git` fallback), + inventory its files, and record the commit so citations deep-link to exact + source. +2. A lead agent explores the clone through openwiki's scoped readers + (`openwiki::src::read` / `src::list` / `src::grep`) and plans a + reading-ordered index. The model decides how many pages the repo needs and + follows the repo's own docs index (`llms.txt`, a `docs/` tree) when present. +3. The lead spawns one writer sub-agent per page in parallel. Each writer reads + its focused files and stores its finished page with `openwiki::write-page`; + openwiki turns citations into pinned-commit source links and rejects a page + that comes back too thin. +4. Pages stream into the UI as each writer lands; the lead submits only the + table of contents. + +## Configuration + +- **Model**: pick one in the browser UI's generate form (populated from the + router's live catalog, grouped by provider), pass `model` to + `openwiki::generate`, or set `OPENWIKI_MODEL`. Any model the router + advertises works. Default `claude-haiku-4-5-20251001`. +- **`refresh_default`**: the auto-refresh cadence new wikis start with (`off` + by default; each wiki overrides it in the UI). Editable in the console like + the other openwiki config, or seed it with `OPENWIKI_REFRESH_DEFAULT`. +- **`OPENWIKI_DATA`**: wiki store and clone directory (default + `/tmp/openwiki-data`). Must resolve inside the shell worker's + `fs.host_roots` when git runs through `shell`. +- **`OPENWIKI_MAX_PARALLEL`**: concurrent page writers (default `3`). + +## License + +Apache-2.0 diff --git a/openwiki/assets/openwiki-dark.png b/openwiki/assets/openwiki-dark.png new file mode 100644 index 000000000..9ab3f2bd7 Binary files /dev/null and b/openwiki/assets/openwiki-dark.png differ diff --git a/openwiki/assets/openwiki-light.png b/openwiki/assets/openwiki-light.png new file mode 100644 index 000000000..dfc6e8068 Binary files /dev/null and b/openwiki/assets/openwiki-light.png differ diff --git a/openwiki/biome.json b/openwiki/biome.json new file mode 100644 index 000000000..36e842754 --- /dev/null +++ b/openwiki/biome.json @@ -0,0 +1,46 @@ +{ + "$schema": "https://biomejs.dev/schemas/2.4.10/schema.json", + "root": false, + "vcs": { "enabled": false, "clientKind": "git" }, + "files": { + "ignoreUnknown": false, + "includes": ["**", "!!**/dist", "!!**/node_modules"] + }, + "formatter": { + "enabled": true, + "indentStyle": "space", + "indentWidth": 2, + "lineWidth": 120 + }, + "assist": { + "enabled": true, + "actions": { + "source": { + "organizeImports": "off" + } + } + }, + "linter": { + "enabled": true, + "rules": { + "recommended": true, + "suspicious": { + "noExplicitAny": "warn", + "noAssignInExpressions": "off" + }, + "style": { + "useNodejsImportProtocol": "error" + }, + "complexity": { + "noForEach": "off" + } + } + }, + "javascript": { + "formatter": { + "quoteStyle": "single", + "trailingCommas": "all", + "semicolons": "always" + } + } +} diff --git a/openwiki/iii.worker.yaml b/openwiki/iii.worker.yaml new file mode 100644 index 000000000..3fafb466c --- /dev/null +++ b/openwiki/iii.worker.yaml @@ -0,0 +1,18 @@ +iii: v1 +name: openwiki +language: javascript +deploy: bundle +manifest: package.json +tags: [wiki, documentation, docs, markdown, knowledge-base, repo] +description: Source-grounded markdown wiki for any git repository — openwiki::* functions generate, search, and incrementally refresh categorized pages with pinned-commit line citations, and the engine serves a browser UI + JSON API under /openwiki. + +runtime: + kind: javascript + +scripts: + start: node ./index.mjs + +dependencies: + state: "^0.21.2" + cron: "^0.21.0" + llm-router: "^1.0.0" diff --git a/openwiki/package.json b/openwiki/package.json new file mode 100644 index 000000000..5ea13ba8c --- /dev/null +++ b/openwiki/package.json @@ -0,0 +1,32 @@ +{ + "name": "openwiki", + "version": "0.1.0", + "private": true, + "description": "Source-grounded markdown wiki for code repositories, as an iii worker: generated and refreshed from the repo itself, with a browser UI and JSON API under /openwiki.", + "license": "Apache-2.0", + "type": "module", + "engines": { + "node": ">=22" + }, + "packageManager": "pnpm@10.18.2", + "scripts": { + "build:bundle": "node scripts/build-bundle.mjs", + "check": "node --check src/index.mjs", + "lint": "biome check .", + "lint:fix": "biome check --write .", + "test": "node --test", + "start": "node src/index.mjs" + }, + "dependencies": { + "iii-sdk": "^0.21.6" + }, + "devDependencies": { + "@biomejs/biome": "2.4.10", + "esbuild": "^0.25.0" + }, + "pnpm": { + "onlyBuiltDependencies": [ + "esbuild" + ] + } +} diff --git a/openwiki/pnpm-lock.yaml b/openwiki/pnpm-lock.yaml new file mode 100644 index 000000000..8b7561cb1 --- /dev/null +++ b/openwiki/pnpm-lock.yaml @@ -0,0 +1,814 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + dependencies: + iii-sdk: + specifier: ^0.21.6 + version: 0.21.8 + devDependencies: + '@biomejs/biome': + specifier: 2.4.10 + version: 2.4.10 + esbuild: + specifier: ^0.25.0 + version: 0.25.12 + +packages: + + '@biomejs/biome@2.4.10': + resolution: {integrity: sha512-xxA3AphFQ1geij4JTHXv4EeSTda1IFn22ye9LdyVPoJU19fNVl0uzfEuhsfQ4Yue/0FaLs2/ccVi4UDiE7R30w==} + engines: {node: '>=14.21.3'} + hasBin: true + + '@biomejs/cli-darwin-arm64@2.4.10': + resolution: {integrity: sha512-vuzzI1cWqDVzOMIkYyHbKqp+AkQq4K7k+UCXWpkYcY/HDn1UxdsbsfgtVpa40shem8Kax4TLDLlx8kMAecgqiw==} + engines: {node: '>=14.21.3'} + cpu: [arm64] + os: [darwin] + + '@biomejs/cli-darwin-x64@2.4.10': + resolution: {integrity: sha512-14fzASRo+BPotwp7nWULy2W5xeUyFnTaq1V13Etrrxkrih+ez/2QfgFm5Ehtf5vSjtgx/IJycMMpn5kPd5ZNaA==} + engines: {node: '>=14.21.3'} + cpu: [x64] + os: [darwin] + + '@biomejs/cli-linux-arm64-musl@2.4.10': + resolution: {integrity: sha512-WrJY6UuiSD/Dh+nwK2qOTu8kdMDlLV3dLMmychIghHPAysWFq1/DGC1pVZx8POE3ZkzKR3PUUnVrtZfMfaJjyQ==} + engines: {node: '>=14.21.3'} + cpu: [arm64] + os: [linux] + + '@biomejs/cli-linux-arm64@2.4.10': + resolution: {integrity: sha512-7MH1CMW5uuxQ/s7FLST63qF8B3Hgu2HRdZ7tA1X1+mk+St4JOuIrqdhIBnnyqeyWJNI+Bww7Es5QZ0wIc1Cmkw==} + engines: {node: '>=14.21.3'} + cpu: [arm64] + os: [linux] + + '@biomejs/cli-linux-x64-musl@2.4.10': + resolution: {integrity: sha512-kDTi3pI6PBN6CiczsWYOyP2zk0IJI08EWEQyDMQWW221rPaaEz6FvjLhnU07KMzLv8q3qSuoB93ua6inSQ55Tw==} + engines: {node: '>=14.21.3'} + cpu: [x64] + os: [linux] + + '@biomejs/cli-linux-x64@2.4.10': + resolution: {integrity: sha512-tZLvEEi2u9Xu1zAqRjTcpIDGVtldigVvzug2fTuPG0ME/g8/mXpRPcNgLB22bGn6FvLJpHHnqLnwliOu8xjYrg==} + engines: {node: '>=14.21.3'} + cpu: [x64] + os: [linux] + + '@biomejs/cli-win32-arm64@2.4.10': + resolution: {integrity: sha512-umwQU6qPzH+ISTf/eHyJ/QoQnJs3V9Vpjz2OjZXe9MVBZ7prgGafMy7yYeRGnlmDAn87AKTF3Q6weLoMGpeqdQ==} + engines: {node: '>=14.21.3'} + cpu: [arm64] + os: [win32] + + '@biomejs/cli-win32-x64@2.4.10': + resolution: {integrity: sha512-aW/JU5GuyH4uxMrNYpoC2kjaHlyJGLgIa3XkhPEZI0uKhZhJZU8BuEyJmvgzSPQNGozBwWjC972RaNdcJ9KyJg==} + engines: {node: '>=14.21.3'} + cpu: [x64] + os: [win32] + + '@esbuild/aix-ppc64@0.25.12': + resolution: {integrity: sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [aix] + + '@esbuild/android-arm64@0.25.12': + resolution: {integrity: sha512-6AAmLG7zwD1Z159jCKPvAxZd4y/VTO0VkprYy+3N2FtJ8+BQWFXU+OxARIwA46c5tdD9SsKGZ/1ocqBS/gAKHg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [android] + + '@esbuild/android-arm@0.25.12': + resolution: {integrity: sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg==} + engines: {node: '>=18'} + cpu: [arm] + os: [android] + + '@esbuild/android-x64@0.25.12': + resolution: {integrity: sha512-5jbb+2hhDHx5phYR2By8GTWEzn6I9UqR11Kwf22iKbNpYrsmRB18aX/9ivc5cabcUiAT/wM+YIZ6SG9QO6a8kg==} + engines: {node: '>=18'} + cpu: [x64] + os: [android] + + '@esbuild/darwin-arm64@0.25.12': + resolution: {integrity: sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [darwin] + + '@esbuild/darwin-x64@0.25.12': + resolution: {integrity: sha512-HQ9ka4Kx21qHXwtlTUVbKJOAnmG1ipXhdWTmNXiPzPfWKpXqASVcWdnf2bnL73wgjNrFXAa3yYvBSd9pzfEIpA==} + engines: {node: '>=18'} + cpu: [x64] + os: [darwin] + + '@esbuild/freebsd-arm64@0.25.12': + resolution: {integrity: sha512-gA0Bx759+7Jve03K1S0vkOu5Lg/85dou3EseOGUes8flVOGxbhDDh/iZaoek11Y8mtyKPGF3vP8XhnkDEAmzeg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [freebsd] + + '@esbuild/freebsd-x64@0.25.12': + resolution: {integrity: sha512-TGbO26Yw2xsHzxtbVFGEXBFH0FRAP7gtcPE7P5yP7wGy7cXK2oO7RyOhL5NLiqTlBh47XhmIUXuGciXEqYFfBQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [freebsd] + + '@esbuild/linux-arm64@0.25.12': + resolution: {integrity: sha512-8bwX7a8FghIgrupcxb4aUmYDLp8pX06rGh5HqDT7bB+8Rdells6mHvrFHHW2JAOPZUbnjUpKTLg6ECyzvas2AQ==} + engines: {node: '>=18'} + cpu: [arm64] + os: [linux] + + '@esbuild/linux-arm@0.25.12': + resolution: {integrity: sha512-lPDGyC1JPDou8kGcywY0YILzWlhhnRjdof3UlcoqYmS9El818LLfJJc3PXXgZHrHCAKs/Z2SeZtDJr5MrkxtOw==} + engines: {node: '>=18'} + cpu: [arm] + os: [linux] + + '@esbuild/linux-ia32@0.25.12': + resolution: {integrity: sha512-0y9KrdVnbMM2/vG8KfU0byhUN+EFCny9+8g202gYqSSVMonbsCfLjUO+rCci7pM0WBEtz+oK/PIwHkzxkyharA==} + engines: {node: '>=18'} + cpu: [ia32] + os: [linux] + + '@esbuild/linux-loong64@0.25.12': + resolution: {integrity: sha512-h///Lr5a9rib/v1GGqXVGzjL4TMvVTv+s1DPoxQdz7l/AYv6LDSxdIwzxkrPW438oUXiDtwM10o9PmwS/6Z0Ng==} + engines: {node: '>=18'} + cpu: [loong64] + os: [linux] + + '@esbuild/linux-mips64el@0.25.12': + resolution: {integrity: sha512-iyRrM1Pzy9GFMDLsXn1iHUm18nhKnNMWscjmp4+hpafcZjrr2WbT//d20xaGljXDBYHqRcl8HnxbX6uaA/eGVw==} + engines: {node: '>=18'} + cpu: [mips64el] + os: [linux] + + '@esbuild/linux-ppc64@0.25.12': + resolution: {integrity: sha512-9meM/lRXxMi5PSUqEXRCtVjEZBGwB7P/D4yT8UG/mwIdze2aV4Vo6U5gD3+RsoHXKkHCfSxZKzmDssVlRj1QQA==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [linux] + + '@esbuild/linux-riscv64@0.25.12': + resolution: {integrity: sha512-Zr7KR4hgKUpWAwb1f3o5ygT04MzqVrGEGXGLnj15YQDJErYu/BGg+wmFlIDOdJp0PmB0lLvxFIOXZgFRrdjR0w==} + engines: {node: '>=18'} + cpu: [riscv64] + os: [linux] + + '@esbuild/linux-s390x@0.25.12': + resolution: {integrity: sha512-MsKncOcgTNvdtiISc/jZs/Zf8d0cl/t3gYWX8J9ubBnVOwlk65UIEEvgBORTiljloIWnBzLs4qhzPkJcitIzIg==} + engines: {node: '>=18'} + cpu: [s390x] + os: [linux] + + '@esbuild/linux-x64@0.25.12': + resolution: {integrity: sha512-uqZMTLr/zR/ed4jIGnwSLkaHmPjOjJvnm6TVVitAa08SLS9Z0VM8wIRx7gWbJB5/J54YuIMInDquWyYvQLZkgw==} + engines: {node: '>=18'} + cpu: [x64] + os: [linux] + + '@esbuild/netbsd-arm64@0.25.12': + resolution: {integrity: sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [netbsd] + + '@esbuild/netbsd-x64@0.25.12': + resolution: {integrity: sha512-Ld5pTlzPy3YwGec4OuHh1aCVCRvOXdH8DgRjfDy/oumVovmuSzWfnSJg+VtakB9Cm0gxNO9BzWkj6mtO1FMXkQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [netbsd] + + '@esbuild/openbsd-arm64@0.25.12': + resolution: {integrity: sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openbsd] + + '@esbuild/openbsd-x64@0.25.12': + resolution: {integrity: sha512-MZyXUkZHjQxUvzK7rN8DJ3SRmrVrke8ZyRusHlP+kuwqTcfWLyqMOE3sScPPyeIXN/mDJIfGXvcMqCgYKekoQw==} + engines: {node: '>=18'} + cpu: [x64] + os: [openbsd] + + '@esbuild/openharmony-arm64@0.25.12': + resolution: {integrity: sha512-rm0YWsqUSRrjncSXGA7Zv78Nbnw4XL6/dzr20cyrQf7ZmRcsovpcRBdhD43Nuk3y7XIoW2OxMVvwuRvk9XdASg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openharmony] + + '@esbuild/sunos-x64@0.25.12': + resolution: {integrity: sha512-3wGSCDyuTHQUzt0nV7bocDy72r2lI33QL3gkDNGkod22EsYl04sMf0qLb8luNKTOmgF/eDEDP5BFNwoBKH441w==} + engines: {node: '>=18'} + cpu: [x64] + os: [sunos] + + '@esbuild/win32-arm64@0.25.12': + resolution: {integrity: sha512-rMmLrur64A7+DKlnSuwqUdRKyd3UE7oPJZmnljqEptesKM8wx9J8gx5u0+9Pq0fQQW8vqeKebwNXdfOyP+8Bsg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [win32] + + '@esbuild/win32-ia32@0.25.12': + resolution: {integrity: sha512-HkqnmmBoCbCwxUKKNPBixiWDGCpQGVsrQfJoVGYLPT41XWF8lHuE5N6WhVia2n4o5QK5M4tYr21827fNhi4byQ==} + engines: {node: '>=18'} + cpu: [ia32] + os: [win32] + + '@esbuild/win32-x64@0.25.12': + resolution: {integrity: sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA==} + engines: {node: '>=18'} + cpu: [x64] + os: [win32] + + '@iii-dev/helpers@0.21.8': + resolution: {integrity: sha512-lLXVvhsX7nuMrGRAcLdAkHmKDmerziYQeyvexQg5lDEmY4lL4fRI5p57+z4oWCvDuGj+245rNN9F0ivDbm7LQg==} + + '@opentelemetry/api-logs@0.57.2': + resolution: {integrity: sha512-uIX52NnTM0iBh84MShlpouI7UKqkZ7MrUszTmaypHBu4r7NofznSnQRfJ+uUeDtQDj6w8eFGg5KBLDAwAPz1+A==} + engines: {node: '>=14'} + + '@opentelemetry/api@1.9.1': + resolution: {integrity: sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==} + engines: {node: '>=8.0.0'} + + '@opentelemetry/context-async-hooks@1.30.1': + resolution: {integrity: sha512-s5vvxXPVdjqS3kTLKMeBMvop9hbWkwzBpu+mUO2M7sZtlkyDJGwFe33wRKnbaYDo8ExRVBIIdwIGrqpxHuKttA==} + engines: {node: '>=14'} + peerDependencies: + '@opentelemetry/api': '>=1.0.0 <1.10.0' + + '@opentelemetry/core@1.30.1': + resolution: {integrity: sha512-OOCM2C/QIURhJMuKaekP3TRBxBKxG/TWWA0TL2J6nXUtDnuCtccy49LUJF8xPFXMX+0LMcxFpCo8M9cGY1W6rQ==} + engines: {node: '>=14'} + peerDependencies: + '@opentelemetry/api': '>=1.0.0 <1.10.0' + + '@opentelemetry/instrumentation@0.57.2': + resolution: {integrity: sha512-BdBGhQBh8IjZ2oIIX6F2/Q3LKm/FDDKi6ccYKcBTeilh6SNdNKveDOLk73BkSJjQLJk6qe4Yh+hHw1UPhCDdrg==} + engines: {node: '>=14'} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/otlp-transformer@0.57.2': + resolution: {integrity: sha512-48IIRj49gbQVK52jYsw70+Jv+JbahT8BqT2Th7C4H7RCM9d0gZ5sgNPoMpWldmfjvIsSgiGJtjfk9MeZvjhoig==} + engines: {node: '>=14'} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/propagator-b3@1.30.1': + resolution: {integrity: sha512-oATwWWDIJzybAZ4pO76ATN5N6FFbOA1otibAVlS8v90B4S1wClnhRUk7K+2CHAwN1JKYuj4jh/lpCEG5BAqFuQ==} + engines: {node: '>=14'} + peerDependencies: + '@opentelemetry/api': '>=1.0.0 <1.10.0' + + '@opentelemetry/propagator-jaeger@1.30.1': + resolution: {integrity: sha512-Pj/BfnYEKIOImirH76M4hDaBSx6HyZ2CXUqk+Kj02m6BB80c/yo4BdWkn/1gDFfU+YPY+bPR2U0DKBfdxCKwmg==} + engines: {node: '>=14'} + peerDependencies: + '@opentelemetry/api': '>=1.0.0 <1.10.0' + + '@opentelemetry/resources@1.30.1': + resolution: {integrity: sha512-5UxZqiAgLYGFjS4s9qm5mBVo433u+dSPUFWVWXmLAD4wB65oMCoXaJP1KJa9DIYYMeHu3z4BZcStG3LC593cWA==} + engines: {node: '>=14'} + peerDependencies: + '@opentelemetry/api': '>=1.0.0 <1.10.0' + + '@opentelemetry/sdk-logs@0.57.2': + resolution: {integrity: sha512-TXFHJ5c+BKggWbdEQ/inpgIzEmS2BGQowLE9UhsMd7YYlUfBQJ4uax0VF/B5NYigdM/75OoJGhAV3upEhK+3gg==} + engines: {node: '>=14'} + peerDependencies: + '@opentelemetry/api': '>=1.4.0 <1.10.0' + + '@opentelemetry/sdk-metrics@1.30.1': + resolution: {integrity: sha512-q9zcZ0Okl8jRgmy7eNW3Ku1XSgg3sDLa5evHZpCwjspw7E8Is4K/haRPDJrBcX3YSn/Y7gUvFnByNYEKQNbNog==} + engines: {node: '>=14'} + peerDependencies: + '@opentelemetry/api': '>=1.3.0 <1.10.0' + + '@opentelemetry/sdk-trace-base@1.30.1': + resolution: {integrity: sha512-jVPgBbH1gCy2Lb7X0AVQ8XAfgg0pJ4nvl8/IiQA6nxOsPvS+0zMJaFSs2ltXe0J6C8dqjcnpyqINDJmU30+uOg==} + engines: {node: '>=14'} + peerDependencies: + '@opentelemetry/api': '>=1.0.0 <1.10.0' + + '@opentelemetry/sdk-trace-node@1.30.1': + resolution: {integrity: sha512-cBjYOINt1JxXdpw1e5MlHmFRc5fgj4GW/86vsKFxJCJ8AL4PdVtYH41gWwl4qd4uQjqEL1oJVrXkSy5cnduAnQ==} + engines: {node: '>=14'} + peerDependencies: + '@opentelemetry/api': '>=1.0.0 <1.10.0' + + '@opentelemetry/semantic-conventions@1.28.0': + resolution: {integrity: sha512-lp4qAiMTD4sNWW4DbKLBkfiMZ4jbAboJIGOQr5DvciMRI494OapieI9qiODpOt0XBr1LjIDy1xAGAnVs5supTA==} + engines: {node: '>=14'} + + '@opentelemetry/semantic-conventions@1.43.0': + resolution: {integrity: sha512-eSYWTm620tTk45EKSedaUL8MFYI8hW164hIXsgIHyxu3VobUB3fFCu5t0hQby6OoWRPsG1KkKUG2M5UadiLiVg==} + engines: {node: '>=14'} + + '@protobufjs/aspromise@1.1.2': + resolution: {integrity: sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==} + + '@protobufjs/base64@1.1.2': + resolution: {integrity: sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==} + + '@protobufjs/codegen@2.0.5': + resolution: {integrity: sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==} + + '@protobufjs/eventemitter@1.1.1': + resolution: {integrity: sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg==} + + '@protobufjs/fetch@1.1.1': + resolution: {integrity: sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw==} + + '@protobufjs/float@1.0.2': + resolution: {integrity: sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==} + + '@protobufjs/path@1.1.2': + resolution: {integrity: sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==} + + '@protobufjs/pool@1.1.0': + resolution: {integrity: sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==} + + '@protobufjs/utf8@1.1.2': + resolution: {integrity: sha512-b1UQwcEZ4yCnMCD8DAL1VlbvBJE9/IX4FTIp7BG1xYpf29SLazLSrqUkj4w7Y5y7cCVP6E5tcqqcI0xemPkHug==} + + '@types/node@26.1.2': + resolution: {integrity: sha512-Vu4a5UFA9rIIFJ7rB/Vaafh9lrCQszopTCx6KjFboXTGQbPNasehVR5TEiithSDGyd1DEiUByggTZsg8jukeIg==} + + '@types/shimmer@1.2.0': + resolution: {integrity: sha512-UE7oxhQLLd9gub6JKIAhDq06T0F6FnztwMNRvYgjeQSBeMc1ZG/tA47EwfduvkuQS8apbkM/lpLpWsaCeYsXVg==} + + acorn-import-attributes@1.9.5: + resolution: {integrity: sha512-n02Vykv5uA3eHGM/Z2dQrcD56kL8TyDb2p1+0P83PClMnC/nc+anbQRhIOWnSq4Ke/KvDPrY3C9hDtC/A3eHnQ==} + peerDependencies: + acorn: ^8 + + acorn@8.18.0: + resolution: {integrity: sha512-lGq+9yr1/GuAWaVYIHRjvvySG5/4VfKIvC8EWxStPdcDh/Ka7FG3twP6v4d5BkravUilhIAsG4Qj83t02LWUPQ==} + engines: {node: '>=0.4.0'} + hasBin: true + + cjs-module-lexer@1.4.3: + resolution: {integrity: sha512-9z8TZaGM1pfswYeXrUpzPrkx8UnWYdhJclsiYMm6x/w5+nN+8Tf/LnAgfLGQCm59qAOxU8WwHEq2vNwF6i4j+Q==} + + debug@4.4.3: + resolution: {integrity: sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + + es-errors@1.3.0: + resolution: {integrity: sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==} + engines: {node: '>= 0.4'} + + esbuild@0.25.12: + resolution: {integrity: sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg==} + engines: {node: '>=18'} + hasBin: true + + function-bind@1.1.2: + resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==} + + hasown@2.0.4: + resolution: {integrity: sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==} + engines: {node: '>= 0.4'} + + iii-sdk@0.21.8: + resolution: {integrity: sha512-fb41PSiMv0XGJ2hwgc7x2Mh062ueh1ccoTg1Z6y5XEWzBo5+KSzNMuHYDR5o7m2J7PE1TMiBN6e/fSY5vNb59A==} + + import-in-the-middle@1.15.0: + resolution: {integrity: sha512-bpQy+CrsRmYmoPMAE/0G33iwRqwW4ouqdRg8jgbH3aKuCtOc8lxgmYXg2dMM92CRiGP660EtBcymH/eVUpCSaA==} + + is-core-module@2.16.2: + resolution: {integrity: sha512-evOr8xfXKxE6qSR0hSXL2r3sd7ALj8+7jQEUvPYcm5sgZFdJ+AYzT6yNmJenvIYQBgIGwfwz08sL8zoL7yq2BA==} + engines: {node: '>= 0.4'} + + long@5.3.2: + resolution: {integrity: sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==} + + module-details-from-path@1.0.4: + resolution: {integrity: sha512-EGWKgxALGMgzvxYF1UyGTy0HXX/2vHLkw6+NvDKW2jypWbHpjQuj4UMcqQWXHERJhVGKikolT06G3bcKe4fi7w==} + + ms@2.1.3: + resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + + path-parse@1.0.7: + resolution: {integrity: sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==} + + protobufjs@7.6.5: + resolution: {integrity: sha512-/FPD0nUc9jH6rfFjji9IBqOz4pcSE3CsT1m7Ep6Mdb0LxSUMj8hgl6GomOvZzpNpAqqGaXA0P3VSrZLFzIhQrw==} + engines: {node: '>=12.0.0'} + + require-in-the-middle@7.5.2: + resolution: {integrity: sha512-gAZ+kLqBdHarXB64XpAe2VCjB7rIRv+mU8tfRWziHRJ5umKsIHN2tLLv6EtMw7WCdP19S0ERVMldNvxYCHnhSQ==} + engines: {node: '>=8.6.0'} + + resolve@1.22.12: + resolution: {integrity: sha512-TyeJ1zif53BPfHootBGwPRYT1RUt6oGWsaQr8UyZW/eAm9bKoijtvruSDEmZHm92CwS9nj7/fWttqPCgzep8CA==} + engines: {node: '>= 0.4'} + hasBin: true + + semver@7.8.5: + resolution: {integrity: sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==} + engines: {node: '>=10'} + hasBin: true + + shimmer@1.2.1: + resolution: {integrity: sha512-sQTKC1Re/rM6XyFM6fIAGHRPVGvyXfgzIDvzoq608vM+jeyVD0Tu1E6Np0Kc2zAIFWIj963V2800iF/9LPieQw==} + + supports-preserve-symlinks-flag@1.0.0: + resolution: {integrity: sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==} + engines: {node: '>= 0.4'} + + undici-types@8.3.0: + resolution: {integrity: sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ==} + + ws@8.21.1: + resolution: {integrity: sha512-+0NTnW77fFN/DjQi6k/Sq/Yvk4Sgajw7urW8V+asjXnRgDs9gyGkdb7EzgfhA4goXsRIZKE28fzIXBHEzhuiWw==} + engines: {node: '>=10.0.0'} + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: '>=5.0.2' + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + +snapshots: + + '@biomejs/biome@2.4.10': + optionalDependencies: + '@biomejs/cli-darwin-arm64': 2.4.10 + '@biomejs/cli-darwin-x64': 2.4.10 + '@biomejs/cli-linux-arm64': 2.4.10 + '@biomejs/cli-linux-arm64-musl': 2.4.10 + '@biomejs/cli-linux-x64': 2.4.10 + '@biomejs/cli-linux-x64-musl': 2.4.10 + '@biomejs/cli-win32-arm64': 2.4.10 + '@biomejs/cli-win32-x64': 2.4.10 + + '@biomejs/cli-darwin-arm64@2.4.10': + optional: true + + '@biomejs/cli-darwin-x64@2.4.10': + optional: true + + '@biomejs/cli-linux-arm64-musl@2.4.10': + optional: true + + '@biomejs/cli-linux-arm64@2.4.10': + optional: true + + '@biomejs/cli-linux-x64-musl@2.4.10': + optional: true + + '@biomejs/cli-linux-x64@2.4.10': + optional: true + + '@biomejs/cli-win32-arm64@2.4.10': + optional: true + + '@biomejs/cli-win32-x64@2.4.10': + optional: true + + '@esbuild/aix-ppc64@0.25.12': + optional: true + + '@esbuild/android-arm64@0.25.12': + optional: true + + '@esbuild/android-arm@0.25.12': + optional: true + + '@esbuild/android-x64@0.25.12': + optional: true + + '@esbuild/darwin-arm64@0.25.12': + optional: true + + '@esbuild/darwin-x64@0.25.12': + optional: true + + '@esbuild/freebsd-arm64@0.25.12': + optional: true + + '@esbuild/freebsd-x64@0.25.12': + optional: true + + '@esbuild/linux-arm64@0.25.12': + optional: true + + '@esbuild/linux-arm@0.25.12': + optional: true + + '@esbuild/linux-ia32@0.25.12': + optional: true + + '@esbuild/linux-loong64@0.25.12': + optional: true + + '@esbuild/linux-mips64el@0.25.12': + optional: true + + '@esbuild/linux-ppc64@0.25.12': + optional: true + + '@esbuild/linux-riscv64@0.25.12': + optional: true + + '@esbuild/linux-s390x@0.25.12': + optional: true + + '@esbuild/linux-x64@0.25.12': + optional: true + + '@esbuild/netbsd-arm64@0.25.12': + optional: true + + '@esbuild/netbsd-x64@0.25.12': + optional: true + + '@esbuild/openbsd-arm64@0.25.12': + optional: true + + '@esbuild/openbsd-x64@0.25.12': + optional: true + + '@esbuild/openharmony-arm64@0.25.12': + optional: true + + '@esbuild/sunos-x64@0.25.12': + optional: true + + '@esbuild/win32-arm64@0.25.12': + optional: true + + '@esbuild/win32-ia32@0.25.12': + optional: true + + '@esbuild/win32-x64@0.25.12': + optional: true + + '@iii-dev/helpers@0.21.8': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/api-logs': 0.57.2 + '@opentelemetry/core': 1.30.1(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation': 0.57.2(@opentelemetry/api@1.9.1) + '@opentelemetry/otlp-transformer': 0.57.2(@opentelemetry/api@1.9.1) + '@opentelemetry/resources': 1.30.1(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-logs': 0.57.2(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-metrics': 1.30.1(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-trace-base': 1.30.1(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-trace-node': 1.30.1(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.43.0 + ws: 8.21.1 + transitivePeerDependencies: + - bufferutil + - supports-color + - utf-8-validate + + '@opentelemetry/api-logs@0.57.2': + dependencies: + '@opentelemetry/api': 1.9.1 + + '@opentelemetry/api@1.9.1': {} + + '@opentelemetry/context-async-hooks@1.30.1(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + + '@opentelemetry/core@1.30.1(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/semantic-conventions': 1.28.0 + + '@opentelemetry/instrumentation@0.57.2(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/api-logs': 0.57.2 + '@types/shimmer': 1.2.0 + import-in-the-middle: 1.15.0 + require-in-the-middle: 7.5.2 + semver: 7.8.5 + shimmer: 1.2.1 + transitivePeerDependencies: + - supports-color + + '@opentelemetry/otlp-transformer@0.57.2(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/api-logs': 0.57.2 + '@opentelemetry/core': 1.30.1(@opentelemetry/api@1.9.1) + '@opentelemetry/resources': 1.30.1(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-logs': 0.57.2(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-metrics': 1.30.1(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-trace-base': 1.30.1(@opentelemetry/api@1.9.1) + protobufjs: 7.6.5 + + '@opentelemetry/propagator-b3@1.30.1(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 1.30.1(@opentelemetry/api@1.9.1) + + '@opentelemetry/propagator-jaeger@1.30.1(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 1.30.1(@opentelemetry/api@1.9.1) + + '@opentelemetry/resources@1.30.1(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 1.30.1(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.28.0 + + '@opentelemetry/sdk-logs@0.57.2(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/api-logs': 0.57.2 + '@opentelemetry/core': 1.30.1(@opentelemetry/api@1.9.1) + '@opentelemetry/resources': 1.30.1(@opentelemetry/api@1.9.1) + + '@opentelemetry/sdk-metrics@1.30.1(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 1.30.1(@opentelemetry/api@1.9.1) + '@opentelemetry/resources': 1.30.1(@opentelemetry/api@1.9.1) + + '@opentelemetry/sdk-trace-base@1.30.1(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 1.30.1(@opentelemetry/api@1.9.1) + '@opentelemetry/resources': 1.30.1(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.28.0 + + '@opentelemetry/sdk-trace-node@1.30.1(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/context-async-hooks': 1.30.1(@opentelemetry/api@1.9.1) + '@opentelemetry/core': 1.30.1(@opentelemetry/api@1.9.1) + '@opentelemetry/propagator-b3': 1.30.1(@opentelemetry/api@1.9.1) + '@opentelemetry/propagator-jaeger': 1.30.1(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-trace-base': 1.30.1(@opentelemetry/api@1.9.1) + semver: 7.8.5 + + '@opentelemetry/semantic-conventions@1.28.0': {} + + '@opentelemetry/semantic-conventions@1.43.0': {} + + '@protobufjs/aspromise@1.1.2': {} + + '@protobufjs/base64@1.1.2': {} + + '@protobufjs/codegen@2.0.5': {} + + '@protobufjs/eventemitter@1.1.1': {} + + '@protobufjs/fetch@1.1.1': + dependencies: + '@protobufjs/aspromise': 1.1.2 + + '@protobufjs/float@1.0.2': {} + + '@protobufjs/path@1.1.2': {} + + '@protobufjs/pool@1.1.0': {} + + '@protobufjs/utf8@1.1.2': {} + + '@types/node@26.1.2': + dependencies: + undici-types: 8.3.0 + + '@types/shimmer@1.2.0': {} + + acorn-import-attributes@1.9.5(acorn@8.18.0): + dependencies: + acorn: 8.18.0 + + acorn@8.18.0: {} + + cjs-module-lexer@1.4.3: {} + + debug@4.4.3: + dependencies: + ms: 2.1.3 + + es-errors@1.3.0: {} + + esbuild@0.25.12: + optionalDependencies: + '@esbuild/aix-ppc64': 0.25.12 + '@esbuild/android-arm': 0.25.12 + '@esbuild/android-arm64': 0.25.12 + '@esbuild/android-x64': 0.25.12 + '@esbuild/darwin-arm64': 0.25.12 + '@esbuild/darwin-x64': 0.25.12 + '@esbuild/freebsd-arm64': 0.25.12 + '@esbuild/freebsd-x64': 0.25.12 + '@esbuild/linux-arm': 0.25.12 + '@esbuild/linux-arm64': 0.25.12 + '@esbuild/linux-ia32': 0.25.12 + '@esbuild/linux-loong64': 0.25.12 + '@esbuild/linux-mips64el': 0.25.12 + '@esbuild/linux-ppc64': 0.25.12 + '@esbuild/linux-riscv64': 0.25.12 + '@esbuild/linux-s390x': 0.25.12 + '@esbuild/linux-x64': 0.25.12 + '@esbuild/netbsd-arm64': 0.25.12 + '@esbuild/netbsd-x64': 0.25.12 + '@esbuild/openbsd-arm64': 0.25.12 + '@esbuild/openbsd-x64': 0.25.12 + '@esbuild/openharmony-arm64': 0.25.12 + '@esbuild/sunos-x64': 0.25.12 + '@esbuild/win32-arm64': 0.25.12 + '@esbuild/win32-ia32': 0.25.12 + '@esbuild/win32-x64': 0.25.12 + + function-bind@1.1.2: {} + + hasown@2.0.4: + dependencies: + function-bind: 1.1.2 + + iii-sdk@0.21.8: + dependencies: + '@iii-dev/helpers': 0.21.8 + '@opentelemetry/api': 1.9.1 + ws: 8.21.1 + transitivePeerDependencies: + - bufferutil + - supports-color + - utf-8-validate + + import-in-the-middle@1.15.0: + dependencies: + acorn: 8.18.0 + acorn-import-attributes: 1.9.5(acorn@8.18.0) + cjs-module-lexer: 1.4.3 + module-details-from-path: 1.0.4 + + is-core-module@2.16.2: + dependencies: + hasown: 2.0.4 + + long@5.3.2: {} + + module-details-from-path@1.0.4: {} + + ms@2.1.3: {} + + path-parse@1.0.7: {} + + protobufjs@7.6.5: + dependencies: + '@protobufjs/aspromise': 1.1.2 + '@protobufjs/base64': 1.1.2 + '@protobufjs/codegen': 2.0.5 + '@protobufjs/eventemitter': 1.1.1 + '@protobufjs/fetch': 1.1.1 + '@protobufjs/float': 1.0.2 + '@protobufjs/path': 1.1.2 + '@protobufjs/pool': 1.1.0 + '@protobufjs/utf8': 1.1.2 + '@types/node': 26.1.2 + long: 5.3.2 + + require-in-the-middle@7.5.2: + dependencies: + debug: 4.4.3 + module-details-from-path: 1.0.4 + resolve: 1.22.12 + transitivePeerDependencies: + - supports-color + + resolve@1.22.12: + dependencies: + es-errors: 1.3.0 + is-core-module: 2.16.2 + path-parse: 1.0.7 + supports-preserve-symlinks-flag: 1.0.0 + + semver@7.8.5: {} + + shimmer@1.2.1: {} + + supports-preserve-symlinks-flag@1.0.0: {} + + undici-types@8.3.0: {} + + ws@8.21.1: {} diff --git a/openwiki/scripts/build-bundle.mjs b/openwiki/scripts/build-bundle.mjs new file mode 100644 index 000000000..143e162a0 --- /dev/null +++ b/openwiki/scripts/build-bundle.mjs @@ -0,0 +1,61 @@ +#!/usr/bin/env node + +/** + * Single-file ESM bundle for openwiki (`dist/bundle/index.mjs`). + * + * Mirrors pi/scripts/build-bundle.mjs: iii-sdk reads its own version at + * module-init via `createRequire(import.meta.url)("../package.json")`, + * which resolves relative to the bundle path at runtime. The + * `inlinePackageJson` plugin rewrites that call to a literal object. + * openwiki itself has no native or subprocess dependencies (git runs + * through the shell worker, with a PATH fallback), so the bundle is + * fully self-contained. + */ + +import { readFile } from 'node:fs/promises'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { build } from 'esbuild'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); +const root = join(__dirname, '..'); + +/** @type {import('esbuild').Plugin} */ +const inlinePackageJson = { + name: 'iii-inline-sdk-package-json', + setup(b) { + b.onLoad({ filter: /iii-sdk[\\/]dist[\\/]index\.mjs$/ }, async (args) => { + const [source, pkg] = await Promise.all([ + readFile(args.path, 'utf8'), + readFile(join(root, 'node_modules/iii-sdk/package.json'), 'utf8'), + ]); + const { version } = JSON.parse(pkg); + const replaced = source.replace( + /createRequire\(\s*import\.meta\.url\s*\)\s*\(\s*"\.\.\/package\.json"\s*\)/g, + JSON.stringify({ version }), + ); + return { contents: replaced, loader: 'js' }; + }); + }, +}; + +await build({ + entryPoints: [join(root, 'src/index.mjs')], + bundle: true, + minify: true, + platform: 'node', + target: 'node22', + format: 'esm', + outfile: join(root, 'dist/bundle/index.mjs'), + legalComments: 'none', + external: ['fsevents'], + banner: { + js: "import{createRequire as __iiiCR}from'module';const require=__iiiCR(import.meta.url);", + }, + define: { + 'process.env.NODE_ENV': '"production"', + }, + plugins: [inlinePackageJson], + logLevel: 'info', +}); diff --git a/openwiki/skills/SKILL.md b/openwiki/skills/SKILL.md new file mode 100644 index 000000000..9875ec945 --- /dev/null +++ b/openwiki/skills/SKILL.md @@ -0,0 +1,56 @@ +--- +name: openwiki +description: Generate and browse a source-grounded markdown wiki for a git repository. +--- + +# openwiki + +Builds and maintains a categorized, interlinked markdown wiki for a code +repository. A model reads the source, plans an outline, writes one page per +topic with file citations, and refreshes pages from git diffs. Wiki content is +stored in iii-state; a browser UI and JSON API are served under `/openwiki`. + +## When to Use + +- You want a navigable wiki that explains a repository, grounded in its actual + source with `path:line` citations. +- An agent needs durable repo context beyond a single instructions file. +- You want the wiki to stay current as the repository changes. + +## Boundaries + +- Reads and clones public git repositories; it does not modify them. +- Generation quality depends on the routed model. It calls `router::complete`; + the provider and credential live in the `llm-router` config, not here. +- Not a chatbot. It maintains structured pages rather than answering free-form + questions. +- Cloned repositories are ephemeral working copies on local disk; the wiki + itself lives in iii-state. + +## Functions + +- `openwiki::generate { repo_url, model? }` — start a wiki build; returns + `{ wiki_id, status }`. Poll `openwiki::status`. +- `openwiki::status { id }` — generation progress. +- `openwiki::wikis` — list generated wikis. +- `openwiki::wiki { id }` — wiki metadata. +- `openwiki::pages { id }` — page index. +- `openwiki::page { id, slug }` — a page's markdown and metadata. +- `openwiki::search { id, q }` — keyword search over a wiki. +- `openwiki::refresh { id }` — git-pull and regenerate changed pages. +- `openwiki::set-schedule { id, schedule }` — set a wiki's auto-refresh cadence + (`off` | `3h` | `6h` | `12h` | `daily` | `weekly` | a cron string). +- `openwiki::delete { id }` — delete a wiki and all its pages. + +The same operations are exposed over HTTP under `/openwiki/api/*`, and +`/openwiki` serves the browser UI. + +## Reactive triggers + +- Each wiki with an auto-refresh cadence gets its own `cron` trigger; when one + fires, `openwiki::cron::refresh-due` runs an incremental refresh of every wiki + whose interval has elapsed. Set the cadence per wiki (`openwiki::set-schedule` + or the UI control); the config worker's `refresh_default` is the default for new + wikis. Nothing is scheduled by default (`off`). +- `openwiki::on-config-change` reloads the default model, page-writer concurrency, + and default refresh cadence when the `openwiki` configuration entry changes. diff --git a/openwiki/src/index.mjs b/openwiki/src/index.mjs new file mode 100644 index 000000000..494ad18c4 --- /dev/null +++ b/openwiki/src/index.mjs @@ -0,0 +1,1522 @@ +// OpenWiki iii worker — source-grounded wiki maintainer. +// Thin orchestrator: owns the wiki schema, page store, file->page map, and the +// UI/API surface. Git runs through the shell worker (git.mjs); persistence +// through iii-state (store.mjs); LLM through llm-router (generate.mjs). +import { registerWorker } from 'iii-sdk'; +import crypto from 'node:crypto'; +import fs from 'node:fs/promises'; + +import { cloneRepo, gitDiff, gitPull } from './lib/git.mjs'; +import { inventoryRepo, readSourceFile } from './lib/inventory.mjs'; +import * as store from './lib/store.mjs'; +import { searchPages } from './lib/search.mjs'; +import { planWiki } from './lib/generate.mjs'; +import { generatePageAny, planViaHarness, runOrchestrator, wikiParentSession, citationUrl } from './lib/harness.mjs'; +import { slugify } from './lib/ask.mjs'; +import { resolveModel, listModels } from './lib/model.mjs'; +import * as turnbus from './lib/turnbus.mjs'; +import { srcRead, srcList, srcGrep, invalidateInventory, getReadStats, resetReadStats } from './lib/src.mjs'; +import { lintWiki } from './lib/lint.mjs'; +import { askWiki } from './lib/ask.mjs'; +import { makeDiagram } from './lib/diagram.mjs'; +import { exportAgentsMd } from './lib/agents_md.mjs'; +import { normalizePlan, navSlugs } from './lib/nav.mjs'; +import { fetchDocsIndex, docsHint as buildDocsHint } from './lib/docs_oracle.mjs'; +import { pushProgress, onProgress } from './lib/progress.mjs'; +import { INDEX_HTML } from './lib/ui.mjs'; +import * as configuration from './lib/configuration.mjs'; +import * as S from './lib/schemas.mjs'; + +const III_URL = process.env.III_URL || process.env.III_ENGINE_URL || 'ws://localhost:49134'; +let cfg = configuration.defaults(); + +const worker = registerWorker(III_URL, { + workerName: 'openwiki', + workerDescription: + 'Source-grounded wiki maintainer: generates and maintains a categorized, interlinked markdown wiki for any git repository, and serves a browser UI + HTTP API to browse and search it.', +}); + +store.setWorker(worker); +store.ensureRoot().catch((e) => console.error('[openwiki] ensureRoot', e)); + +const now = () => new Date().toISOString(); +const inflight = new Set(); +// Live page count during native generation: writer sub-agents store pages via +// openwiki::write-page, which bumps this to drive the progress bar + feed. +const pagesWritten = new Map(); + +// Persist status AND push it to any live SSE subscriber for this wiki. +async function setStatus(wikiId, status) { + await store.updateStatus(wikiId, status); + pushProgress(wikiId, { + kind: 'status', + phase: status.phase, + progress: status.progress, + message: status.message, + error: status.error, + }); +} + +function err(code, message) { + const e = new Error(message || code); + e.code = code; + return e; +} + +// Agents (especially via the harness) routinely guess parameter names — wiki_id +// for id, query for q, path for slug. The engine does not validate request_format +// on agent tool calls, so a wrong name reaches the handler and dies with a cryptic +// state::get error. Resolve the common aliases up front and fail with a clear, +// self-correcting message instead. +function wikiIdOf(a) { + return a?.id || a?.wiki_id || a?.wikiId || a?.wikiID; +} +function needId(a, fn) { + const id = wikiIdOf(a); + if (!id) + throw err( + 'openwiki/bad_request', + `${fn} requires the wiki id as "id" (you passed: ${Object.keys(a || {}).join(', ') || 'nothing'})`, + ); + return id; +} + +async function readRepoReadme(dir) { + for (const name of ['README.md', 'README.mdx', 'readme.md', 'Readme.md']) { + try { + return await fs.readFile(`${dir}/${name}`, 'utf8'); + } catch { + /* try next */ + } + } + return ''; +} + +// Drop nav leaves whose page never landed (a writer failed), and any section +// left empty, so the index never links to a missing page. +function pruneNav(nodes, valid) { + const out = []; + for (const n of nodes || []) { + if (n.children?.length) { + const kids = pruneNav(n.children, valid); + if (kids.length) out.push({ ...n, children: kids }); + else if (n.slug && valid.has(n.slug)) out.push({ title: n.title, slug: n.slug }); + } else if (n.slug && valid.has(n.slug)) { + out.push(n); + } + } + return out; +} + +// Group stored pages into a category nav tree. Used when writers stored pages +// but the lead never submitted a navigation (its final turn failed). +function navFromPages(pages) { + const byCat = new Map(); + for (const p of pages) { + const cat = p.meta?.category || 'Pages'; + if (!byCat.has(cat)) byCat.set(cat, []); + byCat.get(cat).push({ title: p.meta?.title || p.slug, slug: p.slug }); + } + return [...byCat.entries()].map(([title, children]) => ({ title, children })); +} + +function inferRepoName(url) { + return ( + String(url || '') + .replace(/\.git$/, '') + .replace(/\/+$/, '') + .split(/[/:]/) + .filter(Boolean) + .slice(-2) + .join('/') || url + ); +} + +// Write a set of outline items to pages. Shared by full generation and +// incremental refresh; `fullOutline` supplies sibling links for cross-refs so a +// partial refresh still links to the whole wiki. +async function writePages( + wikiId, + { + dir, + itemsToWrite, + fullOutline, + categories, + repoName, + repoUrl, + model, + commit, + parentSessionId, + progressBase = 0.3, + progressSpan = 0.65, + }, +) { + // The native agentic path is runOrchestrator (see runGeneration). This runs + // only as the fallback for the router / heuristic tiers, synchronously in + // bounded parallel. + const resolved = await resolveModel(worker, model); + return writePagesSync(wikiId, { + dir, + itemsToWrite, + fullOutline, + categories, + repoName, + repoUrl, + model, + commit, + parentSessionId, + resolved, + progressBase, + progressSpan, + }); +} + +async function writePagesSync( + wikiId, + { + dir, + itemsToWrite, + fullOutline, + categories, + repoName, + repoUrl, + model, + commit, + parentSessionId, + resolved, + progressBase = 0.3, + progressSpan = 0.65, + }, +) { + const allSlugs = fullOutline.map((o) => o.slug); + const allTitles = fullOutline.map((o) => o.title); + const total = itemsToWrite.length || 1; + const step = Math.max(1, Number(cfg.max_parallel) || 1); + let done = 0; + + for (let i = 0; i < itemsToWrite.length; i += step) { + const batch = itemsToWrite.slice(i, i + step); + await Promise.all( + batch.map(async (item) => { + const reads = []; + for (const p of item.source_paths || []) { + try { + reads.push(await readSourceFile(dir, p, 40_000)); + } catch (e) { + reads.push({ path: p, content: `(unreadable: ${e.message})`, truncated: false }); + } + } + try { + const { markdown, frontmatter } = await generatePageAny(worker, { + wikiId, + outlineItem: item, + sourceReads: reads, + allSlugs, + allTitles, + categories, + repoName, + repoUrl, + commit, + model: resolved.model || model, + provider: resolved.provider, + useHarness: resolved.resolved, + parentSessionId, + onFallback: (err) => store.appendLog(wikiId, `harness fallback for ${item.slug}: ${err?.message || err}`), + }); + await store.savePage(wikiId, item.slug, markdown, frontmatter); + await store.appendLog(wikiId, `Wrote ${item.slug} — ${item.title}`); + pushProgress(wikiId, { kind: 'page', slug: item.slug, title: item.title }); + } catch (e) { + await store.appendLog(wikiId, `FAILED ${item.slug}: ${e?.message || e}`); + await store.savePage( + wikiId, + item.slug, + `# ${item.title}\n\n> Generation failed: ${e?.message || e}\n\n_Source paths_: ${(item.source_paths || []).map((p) => `\`${p}\``).join(', ') || '(none)'}\n`, + { + title: item.title, + slug: item.slug, + category: item.category, + source_paths: item.source_paths || [], + last_updated: now(), + confidence: 'low', + status: 'needs-review', + }, + ); + } + done += 1; + await setStatus(wikiId, { + phase: 'generating', + progress: progressBase + progressSpan * (done / total), + message: `Generated ${done}/${total} pages`, + pages_done: done, + pages_total: total, + updated_at: now(), + }); + }), + ); + } +} + +// ---------- Full generation ---------- + +async function runGeneration(wikiId, { repoUrl, model, ref, steer }) { + inflight.add(wikiId); + resetReadStats(wikiId); + pagesWritten.set(wikiId, 0); + const started = now(); + try { + await store.appendLog(wikiId, `Starting generation for ${repoUrl} (model=${model})`); + await setStatus(wikiId, { phase: 'cloning', progress: 0.05, message: 'Cloning repository', updated_at: now() }); + + const dir = store.repoDir(wikiId); + await fs.rm(dir, { recursive: true, force: true }); + const { commit, name } = await cloneRepo(worker, repoUrl, dir, ref); + invalidateInventory(wikiId); + // Persist the cloned commit + name now so openwiki::write-page can build + // pinned GitHub blob URLs for each page's citations while writers run. + const base = await store.getWiki(wikiId); + if (base) await store.saveWiki(wikiId, { ...base, repo_name: name, commit, updated_at: now() }); + + await setStatus(wikiId, { + phase: 'inventorying', + progress: 0.15, + message: 'Reading source files', + updated_at: now(), + }); + const inventory = await inventoryRepo(dir); + await store.appendLog(wikiId, `Inventoried ${inventory.length} files.`); + + await setStatus(wikiId, { + phase: 'planning', + progress: 0.25, + message: 'Exploring repo and planning structure', + updated_at: now(), + }); + let dHint = ''; + try { + const readme = await readRepoReadme(dir); + const docsIndex = await fetchDocsIndex(worker, { repoUrl, readme, repoDir: dir }); + if (docsIndex) { + dHint = buildDocsHint(docsIndex); + await store.appendLog(wikiId, `docs oracle: ${docsIndex.linkCount} topics (${docsIndex.source})`); + } + } catch { + /* oracle is optional */ + } + // Native path: ONE lead agent researches the repo, plans the pages, and + // spawns one writer sub-agent per page (harness::spawn) — the harness way. + // Each writer stores its OWN page via openwiki::write-page, so the lead only + // returns the summary + navigation; the pages are already on disk here. + let pages = null; + let navigation = []; + let summary = ''; + let categories = []; + // Live progress: writers store pages via openwiki::write-page (which streams + // the page + advances the bar). Spawns are only visible on turn-started, so + // subscribe that to stream the "spawned a writer" feed as the lead delegates. + const orchRoot = wikiParentSession(name, wikiId); + const offLive = turnbus.register(orchRoot, { + onSpawn: (childId) => { + pushProgress(wikiId, { kind: 'spawn', slug: String(childId).split('/').pop() || 'page' }); + }, + }); + let orch = null; + try { + await setStatus(wikiId, { + phase: 'generating', + progress: 0.3, + message: 'Agent researching and spawning writers', + updated_at: now(), + }); + orch = await runOrchestrator(worker, { wikiId, repoName: name, repoUrl, model, docsHint: dHint }); + } catch (e) { + await store.appendLog(wikiId, `lead did not finish cleanly (${e?.message || e})`); + } finally { + offLive(); + } + // The writers store pages directly, so use whatever landed even if the lead + // failed to submit its final nav (a large final turn can time out). Only fall + // back to the router plan when nothing was written at all. + const stored = await store.listPages(wikiId); + if (stored.length) { + navigation = + orch && Array.isArray(orch.navigation) && orch.navigation.length ? orch.navigation : navFromPages(stored); + summary = orch?.summary || ''; + // categories is derived below from the PRUNED navigation (post pruneNav), + // so it is intentionally not computed here. + pages = stored; + await store.appendLog( + wikiId, + `Using ${stored.length} writer-stored page(s)${orch ? '' : ' (lead did not submit nav; grouped by category)'}.`, + ); + } + + if (pages) { + // Pages are already stored by the writers; build the outline from what + // landed and prune the index so it never links to a page a writer failed to + // produce (with the write-page thin-guard, stored pages are all substantial). + const items = pages.map((p) => ({ + slug: p.slug, + title: p.meta?.title || p.slug, + category: p.meta?.category || '', + source_paths: p.meta?.source_paths || [], + })); + const haveSlugs = new Set(items.map((i) => i.slug)); + const missing = navSlugs(navigation).filter((s) => !haveSlugs.has(s)); + if (missing.length) + await store.appendLog( + wikiId, + `pruned ${missing.length} unwritten page(s) from the index: ${missing.join(', ')}`, + ); + navigation = pruneNav(navigation, haveSlugs); + categories = navigation.map((l1) => ({ id: l1.title, title: l1.title })); + await store.saveOutline(wikiId, { navigation, categories, items }); + } else { + // Fallback: two-phase plan + parallel writers (no harness model, or the + // orchestrator failed). Keeps openwiki working without the agentic path. + let planned = null; + try { + planned = await planViaHarness(worker, { wikiId, repoName: name, repoUrl, model, docsHint: dHint }); + } catch (e) { + await store.appendLog(wikiId, `harness plan fallback (${e?.message || e})`); + } + if (!planned) + planned = await planWiki(worker, { inventory, repoName: name, repoUrl, model, repoDir: dir, steer }); + const parentSessionId = planned?.sessionId; + const norm = normalizePlan(planned); + summary = norm.summary; + navigation = norm.navigation; + const invPaths = new Set(inventory.map((e) => e.relPath)); + const outline = norm.outline.map((item) => ({ + ...item, + source_paths: (item.source_paths || []).filter((p) => invPaths.has(p)), + })); + categories = navigation.map((l1) => ({ id: l1.title, title: l1.title })); + await store.saveOutline(wikiId, { navigation, categories, items: outline }); + await store.saveWiki(wikiId, { + id: wikiId, + repo_url: repoUrl, + repo_name: name, + ref: ref || '', + commit, + created_at: started, + updated_at: now(), + page_count: outline.length, + category_count: categories.length, + categories, + navigation, + summary, + model, + steer: steer || undefined, + generating: true, + }); + await writePages(wikiId, { + dir, + itemsToWrite: outline, + fullOutline: outline, + categories, + repoName: name, + repoUrl, + model, + commit, + parentSessionId, + }); + pages = outline; + } + + const content_hash = await store.computeContentHash(wikiId); + // Preserve the auto-refresh cadence across a (re)generation (this saveWiki + // rebuilds meta from scratch), defaulting a brand-new wiki to the config + // default. Then (re)register its cron trigger. + const prior = await store.getWiki(wikiId); + const refresh_schedule = prior?.refresh_schedule || cfg.refresh_default || 'off'; + await store.saveWiki(wikiId, { + id: wikiId, + repo_url: repoUrl, + repo_name: name, + ref: ref || '', + commit, + created_at: started, + updated_at: now(), + page_count: pages.length, + category_count: categories.length, + categories, + navigation, + summary, + model, + steer: steer || undefined, + generating: false, + content_hash, + refresh_schedule, + last_refresh_at: prior?.last_refresh_at || undefined, + }); + applyWikiSchedule(wikiId, refresh_schedule); + await setStatus(wikiId, { phase: 'ready', progress: 1, message: 'Wiki ready', updated_at: now() }); + await store.appendLog(wikiId, 'Wiki ready.'); + try { + const { issues } = await lintWiki(wikiId); + if (issues.length) await store.appendLog(wikiId, `lint: ${issues.length} issue(s) flagged`); + } catch (e) { + await store.appendLog(wikiId, `lint skipped: ${e?.message || e}`); + } + } catch (e) { + console.error('[openwiki] generation error', e); + await store.appendLog(wikiId, `ERROR: ${e?.stack || e?.message || e}`); + await setStatus(wikiId, { phase: 'error', progress: 0, error: String(e?.message || e), updated_at: now() }); + } finally { + inflight.delete(wikiId); + pagesWritten.delete(wikiId); + } +} + +async function startWiki(repoUrl, model, ref, steer) { + if (!repoUrl || typeof repoUrl !== 'string') throw err('openwiki/repo_not_found', 'repo_url required'); + const wikiId = crypto.randomUUID(); + const chosen = model || cfg.model; + await store.saveWiki(wikiId, { + id: wikiId, + repo_url: repoUrl, + repo_name: inferRepoName(repoUrl), + ref: ref || '', + commit: '', + created_at: now(), + updated_at: now(), + page_count: 0, + category_count: 0, + categories: [], + summary: '', + model: chosen, + steer: steer || undefined, + generating: true, + refresh_schedule: cfg.refresh_default || 'off', + }); + await setStatus(wikiId, { phase: 'queued', progress: 0, message: 'Queued', updated_at: now() }); + setImmediate(() => { + runGeneration(wikiId, { repoUrl, model: chosen, ref, steer }).catch((e) => console.error(e)); + }); + return { wiki_id: wikiId, status: 'queued' }; +} + +// ---------- Incremental refresh ---------- + +// Regenerate only the affected pages, then gate on a content hash so an +// identical result does not churn the wiki's updated_at (langchain-ai/openwiki +// anti-churn: git-head gate + content-hash gate). +async function runRefresh(wikiId, { dir, itemsToWrite, outline, meta, newCommit, prevHash }) { + inflight.add(wikiId); + try { + await setStatus(wikiId, { + phase: 'generating', + progress: 0.3, + message: `Refreshing ${itemsToWrite.length} pages`, + updated_at: now(), + }); + await writePages(wikiId, { + dir, + itemsToWrite, + fullOutline: outline.items || [], + categories: outline.categories || meta.categories || [], + repoName: meta.repo_name, + repoUrl: meta.repo_url, + model: meta.model || cfg.model, + commit: newCommit, + }); + const content_hash = await store.computeContentHash(wikiId); + const churned = content_hash !== prevHash; + await store.saveWiki(wikiId, { + ...meta, + commit: newCommit, + content_hash, + page_count: (outline.items || []).length, + updated_at: now(), + }); + await setStatus(wikiId, { + phase: 'ready', + progress: 1, + message: churned ? 'Refresh complete' : 'No content change', + updated_at: now(), + }); + await store.appendLog( + wikiId, + `refresh: regenerated ${itemsToWrite.length} pages (content ${churned ? 'changed' : 'unchanged'})`, + ); + } catch (e) { + await store.appendLog(wikiId, `refresh error: ${e?.message || e}`); + await setStatus(wikiId, { phase: 'error', progress: 0, error: String(e?.message || e), updated_at: now() }); + } finally { + inflight.delete(wikiId); + } +} + +async function refreshWiki(wikiId) { + const meta = await store.getWiki(wikiId); + if (!meta) throw err('openwiki/wiki_not_found', 'wiki not found'); + + // One refresh (or generation) per wiki at a time: concurrent runs would race + // the same clone directory through gitPull/cloneRepo. The marker is held + // until the scheduled runGeneration/runRefresh takes ownership (each re-adds + // it and deletes it in its own finally). + if (inflight.has(wikiId)) { + return { wiki_id: wikiId, refresh: 'in_progress', changed: [], pages_affected: [] }; + } + inflight.add(wikiId); + let handedOff = false; + try { + return await doRefresh(wikiId, meta, () => { + handedOff = true; + }); + } finally { + if (!handedOff) inflight.delete(wikiId); + } +} + +async function doRefresh(wikiId, meta, markHandedOff) { + // Stamp the refresh clock up front so the scheduled due-check advances even + // when this run finds nothing to do or kicks off an async rebuild. + meta.last_refresh_at = now(); + await store.saveWiki(wikiId, meta); + + const dir = store.repoDir(wikiId); + const prevCommit = meta.commit || ''; + let newCommit = null; + + // Ensure a clone exists and is current. + const stat = await fs.stat(dir).catch(() => null); + if (stat?.isDirectory()) { + newCommit = await gitPull(worker, dir); + if (!newCommit) { + await fs.rm(dir, { recursive: true, force: true }); + ({ commit: newCommit } = await cloneRepo(worker, meta.repo_url, dir, meta.ref)); + } + } else { + ({ commit: newCommit } = await cloneRepo(worker, meta.repo_url, dir, meta.ref)); + } + invalidateInventory(wikiId); + + // Anti-churn: HEAD unchanged -> nothing to do. + if (prevCommit && newCommit && prevCommit === newCommit) { + await store.appendLog(wikiId, 'refresh: HEAD unchanged'); + return { wiki_id: wikiId, refresh: 'up_to_date', changed: [], pages_affected: [] }; + } + + // No prior commit / no pages -> full build. + const priorPages = await store.listPages(wikiId); + const fullRebuild = () => { + markHandedOff(); + setImmediate(() => { + runGeneration(wikiId, { + repoUrl: meta.repo_url, + model: meta.model || cfg.model, + ref: meta.ref, + steer: meta.steer, + }).catch((e) => console.error(e)); + }); + return { wiki_id: wikiId, refresh: 'regenerating', changed: [], pages_affected: [] }; + }; + if (!prevCommit || priorPages.length === 0) return fullRebuild(); + + // Diff prev..new, map changed files to affected pages, regenerate only those. + // A failed diff (e.g. the previous commit was pruned from the clone) means + // the change set is unknown; rebuild rather than report up_to_date. + let changed = []; + try { + changed = await gitDiff(worker, dir, prevCommit, newCommit); + } catch (e) { + await store.appendLog(wikiId, `refresh diff failed (${e?.message || e}); treating as unknown changes`); + return fullRebuild(); + } + const changedPaths = changed.map((c) => c.path); + const affected = await store.pagesForPaths(wikiId, changedPaths); + await store.appendLog(wikiId, `refresh: ${changed.length} changed paths -> ${affected.length} affected pages`); + + if (affected.length === 0) { + await store.saveWiki(wikiId, { ...meta, commit: newCommit, updated_at: now() }); + return { wiki_id: wikiId, refresh: 'up_to_date', changed, pages_affected: [] }; + } + + const outline = (await store.getOutline(wikiId)) || { items: [], categories: meta.categories || [] }; + const itemsToWrite = (outline.items || []).filter((o) => affected.includes(o.slug)); + const prevHash = meta.content_hash || (await store.computeContentHash(wikiId)); + markHandedOff(); + setImmediate(() => { + runRefresh(wikiId, { dir, itemsToWrite, outline, meta, newCommit, prevHash }).catch((e) => console.error(e)); + }); + return { wiki_id: wikiId, refresh: 'regenerating', changed, pages_affected: itemsToWrite.map((i) => i.slug) }; +} + +// ---------- iii functions ---------- + +worker.registerFunction( + 'openwiki::generate', + async ({ repo_url, model, ref, steer }) => startWiki(repo_url, model, ref, steer), + { + description: + 'Start generating a source-grounded wiki for a git repository URL. Returns immediately with { wiki_id, status }; poll openwiki::status.', + request_format: S.GENERATE_REQ, + response_format: S.GENERATE_RES, + }, +); + +worker.registerFunction( + 'openwiki::status', + async (a) => (await store.getStatus(wikiIdOf(a))) || { phase: 'unknown', progress: 0, updated_at: now() }, + { description: 'Poll generation status for a wiki id.', request_format: S.STATUS_REQ, response_format: S.STATUS_RES }, +); + +worker.registerFunction('openwiki::wikis', async () => ({ wikis: await store.listWikis() }), { + description: 'List all wikis generated by this worker.', + request_format: S.WIKIS_REQ, + response_format: S.WIKIS_RES, +}); + +worker.registerFunction( + 'openwiki::models', + async () => ({ models: await listModels(worker), default_model: cfg.model }), + { + description: + "Models available via llm-router (for the UI's model picker), plus the configured default. Empty when no provider is configured.", + request_format: S.MODELS_REQ, + response_format: S.MODELS_RES, + }, +); + +worker.registerFunction( + 'openwiki::wiki', + async (a) => { + const id = needId(a, 'openwiki::wiki'); + const m = await store.getWiki(id); + if (!m) throw err('openwiki/wiki_not_found', 'wiki not found'); + return m; + }, + { description: "Fetch a single wiki's metadata.", request_format: S.WIKI_REQ, response_format: S.WIKI_RES }, +); + +worker.registerFunction( + 'openwiki::pages', + async (a) => { + const id = needId(a, 'openwiki::pages'); + return { pages: (await store.listPages(id)).map((x) => ({ slug: x.slug, ...x.meta })) }; + }, + { + description: 'List all pages of a wiki. Params: { id }.', + request_format: S.PAGES_REQ, + response_format: S.PAGES_RES, + }, +); + +worker.registerFunction( + 'openwiki::page', + async (a) => { + const id = needId(a, 'openwiki::page'); + const raw = a.slug || a.page || a.path || a.title; + if (!raw) + throw err('openwiki/bad_request', 'openwiki::page requires a page "slug" (list them with openwiki::pages)'); + const slug = slugify(String(raw)); + const p = await store.getPage(id, slug); + if (!p) { + const have = (await store.listPages(id)).map((x) => x.slug); + throw err('openwiki/page_not_found', `page "${slug}" not found; available slugs: ${have.join(', ') || '(none)'}`); + } + return { slug, ...p.meta, markdown: p.markdown }; + }, + { + description: 'Get a single wiki page (markdown body + metadata). Params: { id, slug } (slug from openwiki::pages).', + request_format: S.PAGE_REQ, + response_format: S.PAGE_RES, + }, +); + +worker.registerFunction( + 'openwiki::search', + async (a) => { + const id = needId(a, 'openwiki::search'); + return { results: await searchPages(id, a.q || a.query || '') }; + }, + { + description: 'Search pages of a wiki by keyword. Params: { id, q }.', + request_format: S.SEARCH_REQ, + response_format: S.SEARCH_RES, + }, +); + +worker.registerFunction('openwiki::refresh', async (a) => refreshWiki(needId(a, 'openwiki::refresh')), { + description: 'Pull the repo and regenerate only the pages whose source changed (incremental).', + request_format: S.WIKI_REQ, + response_format: S.REFRESH_RES, +}); + +worker.registerFunction( + 'openwiki::delete', + async (a) => { + const id = needId(a, 'openwiki::delete'); + if (inflight.has(id)) throw err('openwiki/generating', 'wiki is generating; stop it before deleting'); + applyWikiSchedule(id, 'off'); // tear down any auto-refresh trigger for this wiki + await store.deleteWiki(id); + invalidateInventory(id); + resetReadStats(id); + return { id, deleted: true }; + }, + { + description: 'Delete a wiki and all its pages. Params: { id }.', + request_format: S.WIKI_REQ, + response_format: { + type: 'object', + additionalProperties: false, + required: ['id', 'deleted'], + properties: { id: { type: 'string' }, deleted: { type: 'boolean' } }, + }, + }, +); + +worker.registerFunction('openwiki::lint', async (a) => lintWiki(needId(a, 'openwiki::lint')), { + description: 'Validate every page citation against the clone and flag thin pages.', + request_format: S.LINT_REQ, + response_format: S.LINT_RES, +}); + +worker.registerFunction( + 'openwiki::gen-stats', + async (a) => { + const id = needId(a, 'openwiki::gen-stats'); + const reads = getReadStats(id) || {}; + const pages = await store.listPages(id); + let output_bytes = 0; + for (const p of pages) { + const pg = await store.getPage(id, p.slug); + if (pg) output_bytes += (pg.markdown || '').length; + } + return { reads, page_count: pages.length, output_bytes }; + }, + { + description: 'Measurement: source bytes the agent read and page bytes produced for a generation.', + request_format: S.WIKI_REQ, + response_format: { + type: 'object', + additionalProperties: true, + properties: { page_count: { type: 'integer' }, output_bytes: { type: 'integer' } }, + }, + }, +); + +// ---------- Scoped source readers (the harness's exploration tools) ---------- +// Each is jailed to one wiki's clone; the page-writer harness calls these via +// agent_trigger to explore the repo and cite exact line ranges. + +// Surface the harness's file exploration as live activity. During planning +// (one long opaque plan turn) there is no page progress, so without this the UI +// sits static and reads as frozen. Each read/list/grep pushes a cheap activity +// frame the panel shows as the current line ("reading src/queue.js"). +worker.registerFunction( + 'openwiki::src::read', + async (a) => { + const id = needId(a, 'openwiki::src::read'); + const path = a.path || a.file || a.filename; + if (!path) throw err('openwiki/bad_request', 'openwiki::src::read requires a file "path"'); + pushProgress(id, { kind: 'activity', op: 'read', path }); + return srcRead(id, path, a.from, a.to); + }, + { + description: + "Read a file (optional 1-indexed line window) from a wiki's cloned repo. Params: { id, path, from?, to? }.", + request_format: S.SRC_READ_REQ, + response_format: S.SRC_READ_RES, + }, +); + +worker.registerFunction( + 'openwiki::src::list', + async (a) => { + const id = needId(a, 'openwiki::src::list'); + const dir = a.dir || a.path || ''; + pushProgress(id, { kind: 'activity', op: 'list', path: dir || '.' }); + return srcList(id, dir); + }, + { + description: "List files (path, language, priority) in a wiki's cloned repo. Params: { id, dir? }.", + request_format: S.SRC_LIST_REQ, + response_format: S.SRC_LIST_RES, + }, +); + +worker.registerFunction( + 'openwiki::src::grep', + async (a) => { + const id = needId(a, 'openwiki::src::grep'); + const pattern = a.pattern || a.query || a.q; + if (!pattern) throw err('openwiki/bad_request', 'openwiki::src::grep requires a "pattern"'); + pushProgress(id, { kind: 'activity', op: 'grep', path: pattern }); + return srcGrep(id, pattern, a.max); + }, + { + description: "Search file contents in a wiki's cloned repo. Params: { id, pattern, max? }.", + request_format: S.SRC_GREP_REQ, + response_format: S.SRC_GREP_RES, + }, +); + +// A page-writer sub-agent stores its finished page here directly, so the parent +// never collects the markdown (that assembly was the bottleneck). Guarded to a +// wiki that is actively generating; the concurrent index write is serialized in +// store.savePage. Streams a live page event. +worker.registerFunction( + 'openwiki::write-page', + async ({ id, slug, title, category, markdown, source_paths, citations }) => { + const meta = await store.getWiki(id); + if (!meta?.generating) throw err('openwiki/not_generating', 'wiki is not generating; write-page refused'); + const s = slugify(slug || title || 'page'); + // Reject thin/empty pages so an under-delivering writer cannot store a blank + // page (that landed empty "overview"/"installation" pages in the index). The + // writer sees this error and can retry with real content within its turns. + const body = String(markdown || '').trim(); + if (body.length < 250) + throw err( + 'openwiki/thin_page', + `page "${s}" is too thin (${body.length} chars). Write a substantial, source-grounded page (at least 3 "##" sections and 400+ words) before calling write-page.`, + ); + // Build the References rail: turn the writer's citations (and any bare + // source_paths) into pinned GitHub blob URLs at the cloned commit, so the UI + // renders a References list and makes each source file a clickable deep-link. + const cited = (Array.isArray(citations) ? citations : []).filter((c) => c?.path); + const paths = [...new Set(cited.map((c) => c.path).concat(source_paths || []))]; + const withUrl = cited.map((c) => { + const url = citationUrl(meta.repo_url, meta.commit, c.path, c.start_line, c.end_line); + return { + path: c.path, + ...(c.start_line ? { start_line: c.start_line } : {}), + ...(c.end_line ? { end_line: c.end_line } : {}), + ...(c.note ? { note: c.note } : {}), + ...(url ? { url } : {}), + }; + }); + for (const p of paths) { + if (withUrl.some((c) => c.path === p)) continue; + const url = citationUrl(meta.repo_url, meta.commit, p); + withUrl.push({ path: p, ...(url ? { url } : {}) }); + } + const frontmatter = { + title: title || s, + slug: s, + category: category || '', + source_paths: paths, + citations: withUrl, + last_updated: now(), + confidence: 'medium', + status: 'current', + generator: 'harness', + }; + await store.savePage(id, s, body, frontmatter); + const n = (pagesWritten.get(id) || 0) + 1; + pagesWritten.set(id, n); + await store.appendLog(id, `Wrote ${s} — ${title || s}`); + pushProgress(id, { kind: 'page', slug: s, title: title || s }); + await setStatus(id, { + phase: 'generating', + progress: Math.min(0.92, 0.35 + n * 0.06), + message: `Writers stored ${n} page(s)`, + pages_done: n, + updated_at: now(), + }); + return { slug: s, ok: true }; + }, + { + description: 'A page-writer sub-agent stores its finished page (markdown + metadata) for a generating wiki.', + request_format: S.WRITE_PAGE_REQ, + response_format: S.WRITE_PAGE_RES, + }, +); + +// ---------- Ask / diagram / export ---------- + +worker.registerFunction( + 'openwiki::ask', + async (a) => + askWiki(worker, { + id: needId(a, 'openwiki::ask'), + q: a.q, + mode: a.mode, + file_answer: a.file_answer, + model: a.model, + }), + { + description: 'Ask a question about a wiki; returns a cited answer. mode=fast (router) or deep (harness).', + request_format: S.ASK_REQ, + response_format: S.ASK_RES, + }, +); + +worker.registerFunction( + 'openwiki::diagram', + async (a) => makeDiagram(worker, { id: needId(a, 'openwiki::diagram'), kind: a.kind }), + { + description: 'Generate a Mermaid diagram (architecture|dataflow|deps) of a wiki.', + request_format: S.DIAGRAM_REQ, + response_format: S.DIAGRAM_RES, + }, +); + +worker.registerFunction( + 'openwiki::export-agents-md', + async ({ id, targets, base_url }) => exportAgentsMd(worker, { id, targets, baseUrl: base_url }), + { + description: 'Build the AGENTS.md/CLAUDE.md pointer block for a wiki.', + request_format: S.EXPORT_AGENTS_REQ, + response_format: S.EXPORT_AGENTS_RES, + }, +); + +// ---------- MCP surface (DeepWiki-compatible tool names; mcp bridge exposes these) ---------- + +const MCP_EXPOSE = { mcp: { expose: true } }; + +worker.registerFunction( + 'openwiki::read-wiki-structure', + async (a) => { + const id = needId(a, 'openwiki::read-wiki-structure'); + const m = await store.getWiki(id); + if (!m) throw err('openwiki/wiki_not_found', 'wiki not found'); + const pages = (await store.listPages(id)).map((x) => ({ + slug: x.slug, + title: x.meta?.title, + category: x.meta?.category, + })); + return { repo: m.repo_name, summary: m.summary, categories: m.categories || [], pages }; + }, + { + description: "MCP: list a wiki's structure (categories + pages).", + request_format: S.WIKI_REQ, + response_format: S.MCP_STRUCTURE_RES, + metadata: MCP_EXPOSE, + }, +); + +worker.registerFunction( + 'openwiki::read-wiki-contents', + async (a) => { + const { slug } = a; + const p = await store.getPage(needId(a, 'openwiki::read-wiki-contents'), slug); + if (!p) throw err('openwiki/page_not_found', 'page not found'); + return { slug, ...p.meta, markdown: p.markdown }; + }, + { + description: 'MCP: read one wiki page (markdown + metadata).', + request_format: S.PAGE_REQ, + response_format: S.PAGE_RES, + metadata: MCP_EXPOSE, + }, +); + +worker.registerFunction('openwiki::ask-question', async ({ id, q }) => askWiki(worker, { id, q, mode: 'fast' }), { + description: 'MCP: ask a question about a wiki; returns a cited answer.', + request_format: S.SEARCH_REQ, + response_format: S.ASK_RES, + metadata: MCP_EXPOSE, +}); + +// ---------- HTTP handlers ---------- + +const HTTP_REQ = { + type: 'object', + additionalProperties: true, + properties: { + body: { type: 'string' }, + path_params: { type: 'object', additionalProperties: { type: 'string' } }, + query_params: { type: 'object', additionalProperties: { type: 'string' } }, + headers: { type: 'object', additionalProperties: { type: 'string' } }, + method: { type: 'string' }, + }, +}; +const HTTP_RES = { + type: 'object', + additionalProperties: false, + required: ['status_code', 'body'], + properties: { + status_code: { type: 'integer' }, + headers: { type: 'object', additionalProperties: { type: 'string' } }, + body: { type: 'string' }, + }, +}; +const HTTP_META = (description) => ({ description, request_format: HTTP_REQ, response_format: HTTP_RES }); + +function jsonResponse(status_code, body, extraHeaders = {}) { + return { + status_code, + headers: { 'content-type': 'application/json; charset=utf-8', ...extraHeaders }, + body: JSON.stringify(body), + }; +} +function htmlResponse(status_code, html) { + return { + status_code, + headers: { 'content-type': 'text/html; charset=utf-8', 'cache-control': 'no-store' }, + body: html, + }; +} +function parseBody(body) { + if (body == null) return {}; + if (typeof body === 'object') return body; + if (typeof body === 'string') { + try { + return JSON.parse(body); + } catch { + return {}; + } + } + return {}; +} + +worker.registerFunction( + 'openwiki::http::ui', + async () => htmlResponse(200, INDEX_HTML), + HTTP_META('HTTP: serve the OpenWiki browser UI.'), +); + +worker.registerFunction( + 'openwiki::http::wikis-list', + async () => jsonResponse(200, await store.listWikis()), + HTTP_META('HTTP GET /openwiki/api/wikis'), +); + +worker.registerFunction( + 'openwiki::http::models', + async () => jsonResponse(200, { models: await listModels(worker), default_model: cfg.model }), + HTTP_META('HTTP GET /openwiki/api/models'), +); + +worker.registerFunction( + 'openwiki::http::wikis-create', + async ({ body }) => { + const payload = parseBody(body); + if (!payload.repo_url) return jsonResponse(400, { error: 'repo_url required' }); + const { wiki_id, status } = await startWiki(payload.repo_url, payload.model, payload.ref, payload.steer); + return jsonResponse(202, { wiki_id, status }); + }, + HTTP_META('HTTP POST /openwiki/api/wikis'), +); + +worker.registerFunction( + 'openwiki::http::wiki-get', + async ({ path_params }) => { + const m = await store.getWiki(path_params?.id); + return m ? jsonResponse(200, m) : jsonResponse(404, { error: 'not found' }); + }, + HTTP_META('HTTP GET /openwiki/api/wikis/:id'), +); + +worker.registerFunction( + 'openwiki::http::wiki-status', + async ({ path_params }) => { + const s = await store.getStatus(path_params?.id); + return jsonResponse(200, s || { phase: 'unknown', progress: 0, updated_at: now() }); + }, + HTTP_META('HTTP GET /openwiki/api/wikis/:id/status'), +); + +worker.registerFunction( + 'openwiki::http::wiki-delete', + async ({ path_params }) => { + const id = path_params?.id; + if (!id) return jsonResponse(400, { error: 'id required' }); + if (inflight.has(id)) return jsonResponse(409, { error: 'wiki is generating; stop it before deleting' }); + applyWikiSchedule(id, 'off'); // tear down any auto-refresh trigger for this wiki + await store.deleteWiki(id); + invalidateInventory(id); + resetReadStats(id); + return jsonResponse(200, { id, deleted: true }); + }, + HTTP_META('HTTP DELETE /openwiki/api/wikis/:id'), +); + +worker.registerFunction( + 'openwiki::http::pages-list', + async ({ path_params }) => { + const items = await store.listPages(path_params?.id); + return jsonResponse( + 200, + items.map((x) => ({ slug: x.slug, ...x.meta })), + ); + }, + HTTP_META('HTTP GET /openwiki/api/wikis/:id/pages'), +); + +worker.registerFunction( + 'openwiki::http::page-get', + async ({ path_params }) => { + const p = await store.getPage(path_params?.id, path_params?.slug); + return p + ? jsonResponse(200, { slug: path_params.slug, ...p.meta, markdown: p.markdown }) + : jsonResponse(404, { error: 'not found' }); + }, + HTTP_META('HTTP GET /openwiki/api/wikis/:id/pages/:slug'), +); + +worker.registerFunction( + 'openwiki::http::search', + async ({ path_params, query_params }) => { + const q = query_params?.q || ''; + return jsonResponse(200, await searchPages(path_params?.id, q)); + }, + HTTP_META('HTTP GET /openwiki/api/wikis/:id/search?q='), +); + +worker.registerFunction( + 'openwiki::http::refresh', + async ({ path_params }) => { + const r = await refreshWiki(path_params?.id); + return jsonResponse(200, r); + }, + HTTP_META('HTTP POST /openwiki/api/wikis/:id/refresh'), +); + +worker.registerFunction( + 'openwiki::http::schedule-set', + async ({ path_params, body }) => { + const id = path_params?.id; + const schedule = String(parseBody(body).schedule || 'off'); + if (!SCHEDULE_PRESETS.includes(schedule) && !isRawCron(schedule)) + return jsonResponse(400, { error: 'invalid schedule' }); + const meta = await store.getWiki(id); + if (!meta) return jsonResponse(404, { error: 'not found' }); + await store.saveWiki(id, { ...meta, refresh_schedule: schedule, updated_at: now() }); + applyWikiSchedule(id, schedule); + return jsonResponse(200, { id, schedule, ok: true }); + }, + HTTP_META('HTTP PUT /openwiki/api/wikis/:id/schedule'), +); + +// SSE live progress: stream generation events over the HTTP response channel. +worker.registerFunction( + 'openwiki::http::events', + async (req) => { + const wikiId = req?.path_params?.id; + const w = req?.response; + if (!w?.stream) return jsonResponse(501, { error: 'streaming unsupported' }); + const ctl = (o) => { + try { + w.sendMessage(JSON.stringify(o)); + } catch { + /* ignore */ + } + }; + ctl({ type: 'set_status', status_code: 200 }); + ctl({ + type: 'set_headers', + headers: { + 'content-type': 'text/event-stream', + 'cache-control': 'no-cache', + connection: 'keep-alive', + 'x-accel-buffering': 'no', + }, + }); + + let closed = false; + let off = null; + let hb = null; + const stop = () => { + if (closed) return; + closed = true; + if (hb) clearInterval(hb); + if (off) off(); + try { + w.close(); + } catch { + /* ignore */ + } + }; + // The channel socket can close under us when the worker navigates away. A + // write then emits an ASYNC 'error' event on the Writable — a try/catch around + // write() cannot catch it, and unhandled it crashes the whole worker. Listen + // for it, and never write once closed. + try { + w.stream.on('error', stop); + } catch { + /* ignore */ + } + const write = (s) => { + if (closed) return; + try { + w.stream.write(s); + } catch { + stop(); + } + }; + const send = (evt) => write(`data: ${JSON.stringify(evt)}\n\n`); + + const snap = await store.getStatus(wikiId); + if (snap) send({ kind: 'status', ...snap }); + off = onProgress(wikiId, send); + hb = setInterval(() => write(': ping\n\n'), 15_000); + if (snap && (snap.phase === 'ready' || snap.phase === 'error')) { + send({ kind: 'status', ...snap, final: true }); + stop(); + } + try { + req.request_body?.stream?.on?.('close', stop); + } catch { + /* ignore */ + } + return null; + }, + HTTP_META('HTTP GET /openwiki/api/wikis/:id/events (SSE live progress)'), +); + +worker.registerFunction( + 'openwiki::http::ask', + async ({ path_params, body }) => { + const p = parseBody(body); + if (!p.q) return jsonResponse(400, { error: 'q required' }); + return jsonResponse( + 200, + await askWiki(worker, { id: path_params?.id, q: p.q, mode: p.mode, file_answer: p.file_answer }), + ); + }, + HTTP_META('HTTP POST /openwiki/api/wikis/:id/ask'), +); + +worker.registerFunction( + 'openwiki::http::diagram', + async ({ path_params, query_params }) => { + return jsonResponse(200, await makeDiagram(worker, { id: path_params?.id, kind: query_params?.kind })); + }, + HTTP_META('HTTP GET /openwiki/api/wikis/:id/diagram'), +); + +// ---------- HTTP triggers ---------- +// api_path has NO leading slash: the engine prepends '/', and a leading slash +// double-slashes and 404s. + +function bind(function_id, api_path, http_method = 'GET') { + return worker.registerTrigger({ type: 'http', function_id, config: { api_path, http_method } }); +} +bind('openwiki::http::ui', 'openwiki', 'GET'); +bind('openwiki::http::ui', 'openwiki/', 'GET'); +bind('openwiki::http::wikis-list', 'openwiki/api/wikis', 'GET'); +bind('openwiki::http::models', 'openwiki/api/models', 'GET'); +bind('openwiki::http::wikis-create', 'openwiki/api/wikis', 'POST'); +bind('openwiki::http::wiki-get', 'openwiki/api/wikis/:id', 'GET'); +bind('openwiki::http::wiki-delete', 'openwiki/api/wikis/:id', 'DELETE'); +bind('openwiki::http::wiki-status', 'openwiki/api/wikis/:id/status', 'GET'); +bind('openwiki::http::pages-list', 'openwiki/api/wikis/:id/pages', 'GET'); +bind('openwiki::http::page-get', 'openwiki/api/wikis/:id/pages/:slug', 'GET'); +bind('openwiki::http::search', 'openwiki/api/wikis/:id/search', 'GET'); +bind('openwiki::http::refresh', 'openwiki/api/wikis/:id/refresh', 'POST'); +bind('openwiki::http::schedule-set', 'openwiki/api/wikis/:id/schedule', 'PUT'); +bind('openwiki::http::events', 'openwiki/api/wikis/:id/events', 'GET'); +bind('openwiki::http::ask', 'openwiki/api/wikis/:id/ask', 'POST'); +bind('openwiki::http::diagram', 'openwiki/api/wikis/:id/diagram', 'GET'); + +// ---------- Scheduled refresh (per-wiki cron triggers) ---------- +// Each wiki carries its own auto-refresh cadence, set from the UI, never +// hardcoded. openwiki::set-schedule registers a per-wiki cron trigger; every +// such trigger fires openwiki::cron::refresh-due, which refreshes the wikis whose +// interval has elapsed. Trigger handles live in wikiTriggers so a schedule change +// or delete can unregister the old one; on boot they are re-registered from the +// stored schedules. +const wikiTriggers = new Map(); // wikiId -> unregister fn + +const REFRESH_INTERVALS = { + '3h': { cron: '0 0 */3 * * *', ms: 3 * 3600e3 }, + '6h': { cron: '0 0 */6 * * *', ms: 6 * 3600e3 }, + '12h': { cron: '0 0 */12 * * *', ms: 12 * 3600e3 }, + daily: { cron: '0 0 3 * * *', ms: 24 * 3600e3 }, + weekly: { cron: '0 0 3 * * 1', ms: 7 * 24 * 3600e3 }, +}; +const SCHEDULE_PRESETS = ['off', ...Object.keys(REFRESH_INTERVALS)]; +const isRawCron = (s) => /^(\S+\s+){4,5}\S+$/.test(String(s || '')); + +// Register (or replace) one wiki's cron trigger. 'off' just unregisters. A raw +// 5-6 field cron is accepted for power users; its due window is the trigger +// cadence itself (refresh whenever it fires). +function applyWikiSchedule(wikiId, schedule) { + const prev = wikiTriggers.get(wikiId); + if (prev) { + try { + prev(); + } catch { + /* already gone */ + } + wikiTriggers.delete(wikiId); + } + if (!schedule || schedule === 'off') return; + const cron = REFRESH_INTERVALS[schedule] ? REFRESH_INTERVALS[schedule].cron : schedule; + try { + const { unregister } = worker.registerTrigger({ + type: 'cron', + function_id: 'openwiki::cron::refresh-due', + config: { schedule: cron }, + metadata: { wiki_id: wikiId }, + }); + wikiTriggers.set(wikiId, unregister); + } catch (e) { + console.warn('[openwiki] failed to schedule', wikiId, e?.message || e); + } +} + +// A scheduled cron fired: refresh every wiki whose interval has elapsed. All the +// per-wiki triggers point here; the due check keeps it correct regardless of +// which one fired, and idempotent (content-hash gate + inflight guard). +worker.registerFunction( + 'openwiki::cron::refresh-due', + async () => { + const wikis = await store.listWikis(); + let refreshed = 0; + for (const w of wikis) { + const sched = w.refresh_schedule; + if (!sched || sched === 'off' || inflight.has(w.id)) continue; + const dueMs = REFRESH_INTERVALS[sched]?.ms || 0; // raw cron: due whenever it fires + const last = w.last_refresh_at ? Date.parse(w.last_refresh_at) : 0; + if (Date.now() - last < dueMs - 60_000) continue; // 1-minute slack against cron drift + try { + await refreshWiki(w.id); + refreshed += 1; + } catch (e) { + console.error('[openwiki] scheduled refresh failed', w.id, e?.message); + } + } + return { refreshed }; + }, + { + description: 'Cron: refresh every wiki whose auto-refresh interval has elapsed.', + request_format: { type: 'object', additionalProperties: true, properties: {} }, + response_format: { + type: 'object', + additionalProperties: false, + required: ['refreshed'], + properties: { refreshed: { type: 'integer' } }, + }, + }, +); + +// Set (or clear) a wiki's auto-refresh cadence and (re)register its cron trigger. +worker.registerFunction( + 'openwiki::set-schedule', + async (a) => { + const id = needId(a, 'openwiki::set-schedule'); + const schedule = String(a.schedule || 'off'); + if (!SCHEDULE_PRESETS.includes(schedule) && !isRawCron(schedule)) { + throw err( + 'openwiki/bad_request', + `schedule must be one of ${SCHEDULE_PRESETS.join(', ')} or a 5-6 field cron string`, + ); + } + const meta = await store.getWiki(id); + if (!meta) throw err('openwiki/wiki_not_found', 'wiki not found'); + await store.saveWiki(id, { ...meta, refresh_schedule: schedule, updated_at: now() }); + applyWikiSchedule(id, schedule); + return { id, schedule, ok: true }; + }, + { + description: + 'Set a wiki auto-refresh cadence: off | 3h | 6h | 12h | daily | weekly | a cron string. Params: { id, schedule }.', + request_format: S.SET_SCHEDULE_REQ, + response_format: S.SET_SCHEDULE_RES, + }, +); + +// Real-time page collection: subscribe to the harness's emitted turn-completed +// events and route each to the active generation (turnbus). One broad binding +// (no session filter) sees every turn; unowned roots are a cheap map miss. This +// drives the orchestrator's live per-page progress instead of polling. +worker.registerFunction( + 'openwiki::on-turn-completed', + async (evt) => { + try { + turnbus.deliver(evt?.payload || evt); + } catch (e) { + console.warn('[openwiki] turn-event route failed', e?.message || e); + } + return null; + }, + { + description: 'Internal: routes harness::turn-completed events to the active generation.', + request_format: { type: 'object', additionalProperties: true, properties: {} }, + response_format: { type: 'null' }, + }, +); + +// turn-started drives the "spawned page-writer" line in the live feed. +worker.registerFunction( + 'openwiki::on-turn-started', + async (evt) => { + try { + turnbus.deliverStarted(evt?.payload || evt); + } catch (e) { + console.warn('[openwiki] turn-started route failed', e?.message || e); + } + return null; + }, + { + description: 'Internal: routes harness::turn-started events (sub-agent spawns) to the active generation.', + request_format: { type: 'object', additionalProperties: true, properties: {} }, + response_format: { type: 'null' }, + }, +); + +Promise.resolve() + .then(() => + worker.registerTrigger({ type: 'harness::turn-completed', function_id: 'openwiki::on-turn-completed', config: {} }), + ) + .then(() => + worker.registerTrigger({ type: 'harness::turn-started', function_id: 'openwiki::on-turn-started', config: {} }), + ) + .then(() => { + console.log('[openwiki] subscribed to harness turn events (real-time collection + spawn feed)'); + }) + .catch((e) => console.warn('[openwiki] turn-event subscribe failed; using synchronous fallback:', e?.message || e)); + +// Configuration: register the schema, load the stored value, and hot-reload on +// change. Runs off the boot path so it never delays function registration. +configuration + .registerConfig(worker) + .then(() => configuration.fetchConfig(worker)) + .then((c) => { + cfg = c; + }) + .catch((e) => console.warn('[openwiki] config register failed; using defaults', e?.message || e)); +configuration.bindConfigTrigger(worker, async () => { + cfg = await configuration.fetchConfig(worker); +}); + +// Reap generations orphaned by a restart. On a fresh boot nothing is in-flight, +// so any wiki still flagged generating was interrupted mid-run. Clear the flag, +// reconcile page_count to what actually landed on disk, and mark the status so +// the UI stops showing a forever-running progress panel. The partial pages that +// were written stay browsable. Runs off the boot path. +(async () => { + try { + const wikis = await store.listWikis(); + for (const w of wikis) { + if (!w.generating || inflight.has(w.id)) continue; + const pages = await store.listPages(w.id).catch(() => []); + await store.saveWiki(w.id, { ...w, generating: false, page_count: pages.length, updated_at: now() }); + await store.updateStatus(w.id, { + phase: pages.length ? 'ready' : 'error', + progress: pages.length ? 1 : 0, + message: `Interrupted by a restart with ${pages.length} page(s). Refresh to complete.`, + updated_at: now(), + }); + await store.appendLog(w.id, `Reaped orphaned generation (${pages.length} pages on disk).`); + } + // Cron triggers are in-memory registrations, so re-arm each wiki's stored + // auto-refresh cadence after a restart. + for (const w of wikis) { + if (w.refresh_schedule && w.refresh_schedule !== 'off') applyWikiSchedule(w.id, w.refresh_schedule); + } + } catch (e) { + console.warn('[openwiki] reaper failed', e?.message || e); + } +})(); + +console.log('[openwiki] worker ready — model default =', cfg.model, 'iii url =', III_URL); + +process.on('SIGTERM', async () => { + try { + await worker.shutdown(); + } catch {} + process.exit(0); +}); +process.on('SIGINT', async () => { + try { + await worker.shutdown(); + } catch {} + process.exit(0); +}); diff --git a/openwiki/src/lib/agents_md.mjs b/openwiki/src/lib/agents_md.mjs new file mode 100644 index 000000000..0b8e8ccc1 --- /dev/null +++ b/openwiki/src/lib/agents_md.mjs @@ -0,0 +1,33 @@ +// Produce the AGENTS.md / CLAUDE.md pointer block for a wiki. A coding agent that +// reads this block finds the wiki first and spends less context rediscovering the +// repo (langchain-ai/openwiki's differentiator). The wiki is hosted, not in the +// target repo, so this returns the block to paste rather than writing a file. +import * as store from './store.mjs'; + +function notFound() { + const e = new Error('wiki not found'); + e.code = 'openwiki/wiki_not_found'; + return e; +} + +export function buildAgentsBlock(meta, pages, baseUrl) { + const lines = []; + lines.push('## OpenWiki'); + lines.push(''); + lines.push(`This repository has a generated wiki for **${meta.repo_name}** (${pages.length} pages).`); + lines.push('Read it before exploring the codebase to save context.'); + lines.push(''); + lines.push('Pages:'); + for (const p of pages) lines.push(`- ${p.title || p.slug}${p.category ? ` _(${p.category})_` : ''}`); + lines.push(''); + if (baseUrl) lines.push(`Browse: ${baseUrl}/#/wiki/${meta.id}`); + lines.push(`Ask: \`openwiki::ask { "id": "${meta.id}", "q": "" }\``); + return lines.join('\n'); +} + +export async function exportAgentsMd(_worker, { id, targets, baseUrl }) { + const meta = await store.getWiki(id); + if (!meta) throw notFound(); + const pages = (await store.listPages(id)).map((x) => ({ slug: x.slug, ...x.meta })); + return { content: buildAgentsBlock(meta, pages, baseUrl), targets: targets || ['AGENTS.md', 'CLAUDE.md'] }; +} diff --git a/openwiki/src/lib/ask.mjs b/openwiki/src/lib/ask.mjs new file mode 100644 index 000000000..ea3b8473c --- /dev/null +++ b/openwiki/src/lib/ask.mjs @@ -0,0 +1,217 @@ +// Q&A over a generated wiki. Retrieves the most relevant pages, then synthesises +// a cited answer. Fast mode uses one router completion; deep mode drives the +// harness to explore both the wiki and the clone; both fall back to a heuristic +// answer (stitched page excerpts) so ask works with no provider. A good answer +// can be filed back as a new page so explorations compound into the wiki. +import crypto from 'node:crypto'; +import * as store from './store.mjs'; +import { searchPages } from './search.mjs'; +import { extractAssistantText } from './generate.mjs'; +import { awaitTurn } from './harness.mjs'; + +const ASK_SYSTEM = + 'You are OpenWiki answering a question about a repository using its wiki pages. ' + + 'Answer concisely in Markdown, grounded in the provided pages. Cite page titles and source paths. ' + + 'If the pages do not contain the answer, say so.'; + +function notFound() { + const e = new Error('wiki not found'); + e.code = 'openwiki/wiki_not_found'; + return e; +} + +export function slugify(s) { + return ( + String(s || '') + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-|-$/g, '') + .slice(0, 60) || 'answer' + ); +} + +// First substantive paragraph of a page (skip headings / metadata lines). +export function firstMeaningful(md, cap = 500) { + const lines = String(md || '').split(/\r?\n/); + const out = []; + let started = false; + for (const raw of lines) { + const l = raw.trim(); + if (!started) { + if (!l || l.startsWith('#') || l.startsWith('_') || l.startsWith('>')) continue; + started = true; + out.push(l); + } else { + if (!l || l.startsWith('#')) break; + out.push(l); + } + if (out.join(' ').length > cap) break; + } + return out.join(' ').slice(0, cap); +} + +export function heuristicAnswer(q, blocks) { + if (!blocks.length) return `No wiki pages matched "${q}".`; + const lines = [`The most relevant pages for "${q}":`, '']; + for (const b of blocks) { + lines.push(`### ${b.title || b.slug}`); + if (b.excerpt) lines.push(b.excerpt); + lines.push(''); + } + return lines.join('\n').trim(); +} + +function dedupeCitations(citations) { + const seen = new Set(); + const out = []; + for (const c of citations) { + if (!c?.path) continue; + const key = `${c.path}:${c.start_line || ''}:${c.end_line || ''}`; + if (seen.has(key)) continue; + seen.add(key); + out.push(c); + } + return out; +} + +async function relevantSlugs(id, q, n = 5) { + const results = await searchPages(id, q, { limit: n }); + if (results.length) return results.map((r) => r.slug); + const pages = await store.listPages(id); + return pages.slice(0, n).map((p) => p.slug); +} + +async function askFastLLM(worker, { q, blocks, model }) { + const context = blocks.map((b) => `## ${b.title || b.slug}\n${b.excerpt}`).join('\n\n'); + const res = await worker.trigger({ + function_id: 'router::complete', + payload: { + model, + system_prompt: ASK_SYSTEM, + messages: [{ role: 'user', content: `Question: ${q}\n\nWiki pages:\n${context}` }], + max_output_tokens: 1200, + thinking_level: 'low', + }, + timeoutMs: 120_000, + }); + const text = extractAssistantText(res?.message).trim(); + if (!text) throw new Error('empty answer'); + return text; +} + +// Seed the agent with the pages we already retrieved so it can answer even if it +// never makes a tool call, and spell out the EXACT function signatures — agents +// otherwise guess wiki_id/query/path and every call fails. +function deepAskMessage(id, q, blocks) { + const context = (blocks || []) + .filter((b) => b?.excerpt) + .map((b) => `## ${b.title || b.slug} (slug: ${b.slug})\n${b.excerpt}`) + .join('\n\n'); + return ( + `Question: ${q}\n\n` + + `Wiki id: ${id}\n\n` + + `Relevant wiki pages already retrieved for you:\n${context || '(none matched — use the functions below)'}\n\n` + + `Answer in Markdown, grounded in these pages, citing exact source file paths. ` + + `To dig deeper, call the read functions and ALWAYS pass the wiki id as "id":\n` + + `- openwiki::pages { id }\n` + + `- openwiki::page { id, slug } (slug from openwiki::pages)\n` + + `- openwiki::search { id, q }\n` + + `- openwiki::src::list { id, dir }\n` + + `- openwiki::src::read { id, path }\n` + + `- openwiki::src::grep { id, pattern }` + ); +} + +async function askDeep(worker, { id, q, model, blocks }) { + const { session_id } = await worker.trigger({ + function_id: 'harness::send', + payload: { + message: deepAskMessage(id, q, blocks), + model, + options: { + system_prompt: ASK_SYSTEM, + functions: { + allow: [ + 'openwiki::page', + 'openwiki::pages', + 'openwiki::search', + 'openwiki::src::read', + 'openwiki::src::list', + 'openwiki::src::grep', + ], + }, + max_turns: 12, + }, + }, + timeoutMs: 30_000, + }); + if (!session_id) throw new Error('harness::send returned no session_id'); + const result = await awaitTurn(worker, session_id, { timeoutMs: 240_000 }); + const text = typeof result === 'string' ? result : result?.answer || result?.markdown || ''; + if (!String(text).trim()) throw new Error('empty answer'); + return String(text).trim(); +} + +async function fileAnswer(id, q, answer, citations) { + // Distinct questions can normalize to the same slugify output; a short + // deterministic hash of the exact question keeps their pages separate while + // re-asking the same question still updates its own page. + const qHash = crypto.createHash('sha1').update(String(q)).digest('hex').slice(0, 6); + const slug = `ask-${slugify(q)}-${qHash}`; + const md = `# ${q}\n\n${answer}\n\n_Filed from a question on ${new Date().toISOString()}._\n`; + await store.savePage(id, slug, md, { + title: q.slice(0, 80), + slug, + category: 'answers', + source_paths: [...new Set(citations.map((c) => c.path).filter(Boolean))], + citations, + last_updated: new Date().toISOString(), + confidence: 'medium', + status: 'current', + generator: 'router', + }); + return slug; +} + +export async function askWiki(worker, { id, q, mode = 'fast', file_answer = false, model }) { + const meta = await store.getWiki(id); + if (!meta) throw notFound(); + if (!q || !String(q).trim()) return { answer: '', citations: [] }; + + const slugs = await relevantSlugs(id, q, 5); + const blocks = []; + const citations = []; + for (const slug of slugs) { + const p = await store.getPage(id, slug); + if (!p) continue; + blocks.push({ slug, title: p.meta?.title, excerpt: firstMeaningful(p.markdown, 500) }); + for (const c of p.meta?.citations || []) citations.push(c); + } + + let answer = null; + try { + answer = + mode === 'deep' + ? await askDeep(worker, { id, q, model: model || meta.model, blocks }) + : await askFastLLM(worker, { q, blocks, model: model || meta.model }); + } catch (e) { + console.warn(`[openwiki] ask ${mode} tier failed: ${e?.message || e}`); + answer = null; + } + // Fast mode uses router::complete; if that path is down, try the harness + // (streaming) before dropping to the heuristic stitch. + if (!answer && mode !== 'deep') { + try { + answer = await askDeep(worker, { id, q, model: model || meta.model, blocks }); + } catch (e) { + console.warn(`[openwiki] ask deep fallback failed: ${e?.message || e}`); + answer = null; + } + } + if (!answer) answer = heuristicAnswer(q, blocks); + + const deduped = dedupeCitations(citations); + const out = { answer, citations: deduped }; + if (file_answer) out.filed_slug = await fileAnswer(id, q, answer, deduped); + return out; +} diff --git a/openwiki/src/lib/configuration.mjs b/openwiki/src/lib/configuration.mjs new file mode 100644 index 000000000..c8f4cd905 --- /dev/null +++ b/openwiki/src/lib/configuration.mjs @@ -0,0 +1,102 @@ +// Configuration-worker integration. Registers openwiki's config schema so the +// default model and page-writer concurrency are editable in the console and +// hot-reload on change. Env vars seed the defaults on first registration. +const CONFIG_ID = 'openwiki'; +const CONFIG_FN_ID = 'openwiki::on-config-change'; + +// Sanitize env seeds against the declared schema: a NaN or out-of-range +// max_parallel, or an unknown refresh cadence, would make the registered +// defaults/initial_value violate the schema itself. +const REFRESH_VALUES = ['off', '3h', '6h', '12h', 'daily', 'weekly']; +const rawParallel = parseInt(process.env.OPENWIKI_MAX_PARALLEL || '3', 10); +const envRefresh = process.env.OPENWIKI_REFRESH_DEFAULT || 'off'; + +const DEFAULTS = { + model: process.env.OPENWIKI_MODEL || 'claude-haiku-4-5-20251001', + max_parallel: Math.min(16, Math.max(1, Number.isFinite(rawParallel) ? rawParallel : 3)), + refresh_default: REFRESH_VALUES.includes(envRefresh) ? envRefresh : 'off', +}; + +function schema() { + return { + type: 'object', + additionalProperties: false, + properties: { + model: { + type: 'string', + description: 'Default generation model id, routed via llm-router (e.g. claude-haiku-4-5-20251001).', + default: DEFAULTS.model, + }, + max_parallel: { + type: 'integer', + minimum: 1, + maximum: 16, + description: 'Concurrent page writers per generation.', + default: DEFAULTS.max_parallel, + }, + refresh_default: { + type: 'string', + enum: ['off', '3h', '6h', '12h', 'daily', 'weekly'], + description: + 'Default auto-refresh cadence for new wikis. Each wiki can override it in the UI. "off" means no scheduled refresh.', + default: DEFAULTS.refresh_default, + }, + }, + }; +} + +export function defaults() { + return { ...DEFAULTS }; +} + +export async function registerConfig(iii) { + await iii.trigger({ + function_id: 'configuration::register', + payload: { + id: CONFIG_ID, + name: 'OpenWiki', + description: 'OpenWiki worker: default model, page-writer concurrency, and auto-refresh cadence.', + schema: schema(), + initial_value: DEFAULTS, + }, + }); +} + +export async function fetchConfig(iii) { + try { + const res = await iii.trigger({ function_id: 'configuration::get', payload: { id: CONFIG_ID, raw: false } }); + const v = res && typeof res === 'object' && 'value' in res ? res.value : res; + return { ...DEFAULTS, ...(v || {}) }; + } catch { + return { ...DEFAULTS }; + } +} + +export function bindConfigTrigger(iii, onChange) { + iii.registerFunction( + CONFIG_FN_ID, + async () => { + await onChange(); + return { reloaded: true }; + }, + { + description: 'Reload runtime config when the openwiki configuration entry changes.', + request_format: { type: 'object', additionalProperties: true, properties: {} }, + response_format: { + type: 'object', + additionalProperties: false, + required: ['reloaded'], + properties: { reloaded: { type: 'boolean' } }, + }, + }, + ); + try { + iii.registerTrigger({ + type: 'configuration', + function_id: CONFIG_FN_ID, + config: { configuration_id: CONFIG_ID, event_types: ['configuration:updated'] }, + }); + } catch { + /* configuration worker may be absent; env defaults apply */ + } +} diff --git a/openwiki/src/lib/diagram.mjs b/openwiki/src/lib/diagram.mjs new file mode 100644 index 000000000..5baff4727 --- /dev/null +++ b/openwiki/src/lib/diagram.mjs @@ -0,0 +1,93 @@ +// Mermaid diagram generation for a wiki. Tries a router completion for a rich +// architecture/dataflow/deps diagram; falls back to a deterministic category -> +// pages flowchart derived from the wiki structure, so it works with no provider. +import * as store from './store.mjs'; +import { extractAssistantText } from './generate.mjs'; + +function notFound() { + const e = new Error('wiki not found'); + e.code = 'openwiki/wiki_not_found'; + return e; +} + +// Escape a mermaid node label (labels live inside "..."; quotes/brackets break it). +export function escLabel(s) { + return String(s || '') + .replace(/"/g, "'") + .replace(/[[\]{}|<>]/g, ' ') + .replace(/\s+/g, ' ') + .trim() + .slice(0, 60); +} + +export function heuristicMermaid(meta, pages) { + const lines = ['flowchart TD', ` ROOT["${escLabel(meta.repo_name || 'repository')}"]`]; + const cats = meta.categories || []; + const byCat = new Map(); + for (const p of pages) { + const c = p.category || 'uncategorized'; + if (!byCat.has(c)) byCat.set(c, []); + byCat.get(c).push(p); + } + let ci = 0; + for (const [cid, ps] of byCat) { + const cnode = `C${ci++}`; + const ctitle = cats.find((c) => c.id === cid)?.title || cid; + lines.push(` ${cnode}["${escLabel(ctitle)}"]`); + lines.push(` ROOT --> ${cnode}`); + let pi = 0; + for (const p of ps.slice(0, 8)) { + const pnode = `${cnode}_${pi++}`; + lines.push(` ${pnode}["${escLabel(p.title || p.slug)}"]`); + lines.push(` ${cnode} --> ${pnode}`); + } + } + return lines.join('\n'); +} + +const VALID = /^\s*(flowchart|graph|sequenceDiagram|classDiagram|erDiagram|stateDiagram)/; + +function stripFences(s) { + const m = String(s || '').match(/```(?:mermaid)?\s*\n?([\s\S]*?)\n?```/i); + return (m ? m[1] : String(s || '')).trim(); +} + +async function llmMermaid(worker, { meta, pages, kind, model }) { + const pageList = pages + .map((p) => `- ${p.title || p.slug} [${p.category || ''}]: ${(p.source_paths || []).slice(0, 4).join(', ')}`) + .join('\n'); + const res = await worker.trigger({ + function_id: 'router::complete', + payload: { + model, + system_prompt: + 'You draw Mermaid diagrams of software repositories. Output ONLY a valid Mermaid diagram, no prose, no code fences.', + messages: [ + { + role: 'user', + content: `Repository: ${meta.repo_name}\nDraw a ${kind} diagram (Mermaid flowchart) from these wiki pages and their source files:\n${pageList}`, + }, + ], + max_output_tokens: 900, + thinking_level: 'low', + }, + timeoutMs: 120_000, + }); + return stripFences(extractAssistantText(res?.message)); +} + +export async function makeDiagram(worker, { id, kind = 'architecture', model }) { + const meta = await store.getWiki(id); + if (!meta) throw notFound(); + const pages = (await store.listPages(id)).map((x) => ({ slug: x.slug, ...x.meta })); + + let mermaid = null; + try { + mermaid = await llmMermaid(worker, { meta, pages, kind, model: model || meta.model }); + } catch { + mermaid = null; + } + if (!mermaid || !VALID.test(mermaid)) mermaid = heuristicMermaid(meta, pages); + + return { mermaid, kind }; +} diff --git a/openwiki/src/lib/docs_oracle.mjs b/openwiki/src/lib/docs_oracle.mjs new file mode 100644 index 000000000..b0a8862b7 --- /dev/null +++ b/openwiki/src/lib/docs_oracle.mjs @@ -0,0 +1,134 @@ +// Official-docs oracle. A project's own documentation index is the strongest +// information-architecture signal (vercel-labs/openwiki's biggest quality +// lever). We look for an llms.txt at the doc sites the README links to, and +// fall back to the repo's docs/ tree, then derive a nav hint + page budget the +// planner uses to build breadth comparable to the official docs. +import fs from 'node:fs/promises'; +import path from 'node:path'; + +// Parse an llms.txt: markdown with section headings and "- [title](url)" links. +export function parseLlmsTxt(text) { + const sections = []; + let current = null; + const links = []; + for (const raw of String(text || '').split(/\r?\n/)) { + const h = raw.match(/^#{1,3}\s+(.+)$/); + if (h) { + current = { title: h[1].replace(/[#*`]/g, '').trim(), links: [] }; + sections.push(current); + continue; + } + const l = raw.match(/^\s*[-*]\s+\[([^\]]+)\]\(([^)\s]+)/); + if (l) { + const item = { title: l[1].trim(), url: l[2] }; + links.push(item); + if (current) current.links.push(item); + } + } + return { sections: sections.filter((s) => s.links.length), links }; +} + +export function candidateOrigins(_repoUrl, readme) { + const origins = new Set(); + for (const u of String(readme || '').match(/https?:\/\/[^\s)\]]+/g) || []) { + try { + const o = new URL(u); + if (/github\.com|githubusercontent|shields\.io|npmjs\.com|badge|codecov|circleci/i.test(o.hostname)) continue; + origins.add(o.origin); + } catch { + /* skip bad url */ + } + } + return [...origins].slice(0, 4); +} + +async function webFetch(worker, url, timeoutMs = 15_000) { + try { + const res = await worker.trigger({ + function_id: 'web::fetch', + payload: { url, format: 'markdown' }, + timeoutMs, + }); + const status = res?.status ?? res?.status_code ?? 200; + const body = res?.content ?? res?.body ?? res?.markdown ?? res?.text ?? ''; + if (status >= 400 || !body) return null; + return String(body); + } catch { + return null; + } +} + +export async function fetchDocsIndex(worker, { repoUrl, readme, repoDir }) { + // Overall budget across every probe: up to 8 sequential fetches against + // origins that may all be black holes must not stall planning for minutes. + const deadline = Date.now() + 45_000; + // 1) llms.txt at README-referenced documentation origins. + outer: for (const origin of candidateOrigins(repoUrl, readme)) { + for (const p of ['/llms.txt', '/llms-full.txt']) { + const remaining = deadline - Date.now(); + if (remaining <= 0) break outer; + const txt = await webFetch(worker, origin + p, Math.min(15_000, remaining)); + if (txt && /\]\(https?:/.test(txt)) { + const parsed = parseLlmsTxt(txt); + if (parsed.links.length >= 5) + return { source: origin + p, linkCount: parsed.links.length, sections: parsed.sections }; + } + } + } + // 2) the repo's own docs/ tree. + try { + const docsDir = path.join(repoDir, 'docs'); + const st = await fs.stat(docsDir).catch(() => null); + if (st?.isDirectory()) { + const files = []; + const walk = async (d, rel = '') => { + if (files.length > 400) return; + for (const e of await fs.readdir(d, { withFileTypes: true })) { + if (files.length > 400) return; + if (e.isDirectory()) await walk(path.join(d, e.name), `${rel + e.name}/`); + else if (/\.mdx?$/i.test(e.name)) files.push(rel + e.name); + } + }; + await walk(docsDir); + if (files.length >= 5) { + const groups = {}; + for (const f of files) { + const g = f.includes('/') ? f.split('/')[0] : 'docs'; + (groups[g] = groups[g] || []).push(f); + } + const sections = Object.entries(groups).map(([title, fs2]) => ({ + title, + links: fs2.map((f) => ({ title: f, url: f })), + })); + return { source: 'docs/', linkCount: files.length, sections }; + } + } + } catch { + /* no docs tree */ + } + return null; +} + +// Map a discovered docs index to a page budget (vercel's adaptive ladder). +export function docsBudget(linkCount) { + if (linkCount >= 260) return 48; + if (linkCount >= 160) return 40; + if (linkCount >= 90) return 34; + if (linkCount >= 40) return 26; + return 18; +} + +// A compact hint injected into the plan prompt. +export function docsHint(docsIndex) { + if (!docsIndex) return ''; + const titles = docsIndex.sections + .map((s) => s.title) + .filter(Boolean) + .slice(0, 12); + return ( + `\n\nOfficial documentation index discovered (${docsIndex.linkCount} topics, source: ${docsIndex.source}).\n` + + 'Treat this as the STRONGEST information-architecture signal. Preserve its major sections as top-level nav folders and give the wiki comparable breadth.\n' + + (titles.length ? `Major sections to mirror: ${titles.join(', ')}.\n` : '') + + `Target about ${docsBudget(docsIndex.linkCount)} pages.` + ); +} diff --git a/openwiki/src/lib/generate.mjs b/openwiki/src/lib/generate.mjs new file mode 100644 index 000000000..18c9a4e9c --- /dev/null +++ b/openwiki/src/lib/generate.mjs @@ -0,0 +1,265 @@ +// src/lib/generate.mjs — LLM planning + page generation for OpenWiki +import { readFile } from 'node:fs/promises'; +import path from 'node:path'; + +const PLAN_SYSTEM = + 'You are OpenWiki, an expert technical writer that plans documentation wikis for source code repositories. Your task is to design a small, reader-facing wiki (6–12 pages) organized by category. NEVER invent source paths. Every outline item must cite ≥1 real path from the provided inventory. Prefer conceptual pages that stitch together multiple files over one-page-per-file docs. Output STRICT JSON only.'; + +const PAGE_SYSTEM = + 'You are OpenWiki, a source-grounded wiki maintainer. Write ONE Markdown wiki page. Rules:\n' + + '- Start with a level-1 heading matching the given title.\n' + + '- Include an "Overview" section (2–4 sentences).\n' + + '- Add topic-appropriate sections (architecture, usage, key files, workflow, notes, etc.).\n' + + '- Cite source paths inline using backtick code spans, e.g. `src/index.ts`.\n' + + '- Every substantive claim must be grounded in the provided source files. If a claim would require reading a file you were not shown, mark it as “needs-review” instead of guessing.\n' + + '- Link to sibling pages using relative Markdown links: [Title](./other-slug.md).\n' + + '- End with a "Sources" section listing every source_path with one line of what to look at.\n' + + '- Be concise (≤ 400 lines). Prefer clarity over completeness.\n' + + '- Do NOT emit YAML frontmatter — the wrapper will add it.'; + +export function extractAssistantText(message) { + const c = message?.content; + if (Array.isArray(c)) { + return c + .filter((b) => b && b.type === 'text') + .map((b) => b.text ?? '') + .join(''); + } + return String(message?.content ?? ''); +} + +export function parseJson(text) { + let s = String(text ?? '').trim(); + // Strip ```json ... ``` or ``` ... ``` fences + const fence = s.match(/^```(?:json)?\s*\n?([\s\S]*?)\n?```\s*$/i); + if (fence) s = fence[1].trim(); + try { + return JSON.parse(s); + } catch (_e) { + const snip = s.slice(0, 200).replace(/\s+/g, ' '); + throw new Error(`Model returned invalid JSON: ${snip}`); + } +} + +function extFromPath(p) { + const m = String(p ?? '').match(/\.([A-Za-z0-9]+)$/); + return m ? m[1].toLowerCase() : 'text'; +} + +async function buildKeyDocs(inventory, repoDir) { + const CAP = 40_000; + const PER = 8_000; + const docs = inventory.filter((e) => e.isDoc === true && (e.priority ?? 0) >= 2); + let total = 0; + const parts = []; + for (const e of docs) { + if (total >= CAP) break; + let content = ''; + try { + content = await readFile(path.join(repoDir, e.relPath), 'utf8'); + } catch { + continue; + } + if (content.length > PER) content = `${content.slice(0, PER)}\n...[truncated]`; + const chunk = `### ${e.relPath}\n${content}\n`; + if (total + chunk.length > CAP) { + parts.push(chunk.slice(0, CAP - total)); + total = CAP; + break; + } + parts.push(chunk); + total += chunk.length; + } + return parts.join(''); +} + +function buildFileTree(inventory) { + const sorted = [...inventory].sort((a, b) => (b.priority ?? 0) - (a.priority ?? 0)); + return sorted + .slice(0, 200) + .map((e) => `PRIORITY ${e.priority ?? 0} ${e.language ?? ''} ${e.size ?? 0}b ${e.relPath}`) + .join('\n'); +} + +async function planWikiLLM(worker, { inventory, repoName, repoUrl, repoDir, model = 'claude-sonnet-4-6' }) { + const fileTree = buildFileTree(inventory); + const keyDocs = await buildKeyDocs(inventory, repoDir); + + const userContent = + 'Plan a wiki for the repository below. Return JSON with this exact shape:\n' + + '{\n' + + ' "summary": string, // 2–4 sentences describing what the repo is\n' + + ' "categories": [ { "id": kebab-case, "title": string, "description": string } ], // 3–7 categories\n' + + ' "outline": [ { "slug": kebab-case, "title": string, "category": category.id, "source_paths": [relPath,...], "brief": string } ] // 6–12 items, each with ≥1 source_path drawn from the file tree\n' + + '}\n\n' + + 'Typical categories: overview, architecture, api, workflows, data-model, integrations, operations, decisions.\n' + + 'Ensure every outline item has category ∈ categories[].id.\n\n' + + '---\nREPO: ' + + repoName + + ' (' + + repoUrl + + ')\n\n' + + 'FILE TREE (priority desc):\n' + + fileTree + + '\n\n' + + 'KEY DOCS:\n' + + keyDocs; + + const res = await worker.trigger({ + function_id: 'router::complete', + payload: { + model, + system_prompt: PLAN_SYSTEM, + messages: [{ role: 'user', content: userContent }], + max_output_tokens: 4000, + thinking_level: 'medium', + }, + timeoutMs: 120_000, + }); + + const text = extractAssistantText(res?.message); + const parsed = parseJson(text); + + const summary = String(parsed.summary ?? '').trim(); + const categories = Array.isArray(parsed.categories) ? parsed.categories : []; + let outline = Array.isArray(parsed.outline) ? parsed.outline : []; + + if (!summary) throw new Error('planWiki: missing summary'); + if (categories.length < 1) throw new Error('planWiki: missing categories'); + + const catIds = new Set(categories.map((c) => c.id)); + const invPaths = new Set(inventory.map((e) => e.relPath)); + + outline = outline.filter((item) => { + if (!item || typeof item !== 'object') return false; + if (!item.slug || !item.title || !item.category) { + console.warn('[planWiki] dropping outline item missing required fields:', item?.slug ?? ''); + return false; + } + if (!catIds.has(item.category)) { + console.warn(`[planWiki] dropping "${item.slug}": unknown category "${item.category}"`); + return false; + } + const paths = Array.isArray(item.source_paths) ? item.source_paths : []; + const real = paths.filter((p) => invPaths.has(p)); + const dropped = paths.filter((p) => !invPaths.has(p)); + if (dropped.length) { + console.warn(`[planWiki] "${item.slug}": pruning non-existent source_paths:`, dropped); + } + if (real.length === 0) { + console.warn(`[planWiki] dropping "${item.slug}": no valid source_paths remain`); + return false; + } + item.source_paths = real; + return true; + }); + + if (outline.length < 3) { + throw new Error(`planWiki: only ${outline.length} valid outline items after validation (need ≥3)`); + } + + return { summary, categories, outline }; +} + +async function generatePageLLM( + worker, + { outlineItem, sourceReads, allSlugs, allTitles, categories, repoName, repoUrl, model = 'claude-sonnet-4-6' }, +) { + const cat = (categories ?? []).find((c) => c.id === outlineItem.category); + const categoryTitle = cat ? cat.title : outlineItem.category; + + const siblings = (allSlugs ?? []).map((s, i) => `- ${s} — ${(allTitles ?? [])[i] ?? ''}`).join('\n'); + + const sourceBlocks = (sourceReads ?? []) + .map( + (sr) => + `\n### FILE: ${sr.path}\n\n\`\`\`${extFromPath(sr.path)}\n${sr.content}${sr.truncated ? '\n...[truncated]' : ''}\n\`\`\``, + ) + .join('\n'); + + const userContent = + 'Repository: ' + + repoName + + ' (' + + repoUrl + + ')\n' + + 'Category: ' + + categoryTitle + + '\n' + + 'Page title: ' + + outlineItem.title + + '\n' + + 'Page slug: ' + + outlineItem.slug + + '\n' + + 'Brief: ' + + outlineItem.brief + + '\n\n' + + 'Sibling pages you may link to (slug — title):\n' + + siblings + + '\n\n' + + 'SOURCE FILES (verbatim, may be truncated):\n' + + sourceBlocks; + + const res = await worker.trigger({ + function_id: 'router::complete', + payload: { + model, + system_prompt: PAGE_SYSTEM, + messages: [{ role: 'user', content: userContent }], + max_output_tokens: 3500, + thinking_level: 'low', + }, + timeoutMs: 120_000, + }); + + let markdown = extractAssistantText(res?.message).trim(); + const wrap = markdown.match(/^```(?:markdown|md)?\s*\n([\s\S]*?)\n```\s*$/i); + if (wrap) markdown = wrap[1].trim(); + + const frontmatter = { + title: outlineItem.title, + slug: outlineItem.slug, + category: outlineItem.category, + source_paths: outlineItem.source_paths, + last_updated: new Date().toISOString(), + confidence: 'medium', + status: 'current', + }; + + return { markdown, frontmatter }; +} + +// ------- LLM+heuristic wrappers ------- +import { planWikiHeuristic, generatePageHeuristic } from './heuristic.mjs'; + +export async function planWiki(worker, opts) { + try { + return await planWikiLLM(worker, opts); + } catch (e) { + console.warn(`[openwiki] planWiki LLM failed (${e?.message || e}); using heuristic fallback`); + return await planWikiHeuristic({ + inventory: opts.inventory, + repoName: opts.repoName, + repoUrl: opts.repoUrl, + repoDir: opts.repoDir, + }); + } +} + +export async function generatePage(worker, opts) { + try { + const out = await generatePageLLM(worker, opts); + const md = String(out?.markdown || '').trim(); + if (!md) throw new Error('LLM returned empty markdown'); + return out; + } catch (e) { + console.warn( + '[openwiki] generatePage LLM failed (' + + (e?.message || e) + + '); using heuristic fallback for ' + + opts?.outlineItem?.slug, + ); + return await generatePageHeuristic(opts); + } +} diff --git a/openwiki/src/lib/git.mjs b/openwiki/src/lib/git.mjs new file mode 100644 index 000000000..543c23319 --- /dev/null +++ b/openwiki/src/lib/git.mjs @@ -0,0 +1,138 @@ +// Git access for openwiki. Routes through the `shell` worker (`shell::exec`) so +// cloning, HEAD reads, and diffs run on the iii bus like everything else. Falls +// back to a local child_process when the shell worker is absent, so the worker +// still runs standalone. +// +// worktree deployment note: the `shell` worker jails exec/fs under +// `fs.host_roots`. The clone directory (OPENWIKI_DATA/repos/, default +// /tmp/openwiki-data) MUST resolve inside those roots, or shell::exec refuses +// the clone (S215). Point OPENWIKI_DATA inside host_roots, or run without the +// shell worker to use the local-git fallback. +import { spawn } from 'node:child_process'; + +function runLocal(cmd, args, opts = {}) { + return new Promise((resolve, reject) => { + const child = spawn(cmd, args, { ...opts }); + let stdout = ''; + let stderr = ''; + child.stdout?.on('data', (d) => { + stdout += d.toString(); + }); + child.stderr?.on('data', (d) => { + stderr += d.toString(); + }); + child.on('error', reject); + child.on('close', (code) => { + if (code === 0) resolve({ stdout, stderr }); + else reject(new Error(`${cmd} ${args.join(' ')} exited ${code}: ${stderr.trim()}`)); + }); + }); +} + +// Run a command via shell::exec; fall back to local child_process when the +// shell worker is unreachable. A non-zero exit is surfaced (never falls back — +// that would double-run side effects like a clone). +async function sh(worker, command, args, { cwd, timeoutMs } = {}) { + if (worker) { + let res = null; + try { + res = await worker.trigger({ + function_id: 'shell::exec', + payload: { + command, + args, + ...(cwd ? { cwd } : {}), + ...(timeoutMs ? { timeout_ms: timeoutMs } : {}), + }, + timeoutMs: (timeoutMs || 120_000) + 10_000, + }); + } catch { + res = null; // shell worker unreachable -> local fallback + } + if (res != null) { + // Reached the worker. Never fall back from here: a fallback could + // double-run side effects (a second clone). Surface the outcome. + if (typeof res.exit_code === 'number') { + if (res.exit_code === 0) return { stdout: res.stdout || '', stderr: res.stderr || '' }; + throw new Error(`${command} ${args.join(' ')} exited ${res.exit_code}: ${String(res.stderr || '').trim()}`); + } + throw new Error(`${command} ${args.join(' ')}: shell::exec returned no exit_code`); + } + } + return runLocal(command, args, { cwd }); +} + +export function repoName(repoUrl) { + const u = String(repoUrl) + .trim() + .replace(/\.git$/, '') + .replace(/\/+$/, ''); + const parts = u.split(/[/:]/).filter(Boolean); + if (parts.length >= 2) return parts.slice(-2).join('/'); + return parts[parts.length - 1] || u; +} + +// Guard the values that reach git's argv. A leading "-" would be parsed as an +// option (e.g. `--upload-pack=...`), and transports like ext:: execute +// arbitrary commands. Allow only http(s)/git/ssh URLs and scp-like +// user@host:path remotes; end option parsing with "--" before the positionals. +const URL_SCHEME_RE = /^(?:https?|git|ssh):\/\/\S+$/i; +const SCP_LIKE_RE = /^[\w.-]+@[\w.-]+:[^\s-][^\s]*$/; + +function assertCloneArgs(repoUrl, ref) { + const u = String(repoUrl || '').trim(); + if (!u || u.startsWith('-') || !(URL_SCHEME_RE.test(u) || SCP_LIKE_RE.test(u))) { + throw new Error(`invalid repo_url: expected an http(s)/git/ssh URL, got "${u.slice(0, 100)}"`); + } + if (ref != null && ref !== '') { + const r = String(ref).trim(); + if (!r || r.startsWith('-') || /\s/.test(r)) throw new Error(`invalid ref: "${r.slice(0, 100)}"`); + } +} + +export async function cloneRepo(worker, repoUrl, destDir, ref) { + assertCloneArgs(repoUrl, ref); + const args = ['clone', '--depth', '1']; + if (ref) args.push('--branch', String(ref).trim()); + args.push('--', String(repoUrl).trim(), destDir); + await sh(worker, 'git', args, { timeoutMs: 120_000 }); + const { stdout } = await sh(worker, 'git', ['-C', destDir, 'rev-parse', 'HEAD']); + return { commit: stdout.trim(), dir: destDir, name: repoName(repoUrl), url: repoUrl }; +} + +export async function currentCommit(worker, dir) { + const { stdout } = await sh(worker, 'git', ['-C', dir, 'rev-parse', 'HEAD']); + return stdout.trim(); +} + +// Best-effort update of an existing clone. Returns the new HEAD, or null on +// failure (caller decides whether to re-clone). +export async function gitPull(worker, dir) { + try { + // No --depth here: a depth-1 fetch would detach the new HEAD from the + // previously recorded commit and starve the refresh diff of history. + await sh(worker, 'git', ['-C', dir, 'fetch', 'origin'], { timeoutMs: 120_000 }); + await sh(worker, 'git', ['-C', dir, 'reset', '--hard', 'FETCH_HEAD']); + return await currentCommit(worker, dir); + } catch { + return null; + } +} + +export async function gitDiff(worker, dir, baseRef, headRef = 'HEAD') { + const { stdout } = await sh(worker, 'git', ['-C', dir, 'diff', '--name-status', `${baseRef}..${headRef}`]); + return parseNameStatus(stdout); +} + +export function parseNameStatus(stdout) { + const out = []; + for (const line of String(stdout).split('\n')) { + if (!line.trim()) continue; + const parts = line.split(/\t/); + const status = parts[0][0]; + if (!'AMDR'.includes(status)) continue; + const p = status === 'R' ? parts[2] : parts[1]; + if (p) out.push({ status, path: p }); + } + return out; +} diff --git a/openwiki/src/lib/harness.mjs b/openwiki/src/lib/harness.mjs new file mode 100644 index 000000000..14e3c572b --- /dev/null +++ b/openwiki/src/lib/harness.mjs @@ -0,0 +1,468 @@ +// Harness-backed page generation. Instead of feeding pre-selected source files +// to one completion (generate.mjs), this drives the `harness` worker: the agent +// explores the checked-out repo through openwiki's scoped read functions +// (openwiki::src::*) and returns a validated page via a JSON output contract. +// Falls back to the router/heuristic path (generate.mjs) when the harness is +// absent or a turn fails. +import { generatePage } from './generate.mjs'; +import { resolveModel } from './model.mjs'; +import { getPageQualityIssues, pageRepairFeedback } from './quality.mjs'; +import { PAGE_HARNESS_OUT, PLAN_HARNESS_OUT } from './schemas.mjs'; + +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); + +// Build a GitHub blob deep-link at the pinned commit. Returns null for non- +// GitHub hosts or when the commit is unknown. +export function citationUrl(repoUrl, commit, path, from, to) { + if (!commit || !path) return null; + const m = String(repoUrl || '') + .replace(/\.git$/, '') + .match(/github\.com[:/]+([^/]+)\/([^/]+)/i); + if (!m) return null; + const owner = m[1]; + const repo = m[2].replace(/\/+$/, ''); + let frag = ''; + if (from) frag = `#L${from}${to && to !== from ? `-L${to}` : ''}`; + // Encode each path segment (spaces, #, ?) but keep the slash separators. + const safePath = String(path).split('/').map(encodeURIComponent).join('/'); + return `https://github.com/${owner}/${repo}/blob/${commit}/${safePath}${frag}`; +} + +// Slice a 1-indexed inclusive line window out of a file's content. +export function lineWindow(content, from, to) { + const lines = String(content ?? '').split(/\r?\n/); + const total = lines.length; + if (!from && !to) return { text: String(content ?? ''), from: 1, to: total, total_lines: total, truncated: false }; + const a = Math.max(1, from || 1); + const b = Math.min(total, to || total); + return { text: lines.slice(a - 1, b).join('\n'), from: a, to: b, total_lines: total, truncated: a > 1 || b < total }; +} + +const PAGE_SYSTEM = + 'You are OpenWiki, a source-grounded technical writer producing ONE page of a repository wiki.\n' + + 'Explore the repository with these functions (always pass the given wiki id):\n' + + '- openwiki::src::list { id, dir? } — the file tree (path, language, priority).\n' + + '- openwiki::src::read { id, path, from?, to? } — read a file or a line window.\n' + + '- openwiki::src::grep { id, pattern } — search file contents.\n' + + 'Read the relevant source BEFORE writing. Ground every claim in real code; never invent files, APIs, or behavior.\n\n' + + 'Write a substantial, well-structured page:\n' + + '- Start with a single "# Title" heading.\n' + + '- Include AT LEAST 3 "##" sections, chosen as the evidence supports: Purpose and Scope, Relevant Source Files,\n' + + ' System-to-Code Mapping, Core Concepts, Execution Flow, Key Types and Interfaces, Configuration,\n' + + ' Extension Points, Things to Watch When Editing, Testing Signals.\n' + + '- ALWAYS include a "## Relevant Source Files" section: bullets naming each key file and one line on why it matters.\n' + + '- Explain WHY the code is shaped this way, not only what it does.\n' + + '- Ground concrete claims with visible "Sources: path/a.ts, path/b.ts" lines in the prose (copy paths exactly).\n' + + '- Where a picture aids understanding (architecture, data flow, execution flow, a state machine, a class or module\n' + + ' relationship), embed a Mermaid diagram INLINE as a ```mermaid fenced code block, placed in the relevant section.\n' + + ' Keep each diagram small and valid (flowchart/sequenceDiagram/classDiagram). Do not add a diagram just to have one.\n' + + '- Aim for 400-900 words of real explanation (code blocks and bare paths do not count).\n' + + '- Link sibling pages inline as [Title](./slug.md).\n\n' + + 'Return JSON matching the schema: { title, markdown, citations, links, confidence, status }.\n' + + '- markdown: the page body only (no YAML frontmatter).\n' + + '- citations: exact { path, start_line, end_line, note } for the code each claim rests on; paths must be files you read.\n' + + '- If a claim cannot be verified from source, set status to "needs-review".'; + +function buildUserPrompt({ + wikiId, + outlineItem, + repoName, + repoUrl, + categories, + allSlugs, + allTitles, + feedback, + previousMarkdown, +}) { + const cat = (categories || []).find((c) => c.id === outlineItem.category); + const categoryTitle = cat ? cat.title : outlineItem.category; + const siblings = (allSlugs || []).map((s, i) => `- ${s} — ${(allTitles || [])[i] || ''}`).join('\n'); + let prompt = + `Repository: ${repoName} (${repoUrl})\n` + + `Wiki id (pass as "id" to every openwiki::src::* call): ${wikiId}\n` + + `Category: ${categoryTitle}\n` + + `Page title: ${outlineItem.title}\n` + + `Page slug: ${outlineItem.slug}\n` + + `Brief: ${outlineItem.brief || ''}\n` + + `Suggested starting files: ${(outlineItem.source_paths || []).join(', ') || '(discover via openwiki::src::list)'}\n\n` + + `Sibling pages you may link to (slug — title):\n${siblings}\n\n` + + 'Explore the source, then return the page JSON.'; + if (feedback) { + prompt += `\n\n${feedback}`; + if (previousMarkdown) + prompt += `\n\nYour previous draft (improve it, keep the accurate parts):\n${previousMarkdown.slice(0, 3500)}`; + } + return prompt; +} + +// One harness turn for a page. Runs in a named child session titled with the +// page, linked to the plan session via metadata.parent_session_id — the same +// linkage shape harness uses for real sub-agents, so the console renders each +// page as a titled child under the wiki's plan session (not an opaque s_… id). +// harness::spawn is not used here: a direct spawn call has no parent (it links +// only when dispatched from inside a running turn), so it can neither name nor +// nest. send + SessionInit does both. +async function runPageTurn( + worker, + { + message, + model, + provider, + parentSessionId, + childSessionId, + title, + options, + timeoutMs, + outlineItem, + repoUrl, + commit, + }, +) { + const payload = { message, model, ...(provider ? { provider } : {}), options }; + if (childSessionId) { + payload.session_id = childSessionId; + payload.session = { + title: title || outlineItem.title, + ...(parentSessionId ? { metadata: { parent_session_id: parentSessionId, depth: 1 } } : {}), + }; + } + const r = await worker.trigger({ function_id: 'harness::send', payload, timeoutMs: 30_000 }); + const sessionId = r?.session_id; + if (!sessionId) throw new Error('harness::send returned no session_id'); + const result = await awaitTurn(worker, sessionId, { timeoutMs }); + return mapResult(result, { outlineItem, repoUrl, commit }); +} + +// A session id is any stable string; the console shows the title, not the id. +// Keep ids readable and greppable so a whole wiki's turns share a prefix. +export function sessionSlug(s) { + return String(s || '') + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-|-$/g, '') + .slice(0, 48); +} +export function wikiParentSession(repoName, wikiId) { + return `openwiki:${sessionSlug(repoName) || 'repo'}:${String(wikiId || '').slice(0, 6)}`; +} + +// Poll harness::status until the turn is terminal; return its output-contract +// result. Throws on failure/timeout (best-effort stop on timeout). +export async function awaitTurn(worker, session_id, { timeoutMs = 240_000, intervalMs = 1500 } = {}) { + const deadline = Date.now() + timeoutMs; + for (;;) { + let s; + try { + s = await worker.trigger({ function_id: 'harness::status', payload: { session_id } }); + } catch (e) { + throw new Error(`harness::status failed: ${e?.message || e}`); + } + const status = s?.status; + if (status === 'completed') return s.result; + if (status === 'failed' || status === 'cancelled') { + throw new Error(`harness turn ${status}${s?.result_error ? `: ${s.result_error}` : ''}`); + } + if (Date.now() > deadline) { + try { + await worker.trigger({ function_id: 'harness::stop', payload: { session_id } }); + } catch { + /* best effort */ + } + throw new Error('harness turn timed out'); + } + await sleep(intervalMs); + } +} + +export function mapResult(result, { outlineItem, repoUrl, commit }) { + if (!result || typeof result !== 'object') throw new Error('harness returned no result'); + const markdown = String(result.markdown || '').trim(); + if (!markdown) throw new Error('harness returned empty markdown'); + const citations = (result.citations || []) + .filter((c) => c?.path) + .map((c) => { + const url = citationUrl(repoUrl, commit, c.path, c.start_line, c.end_line); + return { path: c.path, start_line: c.start_line, end_line: c.end_line, note: c.note, ...(url ? { url } : {}) }; + }); + const sourcePaths = [...new Set(citations.map((c) => c.path).concat(outlineItem.source_paths || []))]; + return { + markdown, + frontmatter: { + title: result.title || outlineItem.title, + slug: outlineItem.slug, + category: outlineItem.category, + source_paths: sourcePaths, + citations, + last_updated: new Date().toISOString(), + confidence: result.confidence || 'medium', + status: result.status || 'current', + generator: 'harness', + }, + }; +} + +// Generate one page, then run a deterministic quality gate and, if it fails, +// repair by re-prompting with the exact failing reasons (up to maxAttempts). +export async function generatePageViaHarness(worker, opts) { + const { + wikiId, + outlineItem, + repoName, + repoUrl, + commit, + categories, + allSlugs, + allTitles, + model, + provider, + parentSessionId, + maxTurns = 12, + timeoutMs = 240_000, + maxAttempts = 3, + minWords = 300, + } = opts; + const options = { + system_prompt: PAGE_SYSTEM, + output: { type: 'json', schema: PAGE_HARNESS_OUT }, + functions: { allow: ['openwiki::src::read', 'openwiki::src::list', 'openwiki::src::grep'] }, + max_turns: maxTurns, + }; + + // Name the page's session under the plan session so the console shows a + // titled child per page instead of a fresh opaque id. Repair attempts reuse + // the same session, so retries read as follow-up turns on that page. + const childSessionId = parentSessionId ? `${parentSessionId}/${outlineItem.slug}` : null; + let feedback = ''; + let previousMarkdown = ''; + let best = null; + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + const message = buildUserPrompt({ + wikiId, + outlineItem, + repoName, + repoUrl, + categories, + allSlugs, + allTitles, + feedback, + previousMarkdown, + }); + const out = await runPageTurn(worker, { + message, + model, + provider, + parentSessionId, + childSessionId, + title: outlineItem.title, + options, + timeoutMs, + outlineItem, + repoUrl, + commit, + }); + const issues = getPageQualityIssues(out.markdown, { minWords }); + best = out; + if (!issues.length || attempt === maxAttempts) { + out.frontmatter.quality_issues = issues.length; + if (issues.length) out.frontmatter.status = 'needs-review'; + return out; + } + feedback = pageRepairFeedback(issues); + previousMarkdown = out.markdown; + } + return best; +} + +// Native orchestrator: ONE lead agent session runs the whole job the harness +// way. The lead researches once, decides the pages, then DELEGATES each page to a +// writer sub-agent via harness::spawn (in-turn) — that is how the harness loop +// delegates, not an injected "now spawn" directive. Each writer stores its own +// finished page by calling openwiki::write-page, so the lead never collects +// markdown (parent-side assembly of N pages was the bottleneck). The lead only +// submits the summary + navigation; the pages are already in the store. +const ORCHESTRATOR_SYSTEM = + 'You are OpenWiki. Turn a code repository into a source-grounded wiki by RESEARCHING it once and then DELEGATING each page to a writer sub-agent.\n\n' + + 'You have these functions:\n' + + '- openwiki::src::list { id, dir? } — the file tree.\n' + + '- openwiki::src::read { id, path, from?, to? } — read a file.\n' + + '- openwiki::src::grep { id, pattern } — search contents.\n' + + '- harness::spawn — start a writer sub-agent that does a focused piece of work and returns its result to you.\n\n' + + 'Steps:\n' + + '1. RESEARCH the repository LIGHTLY with openwiki::src::* (always pass the given id). Map it first with openwiki::src::list and openwiki::src::grep, which are cheap, and openwiki::src::read only a few key files (the README, the entry point, one or two core modules) to grasp the architecture. Do NOT read the whole repo. Your own turn holds everything you read, so reading too much bloats your context and can fail the run; leave the deep, per-page file reading to the writer sub-agents. You only need enough to plan the pages and write a short brief.\n' + + '2. PLAN the wiki as a proper documentation INDEX — the way DeepWiki or a good docs site is organized: a hierarchical table of contents that both a human and an LLM can navigate top to bottom. Decide the pages (one per real subsystem or concept) AND how they group into sections. Let the REPOSITORY drive the shape, do not impose a fixed template: a small library is a handful of pages under 2-4 sections; a large repo gets many sections, each with several pages, nested more than one level where a subsystem has sub-topics. Do NOT compress a big, varied repo into a few broad buckets, and do NOT invent sections the repo does not have. ORDER the whole index the way a reader learns the project: Overview / what-is-this FIRST, then Getting Started / Installation, then Core Concepts & Architecture, then one section per major subsystem or feature in dependency order, then API Reference, then Integration / Examples, and finally Advanced / Internals / Deployment / Performance. API Reference is never first. Scale the page count to the repo (tiny <25 files: 3-6 pages; small: 8-14; medium: 16-28; large or doc-heavy: 30-48+). For each page decide: a slug, a title, its section, the 2-6 source files that page must cover, and a one-line angle. Write a short shared "repo brief" (what the repo is, its architecture, the main modules) that you will hand to every writer so they do not re-discover it.\n' + + '3. WRITE the wiki by spawning ONE writer sub-agent per page with harness::spawn. Put ALL the spawns in a SINGLE message so they run in parallel. For each spawn: give it session_id "/"; allow it ONLY the functions openwiki::src::read, openwiki::src::list, openwiki::src::grep, and openwiki::write-page; set its output contract to json {slug, ok}. Do NOT set a model or provider on the spawns — the sub-agents automatically use yours; never name a model. Do NOT write any page yourself, and NEVER call openwiki::write-page yourself: it is only for the writer sub-agents you spawn. Writing pages inline would bloat your single turn and can fail the whole run on a large repo. Your job is to research, plan, spawn, and submit navigation, nothing more.\n' + + ' Each writer task MUST contain: the shared repo brief; the page slug, title, and category; the specific source files to read first (it reads them with openwiki::src::* using the same id — only its focused files, not the whole repo); and this instruction: produce a substantial, source-grounded page — at least 3 "##" sections; a "## Relevant Source Files" section naming the key files; explain WHY the code is shaped this way, not only what it does; ground claims with inline path:line references; 400-900 words of real prose. CRITICAL requirements the writer must follow:\n' + + ' - API REFERENCE: if the page documents functions, methods, options, config keys, CLI flags, or return shapes, include an "## API Reference" section that presents them as a GitHub-flavored Markdown TABLE (e.g. columns Parameter | Type | Description, and a Returns row/table). Use real signatures and types read from the source.\n' + + ' - DIAGRAMS: wherever a picture aids understanding (architecture, data flow, execution flow, a state machine, a class or module relationship), embed a small valid Mermaid diagram INLINE as a ```mermaid fenced code block (flowchart / sequenceDiagram / classDiagram).\n' + + ' - REFERENCES: when the writer calls openwiki::write-page, it MUST pass a citations array — one entry per source it actually used, each { path, start_line, end_line, note } with the REAL line range it read for that claim (the note is a short description). These become the clickable References list on the page, so they must be accurate. Also pass source_paths (the files the page covers).\n' + + ' When the page is ready the writer MUST call openwiki::write-page { id, slug, title, category, markdown, source_paths, citations } to store it (write-page REJECTS pages under ~250 characters, so EVERY page — including overview, getting-started, and installation — must be genuinely substantial; ground even those in the README and entry files), then return { slug, ok:true }. Tell each writer this explicitly.\n' + + '4. When every writer has returned ok, submit your final result. The pages are already stored, so do NOT include page bodies.\n\n' + + "Final result JSON: { summary, navigation }. navigation IS the wiki's table of contents: a nested tree where a SECTION is { title, children:[...] } with no slug, and a PAGE is { title, slug } whose slug matches a page you delegated. Sections and pages MUST appear in the reading order from step 2 (Overview first; API Reference, Advanced, Deployment later) and may nest more than one level deep. Make it complete and well-ordered — every page you delegated appears exactly once."; + +const WIKI_ORCHESTRATOR_OUT = { + type: 'object', + additionalProperties: true, + required: ['navigation'], + properties: { + summary: { type: 'string' }, + navigation: { type: 'array', items: { type: 'object', additionalProperties: true } }, + }, +}; + +// Drive the whole generation as one native agentic session. The writer children +// store their pages directly (openwiki::write-page); this returns only what the +// lead submits — summary + navigation — plus the root session id (children nest +// under it in the console). +export async function runOrchestrator( + worker, + { wikiId, repoName, repoUrl, model, docsHint = '', maxTurns = 60, timeoutMs = 600_000 }, +) { + const resolved = await resolveModel(worker, model); + if (!resolved.resolved) throw new Error('no model available for the orchestrator'); + const root = wikiParentSession(repoName, wikiId); + const message = + `Repository: ${repoName} (${repoUrl})\n` + + `Wiki id (pass as "id" to every openwiki::src::* and openwiki::write-page call, and use "${root}/" as each writer's session_id): ${wikiId}` + + (docsHint || '') + + '\nResearch the repo, plan the pages, spawn one writer sub-agent per page (each writer stores its page with openwiki::write-page), then submit the summary and navigation.'; + const { session_id } = await worker.trigger({ + function_id: 'harness::send', + payload: { + session_id: root, + session: { title: `openwiki: ${repoName}` }, + message, + model: resolved.model, + ...(resolved.provider ? { provider: resolved.provider } : {}), + options: { + system_prompt: ORCHESTRATOR_SYSTEM, + output: { type: 'json', schema: WIKI_ORCHESTRATOR_OUT }, + functions: { + allow: [ + 'harness::spawn', + 'openwiki::src::read', + 'openwiki::src::list', + 'openwiki::src::grep', + 'openwiki::write-page', + ], + }, + max_turns: maxTurns, + }, + }, + timeoutMs: 30_000, + }); + if (!session_id) throw new Error('orchestrator send returned no session_id'); + const result = await awaitTurn(worker, session_id, { timeoutMs }); + if (!result) throw new Error('orchestrator returned no result'); + return { + summary: result.summary || '', + navigation: Array.isArray(result.navigation) ? result.navigation : [], + sessionId: session_id, + }; +} + +const PLAN_SYSTEM = + 'You are OpenWiki planning a documentation wiki for a code repository.\n' + + 'Explore the repository first (always pass the given id):\n' + + '- openwiki::src::list { id, dir? } — the file tree (path, language, priority).\n' + + '- openwiki::src::read { id, path, from?, to? } — read a file.\n' + + '- openwiki::src::grep { id, pattern } — search contents.\n' + + 'Read entry points, key modules, config, and docs before deciding structure.\n\n' + + 'Design a reader-facing wiki from the ACTUAL modules, subsystems, and concepts in the source.\n' + + 'Page budget scales to the repo: tiny (<25 files) 3-6 pages; small 8-14; medium 16-28; large or doc-heavy 30-48.\n' + + 'Build a real information architecture, not a flat file list:\n' + + '- navigation is a NESTED tree, up to 3 levels: top-level folders -> optional sub-folders -> leaf pages.\n' + + '- A folder node has a title and children and NO slug. A leaf node has a title and a slug (a real page).\n' + + '- Start with a "Start Here" area (overview, install/getting-started), then group the rest by real subsystem or concept.\n' + + '- Every leaf slug must appear exactly once in pages[]. Every page must reference at least one real source path.\n' + + '- Do not create a folder that holds only one leaf; make it a page instead.\n\n' + + 'Return JSON: { summary, pages:[{slug,title,brief,source_paths}], navigation:[navNode] }.'; + +// Plan a wiki by having the harness explore the clone (openwiki::src::*), so the +// structure reflects the real repo rather than a heuristic template. Throws when +// the harness is unavailable; the caller falls back to the router/heuristic plan. +export async function planViaHarness( + worker, + { + wikiId, + repoName, + repoUrl, + model, + docsHint = '', + parentSessionId, + maxTurns = 24, + timeoutMs = 300_000, + maxAttempts = 3, + onRetry, + }, +) { + const resolved = await resolveModel(worker, model); + if (!resolved.resolved) throw new Error('no model available for harness plan'); + const base = parentSessionId || wikiParentSession(repoName, wikiId); + const message = + `Repository: ${repoName} (${repoUrl})\n` + + `Wiki id (pass as "id" to every openwiki::src::* call): ${wikiId}` + + (docsHint || '') + + '\nExplore the repository, then return the wiki plan JSON.'; + // The plan is a long agentic turn (explore + structured output). Provider + // streaming is the flakiest part of it ("stream ended without a terminal + // frame"), and one failure would otherwise sink the whole generation. Retry + // on a FRESH session each attempt so a failed turn's partial transcript never + // bloats the retry (which would only make the next stream more likely to drop). + let lastErr; + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + const parent = attempt === 1 ? base : `${base}:r${attempt}`; + try { + const { session_id } = await worker.trigger({ + function_id: 'harness::send', + payload: { + session_id: parent, + session: { title: `openwiki: ${repoName}` }, + message, + model: resolved.model, + ...(resolved.provider ? { provider: resolved.provider } : {}), + options: { + system_prompt: PLAN_SYSTEM, + output: { type: 'json', schema: PLAN_HARNESS_OUT }, + functions: { allow: ['openwiki::src::read', 'openwiki::src::list', 'openwiki::src::grep'] }, + max_turns: maxTurns, + }, + }, + timeoutMs: 30_000, + }); + if (!session_id) throw new Error('harness::send returned no session_id for plan'); + const result = await awaitTurn(worker, session_id, { timeoutMs }); + if (!result || !Array.isArray(result.pages) || result.pages.length < 1) + throw new Error('harness returned an empty plan'); + return { + summary: result.summary || '', + pages: result.pages, + navigation: Array.isArray(result.navigation) ? result.navigation : [], + sessionId: session_id, + }; + } catch (e) { + lastErr = e; + if (attempt < maxAttempts) { + try { + onRetry?.(attempt, e); + } catch { + /* ignore */ + } + await sleep(1500 * attempt); + } + } + } + throw lastErr || new Error('harness plan failed'); +} + +// Tiered page writer: harness (agentic, cited) -> router/heuristic (generate.mjs). +export async function generatePageAny(worker, opts) { + if (opts.wikiId && opts.model && opts.useHarness !== false) { + try { + const out = await generatePageViaHarness(worker, opts); + if (String(out?.markdown || '').trim()) return out; + } catch (e) { + if (typeof opts.onFallback === 'function') opts.onFallback(e); + } + } + return generatePage(worker, opts); +} diff --git a/openwiki/src/lib/heuristic.mjs b/openwiki/src/lib/heuristic.mjs new file mode 100644 index 000000000..1d9ad7910 --- /dev/null +++ b/openwiki/src/lib/heuristic.mjs @@ -0,0 +1,315 @@ +// Heuristic fallback for OpenWiki: produces a serviceable, source-grounded wiki +// without an LLM. Used when router::complete is unavailable or errors. +import { readFile } from 'node:fs/promises'; +import path from 'node:path'; + +function titleCase(s) { + return String(s || '') + .replace(/[-_]/g, ' ') + .replace(/\b\w/g, (c) => c.toUpperCase()) + .trim(); +} + +function slugify(s) { + return ( + String(s || '') + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-|-$/g, '') + .slice(0, 60) || 'page' + ); +} + +function firstParagraph(text, cap = 400) { + if (!text) return ''; + const lines = text.split(/\r?\n/); + const out = []; + let started = false; + for (const raw of lines) { + const l = raw.trim(); + if (!started) { + if (!l) continue; + if (l.startsWith('#')) continue; + if (l.startsWith('!') || l.startsWith('[![')) continue; + started = true; + out.push(l); + } else { + if (!l) break; + if (l.startsWith('#')) break; + out.push(l); + } + if (out.join(' ').length > cap) break; + } + return out.join(' ').slice(0, cap); +} + +async function readIfExists(dir, rel, cap = 12_000) { + try { + const buf = await readFile(path.join(dir, rel), 'utf8'); + return buf.length > cap ? buf.slice(0, cap) : buf; + } catch { + return null; + } +} + +function topLevelDirs(inventory) { + const dirs = new Map(); + for (const e of inventory) { + if ((e.priority ?? 0) <= 0) continue; + const first = e.relPath.split('/')[0]; + if (!first || first.includes('.')) continue; + if (!dirs.has(first)) dirs.set(first, []); + dirs.get(first).push(e); + } + return [...dirs.entries()].filter(([, files]) => files.length >= 2).sort((a, b) => b[1].length - a[1].length); +} + +function packageInfo(pkgText) { + if (!pkgText) return null; + try { + const j = JSON.parse(pkgText); + return { + name: j.name || null, + version: j.version || null, + description: j.description || null, + scripts: j.scripts || {}, + dependencies: Object.keys(j.dependencies || {}), + devDependencies: Object.keys(j.devDependencies || {}), + }; + } catch { + return null; + } +} + +export async function planWikiHeuristic({ inventory, repoName, repoDir }) { + const readme = (await readIfExists(repoDir, 'README.md')) || (await readIfExists(repoDir, 'README.mdx')) || ''; + const pkg = packageInfo((await readIfExists(repoDir, 'package.json')) || ''); + const summary = + firstParagraph(readme, 400) || + (pkg?.description ? String(pkg.description) : '') || + `Source-grounded wiki for ${repoName}.`; + + const categories = [ + { id: 'overview', title: 'Overview', description: 'What this repository is and how it fits together.' }, + { id: 'architecture', title: 'Architecture', description: 'How the source tree is organized.' }, + { id: 'reference', title: 'Reference', description: 'File-by-file inventory and configuration.' }, + { id: 'docs', title: 'Docs', description: 'Repository documentation, verbatim.' }, + ]; + + const outline = []; + const invPaths = new Set(inventory.map((e) => e.relPath)); + + // 1) Overview page (always) + const overviewSources = []; + if (invPaths.has('README.md')) overviewSources.push('README.md'); + else if (invPaths.has('README.mdx')) overviewSources.push('README.mdx'); + if (invPaths.has('package.json')) overviewSources.push('package.json'); + outline.push({ + slug: 'overview', + title: 'Overview', + category: 'overview', + source_paths: overviewSources.length ? overviewSources : [inventory[0]?.relPath].filter(Boolean), + brief: `High-level summary of ${repoName}.`, + }); + + // 2) Getting Started (if package.json or install docs exist) + if (invPaths.has('package.json')) { + outline.push({ + slug: 'getting-started', + title: 'Getting Started', + category: 'overview', + source_paths: ['package.json', ...['README.md', 'README.mdx'].filter((p) => invPaths.has(p))], + brief: 'Install, scripts, and quick start.', + }); + } + + // 3) Architecture pages — one per top-level dir with code + const dirs = topLevelDirs(inventory); + for (const [dir, files] of dirs.slice(0, 5)) { + const top = files + .sort((a, b) => (b.priority ?? 0) - (a.priority ?? 0)) + .slice(0, 12) + .map((f) => f.relPath); + outline.push({ + slug: `dir-${slugify(dir)}`, + title: `${titleCase(dir)} Directory`, + category: 'architecture', + source_paths: top, + brief: `Contents and role of the \`${dir}/\` directory.`, + }); + } + + // 4) One page per doc file (up to 6) + const docs = inventory.filter((e) => e.isDoc && e.relPath !== 'README.md' && e.relPath !== 'README.mdx').slice(0, 6); + for (const d of docs) { + outline.push({ + slug: `doc-${slugify(d.relPath)}`, + title: titleCase(path.basename(d.relPath).replace(/\.[^.]+$/, '')), + category: 'docs', + source_paths: [d.relPath], + brief: `Repository documentation: ${d.relPath}.`, + }); + } + + // 5) File reference page — full high-priority inventory + const refTop = inventory.slice(0, 30).map((e) => e.relPath); + outline.push({ + slug: 'file-reference', + title: 'File Reference', + category: 'reference', + source_paths: refTop, + brief: 'Curated inventory of the most relevant files in the repository.', + }); + + return { summary, categories, outline }; +} + +export async function generatePageHeuristic({ + outlineItem, + sourceReads, + allSlugs, + allTitles, + categories, + repoName, + repoUrl, +}) { + // This runs as the last-resort fallback after an LLM failure, so the caller + // may hand it partial opts; default the collections rather than adding a + // TypeError on top of the original error. + const cats = categories || []; + const reads = sourceReads || []; + const title = outlineItem.title; + const category = cats.find((c) => c.id === outlineItem.category); + const catTitle = category ? category.title : outlineItem.category; + + const lines = []; + lines.push(`# ${title}`); + lines.push(''); + lines.push(`_Category: **${catTitle}** · Repo: [${repoName}](${repoUrl})_`); + lines.push(''); + lines.push('## Overview'); + lines.push(''); + lines.push(outlineItem.brief || `Notes on ${title.toLowerCase()} for ${repoName}.`); + lines.push(''); + + // Try to derive a short description from the first source's leading comment/paragraph + const first = reads[0]; + if (first) { + const preview = extractSummary(first.content, first.path); + if (preview) { + lines.push(preview); + lines.push(''); + } + } + + // Key files section + lines.push('## Key files'); + lines.push(''); + for (const sr of reads) { + const short = oneLine(sr.content); + lines.push(`- \`${sr.path}\`${short ? ` — ${short}` : ''}`); + } + lines.push(''); + + // Excerpts + const excerpts = reads.filter((sr) => !isBinaryLikely(sr.path) && sr.content && sr.content.length > 0).slice(0, 4); + if (excerpts.length) { + lines.push('## Excerpts'); + lines.push(''); + for (const sr of excerpts) { + const ext = extFromPath(sr.path); + const body = truncateLines(sr.content, 40); + lines.push(`### \`${sr.path}\``); + lines.push(''); + lines.push(`\`\`\`${ext}`); + lines.push(body); + lines.push('```'); + lines.push(''); + } + } + + // Related pages + const related = []; + const slugs = allSlugs || []; + const titles = allTitles || []; + for (let i = 0; i < slugs.length; i++) { + if (slugs[i] === outlineItem.slug) continue; + related.push(`- [${titles[i] || slugs[i]}](./${slugs[i]}.md)`); + if (related.length >= 6) break; + } + if (related.length) { + lines.push('## Related pages'); + lines.push(''); + lines.push(...related); + lines.push(''); + } + + // Sources + lines.push('## Sources'); + lines.push(''); + for (const p of outlineItem.source_paths || []) { + lines.push(`- \`${p}\``); + } + lines.push(''); + lines.push(`_Generated heuristically (no LLM) at ${new Date().toISOString()}._`); + + const markdown = lines.join('\n'); + const frontmatter = { + title, + slug: outlineItem.slug, + category: outlineItem.category, + source_paths: outlineItem.source_paths, + last_updated: new Date().toISOString(), + confidence: 'medium', + status: 'current', + generator: 'heuristic', + }; + return { markdown, frontmatter }; +} + +function extFromPath(p) { + const m = String(p ?? '').match(/\.([A-Za-z0-9]+)$/); + return m ? m[1].toLowerCase() : 'text'; +} + +function isBinaryLikely(p) { + return /\.(png|jpg|jpeg|gif|webp|ico|pdf|zip|tar|gz|bin|wasm|woff2?|ttf|otf|mp[34]|mov|svg)$/i.test(p); +} + +function oneLine(content) { + if (!content) return ''; + const lines = String(content).split(/\r?\n/); + for (const raw of lines) { + const l = raw.replace(/^[\s#*/-]+/, '').trim(); + if (l && !l.startsWith('```')) return l.slice(0, 120); + } + return ''; +} + +function extractSummary(content, filePath) { + if (!content) return ''; + // Prefer a top-of-file block comment / docstring + const m1 = content.match(/^(?:\s*\/\*\*?([\s\S]*?)\*\/)/); + if (m1) { + const body = m1[1] + .split(/\r?\n/) + .map((l) => l.replace(/^\s*\*?\s?/, '').trim()) + .filter(Boolean) + .join(' ') + .trim(); + if (body.length > 20) return body.slice(0, 400); + } + // Python docstring + const m2 = content.match(/^\s*"""([\s\S]*?)"""/); + if (m2 && m2[1].trim().length > 20) return m2[1].trim().slice(0, 400); + // Markdown: first paragraph + if (/\.mdx?$/i.test(filePath)) return firstParagraph(content, 400); + // Fallback: first meaningful line as a summary + return ''; +} + +function truncateLines(text, n) { + const arr = String(text).split(/\r?\n/); + if (arr.length <= n) return text; + return `${arr.slice(0, n).join('\n')}\n// ...[truncated]`; +} diff --git a/openwiki/src/lib/inventory.mjs b/openwiki/src/lib/inventory.mjs new file mode 100644 index 000000000..ec0126987 --- /dev/null +++ b/openwiki/src/lib/inventory.mjs @@ -0,0 +1,192 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import crypto from 'node:crypto'; + +export const DEFAULT_EXCLUDES = [ + '.git', + 'node_modules', + 'dist', + 'build', + '.venv', + '__pycache__', + 'target', + '.next', + '.turbo', + 'coverage', + '.cache', +]; +export const DEFAULT_MAX_FILE_BYTES = 200_000; +export const LANG_BY_EXT = { + '.ts': 'typescript', + '.tsx': 'typescript', + '.js': 'javascript', + '.jsx': 'javascript', + '.mjs': 'javascript', + '.cjs': 'javascript', + '.py': 'python', + '.rs': 'rust', + '.go': 'go', + '.java': 'java', + '.kt': 'kotlin', + '.swift': 'swift', + '.c': 'c', + '.h': 'c', + '.cpp': 'cpp', + '.cc': 'cpp', + '.hpp': 'cpp', + '.rb': 'ruby', + '.php': 'php', + '.md': 'markdown', + '.mdx': 'markdown', + '.yml': 'yaml', + '.yaml': 'yaml', + '.json': 'json', + '.toml': 'toml', + '.sh': 'shell', + '.html': 'html', + '.css': 'css', + '.scss': 'scss', + '.sql': 'sql', + '.proto': 'protobuf', +}; + +const ROOT_META_PRIO2 = new Set([ + 'package.json', + 'pyproject.toml', + 'Cargo.toml', + 'go.mod', + 'tsconfig.json', + 'pnpm-workspace.yaml', +]); +const DOC_BASENAMES = ['README', 'CHANGELOG', 'CONTRIBUTING', 'LICENSE']; + +function stripExt(base) { + const i = base.lastIndexOf('.'); + return i > 0 ? base.slice(0, i) : base; +} + +function isDocFile(relPath, ext) { + const base = path.posix.basename(relPath); + const stem = stripExt(base).toUpperCase(); + if (DOC_BASENAMES.includes(stem)) return true; + if (relPath.startsWith('docs/')) return true; + if (ext === '.md' || ext === '.mdx') return true; + return false; +} + +function computePriority(relPath, _ext, language) { + const base = path.posix.basename(relPath); + const stem = stripExt(base).toUpperCase(); + const atRoot = !relPath.includes('/'); + + if (atRoot && (base === 'README.md' || base === 'README.mdx')) return 3; + if (atRoot && stem === 'CHANGELOG') return 3; + if (relPath.startsWith('docs/')) return 3; + + if (atRoot && ROOT_META_PRIO2.has(base)) return 2; + if (atRoot && (stem === 'LICENSE' || stem === 'CONTRIBUTING')) return 2; + const entryStems = ['index', 'main']; + if (entryStems.includes(stripExt(base))) { + if (atRoot) return 2; + if (relPath.startsWith('src/') && relPath.split('/').length === 2) return 2; + } + + if (language && language !== 'text') { + if (['json', 'yaml', 'toml'].includes(language)) return 0; + return 1; + } + return 0; +} + +function matchesSuffixGlob(name, globs) { + if (!globs?.length) return false; + for (const g of globs) { + if (!g) continue; + if (g.startsWith('*')) { + if (name.endsWith(g.slice(1))) return true; + } else if (name === g) return true; + else if (name.endsWith(g)) return true; + } + return false; +} + +export async function inventoryRepo(repoDir, opts = {}) { + const maxBytes = opts.maxFileBytes ?? DEFAULT_MAX_FILE_BYTES; + const excludeGlobs = opts.excludeGlobs || []; + const out = []; + + async function walk(dir) { + let entries; + try { + entries = await fs.readdir(dir, { withFileTypes: true }); + } catch { + return; + } + for (const e of entries) { + const full = path.join(dir, e.name); + if (e.isDirectory()) { + if (DEFAULT_EXCLUDES.includes(e.name)) continue; + if (matchesSuffixGlob(e.name, excludeGlobs)) continue; + await walk(full); + } else if (e.isFile()) { + if (matchesSuffixGlob(e.name, excludeGlobs)) continue; + const rel = path.relative(repoDir, full).split(path.sep).join('/'); + let st; + try { + st = await fs.stat(full); + } catch { + continue; + } + const size = st.size; + const ext = path.extname(e.name).toLowerCase(); + const language = LANG_BY_EXT[ext] || 'text'; + const isDoc = isDocFile(rel, ext); + const priority = computePriority(rel, ext, language); + + let truncated = false; + let buf; + try { + const fh = await fs.open(full, 'r'); + try { + const readLen = Math.min(size, maxBytes); + buf = Buffer.alloc(readLen); + if (readLen > 0) await fh.read(buf, 0, readLen, 0); + if (size > maxBytes) truncated = true; + } finally { + await fh.close(); + } + } catch { + buf = Buffer.alloc(0); + } + const sha = crypto.createHash('sha1').update(buf).digest('hex').slice(0, 12); + + out.push({ relPath: rel, size, ext, language, isDoc, priority, sha, truncated }); + } + } + } + + await walk(repoDir); + out.sort((a, b) => b.priority - a.priority || (a.relPath < b.relPath ? -1 : a.relPath > b.relPath ? 1 : 0)); + return out; +} + +export async function readSourceFile(repoDir, relPath, maxBytes = 200_000) { + const full = path.join(repoDir, relPath); + // Bounded read: request maxBytes + 1 through a handle instead of loading the + // whole file, so a giant file never lands in memory. The extra byte is only + // the truncation probe. + const fh = await fs.open(full, 'r'); + let bytesRead = 0; + let buf; + try { + buf = Buffer.alloc(maxBytes + 1); + ({ bytesRead } = await fh.read(buf, 0, maxBytes + 1, 0)); + } finally { + await fh.close(); + } + const truncated = bytesRead > maxBytes; + const content = truncated + ? `${buf.subarray(0, maxBytes).toString('utf8')}\n...[truncated]` + : buf.subarray(0, bytesRead).toString('utf8'); + return { path: relPath, content, truncated }; +} diff --git a/openwiki/src/lib/lint.mjs b/openwiki/src/lib/lint.mjs new file mode 100644 index 000000000..fdd2c39f7 --- /dev/null +++ b/openwiki/src/lib/lint.mjs @@ -0,0 +1,58 @@ +// Lint pass: validate that a wiki's pages are still grounded. Checks every +// citation resolves to a real file and a line range inside it, and flags thin +// pages. Runs after generation/refresh and on the nightly cron. Orphan and +// missing-cross-reference checks are a later addition. +import fs from 'node:fs/promises'; +import * as store from './store.mjs'; +import { readSourceFile } from './inventory.mjs'; + +const THIN_CHARS = 200; + +export async function lintWiki(wikiId) { + const meta = await store.getWiki(wikiId); + if (!meta) { + const e = new Error('wiki not found'); + e.code = 'openwiki/wiki_not_found'; + throw e; + } + const dir = store.repoDir(wikiId); + const haveClone = !!(await fs.stat(dir).catch(() => null)); + const pages = await store.listPages(wikiId); + const issues = []; + let checked = 0; + + for (const { slug, meta: pm } of pages) { + checked += 1; + + if (haveClone) { + for (const c of pm?.citations || []) { + if (!c?.path) continue; + let content; + let truncated = false; + try { + ({ content, truncated } = await readSourceFile(dir, c.path, 200_000)); + } catch { + issues.push({ slug, kind: 'broken-citation', detail: `missing file ${c.path}` }); + continue; + } + // A truncated read undercounts the file's lines; a line-range check + // against it would flag valid citations as broken. + if (!truncated && (c.start_line || c.end_line)) { + const total = content.split(/\r?\n/).length; + if (c.start_line && c.start_line > total) { + issues.push({ slug, kind: 'broken-citation', detail: `${c.path}:${c.start_line} beyond ${total} lines` }); + } else if (c.end_line && c.end_line > total) { + issues.push({ slug, kind: 'broken-citation', detail: `${c.path}:${c.end_line} beyond ${total} lines` }); + } + } + } + } + + const page = await store.getPage(wikiId, slug); + if (page && String(page.markdown || '').trim().length < THIN_CHARS) { + issues.push({ slug, kind: 'thin', detail: `page body under ${THIN_CHARS} chars` }); + } + } + + return { checked, issues }; +} diff --git a/openwiki/src/lib/model.mjs b/openwiki/src/lib/model.mjs new file mode 100644 index 000000000..607c0ab1a --- /dev/null +++ b/openwiki/src/lib/model.mjs @@ -0,0 +1,70 @@ +// Resolve a model id against the live router catalog. The harness and its +// output contract need a real (model, provider) pair, and structured-output +// support decides whether the contract rides provider-native JSON or the +// harness's submit_result fallback. Never hardcode a model id — validate it. + +const cache = new Map(); // preferred -> resolved (per worker run) + +export function pickModel(models, preferred) { + const list = Array.isArray(models) ? models : []; + const byId = (id) => list.find((m) => m && m.id === id); + let m = preferred ? byId(preferred) : null; + if (!m && list.length) { + m = + list.find((x) => x?.supports_structured_output && x.supports_tools) || + list.find((x) => x?.supports_tools) || + list.find(Boolean); + } + if (!m) { + return { model: preferred || null, provider: undefined, supports_structured_output: false, resolved: false }; + } + return { + model: m.id, + provider: m.provider, + supports_structured_output: !!m.supports_structured_output, + resolved: true, + }; +} + +export async function resolveModel(worker, preferred) { + const key = preferred || ''; + if (cache.has(key)) return cache.get(key); + + let models = []; + try { + const res = await worker.trigger({ function_id: 'router::models::list', payload: {} }); + models = res?.models || (Array.isArray(res) ? res : []); + } catch { + // router unavailable — fall back to the preferred id unresolved; harness + // will error and the caller drops to the router/heuristic page path. + } + const out = pickModel(models, preferred); + // Only cache successful resolutions: a router that was down at first call + // must not pin the worker to the unresolved (heuristic) tier forever. + if (out.resolved) cache.set(key, out); + return out; +} + +export function clearModelCache() { + cache.clear(); +} + +// The router's live model catalog, trimmed to what the UI's picker needs. +// Returns [] when llm-router is absent (no provider configured), which the UI +// treats as "set up a provider in the console". +export async function listModels(worker) { + let models = []; + try { + const res = await worker.trigger({ function_id: 'router::models::list', payload: {} }); + models = res?.models || (Array.isArray(res) ? res : []); + } catch { + return []; + } + return models + .filter((m) => m?.id) + .map((m) => ({ + id: m.id, + provider: m.provider || 'unknown', + supports_structured_output: !!m.supports_structured_output, + })); +} diff --git a/openwiki/src/lib/nav.mjs b/openwiki/src/lib/nav.mjs new file mode 100644 index 000000000..347bfe774 --- /dev/null +++ b/openwiki/src/lib/nav.mjs @@ -0,0 +1,75 @@ +// Navigation-tree helpers. The planner returns a nested nav tree (folders + +// leaf pages) plus a flat page list; these normalize the two planner shapes +// (harness: {pages, navigation}; fallback: {categories, outline}) into a single +// { summary, outline, navigation } the generator and UI both use. + +// Flatten every leaf slug in a nav tree. +export function navSlugs(navigation) { + const out = []; + const walk = (nodes) => { + for (const n of nodes || []) { + if (n.slug) out.push(n.slug); + if (n.children) walk(n.children); + } + }; + walk(navigation); + return out; +} + +// Map each leaf slug to its top-level folder title (used as page.category). +export function slugToSection(navigation) { + const map = {}; + for (const l1 of navigation || []) { + const section = l1.title; + const mark = (nodes) => { + for (const n of nodes || []) { + if (n.slug) map[n.slug] = section; + if (n.children) mark(n.children); + } + }; + if (l1.slug) map[l1.slug] = section; + mark(l1.children); + } + return map; +} + +// Build a flat nav tree from {categories, outline} (fallback / heuristic plan). +export function navFromCategories(categories, outline) { + const cats = categories?.length ? categories : []; + const nav = []; + const covered = new Set(); + for (const c of cats) { + const leaves = (outline || []) + .filter((o) => (o.category || '') === c.id) + .map((o) => { + covered.add(o.slug); + return { title: o.title, slug: o.slug }; + }); + if (leaves.length) nav.push({ title: c.title || c.id, children: leaves }); + } + const rest = (outline || []).filter((o) => !covered.has(o.slug)).map((o) => ({ title: o.title, slug: o.slug })); + if (rest.length) nav.push({ title: cats.length ? 'Other' : 'Pages', children: rest }); + return nav; +} + +// Normalize any planner output into { summary, outline, navigation }. +export function normalizePlan(planned) { + if (Array.isArray(planned.pages)) { + const navigation = planned.navigation?.length + ? planned.navigation + : [{ title: 'Pages', children: planned.pages.map((p) => ({ title: p.title, slug: p.slug })) }]; + const sect = slugToSection(navigation); + const outline = planned.pages.map((p) => ({ + slug: p.slug, + title: p.title, + brief: p.brief, + source_paths: p.source_paths || [], + category: sect[p.slug] || 'Pages', + })); + return { summary: planned.summary || '', outline, navigation }; + } + const navigation = navFromCategories(planned.categories || [], planned.outline || []); + const sect = slugToSection(navigation); + const outline = (planned.outline || []).map((o) => ({ ...o, category: sect[o.slug] || o.category || 'Pages' })); + return { summary: planned.summary || '', outline, navigation }; +} diff --git a/openwiki/src/lib/progress.mjs b/openwiki/src/lib/progress.mjs new file mode 100644 index 000000000..778d2b6eb --- /dev/null +++ b/openwiki/src/lib/progress.mjs @@ -0,0 +1,17 @@ +// In-process progress bus. Generation runs in this worker; the SSE handler +// (openwiki::http::events) subscribes per wiki id and pushes frames to the +// browser over the HTTP response channel. Single-process worker; if openwiki +// ever runs multiple instances, swap this for stream::send + a stream trigger. +import { EventEmitter } from 'node:events'; + +const bus = new EventEmitter(); +bus.setMaxListeners(0); + +export function pushProgress(wikiId, evt) { + bus.emit(wikiId, evt); +} + +export function onProgress(wikiId, cb) { + bus.on(wikiId, cb); + return () => bus.off(wikiId, cb); +} diff --git a/openwiki/src/lib/quality.mjs b/openwiki/src/lib/quality.mjs new file mode 100644 index 000000000..dd57411f7 --- /dev/null +++ b/openwiki/src/lib/quality.mjs @@ -0,0 +1,60 @@ +// Deterministic page quality gate. Adapted from vercel-labs/openwiki: validate a +// generated page with pure functions, then feed the exact failing reasons back +// to the model for a repair pass. This is what turns thin agent output into +// dense, source-grounded pages without an agent loop deciding when it is "done". + +// Count words of ACTUAL prose: strip fenced code, inline code, whole "Sources:" +// lines, URLs, and path-like tokens before counting. Defeats padding with +// tables/paths/code so a word target means real explanation. +export function countWords(markdown) { + const stripped = String(markdown || '') + .replace(/```[\s\S]*?```/g, ' ') + .replace(/`[^`\n]+`/g, ' ') + .replace(/^.*\bSources:\s*.*$/gim, ' ') + .replace(/https?:\/\/\S+/g, ' ') + .replace(/[A-Za-z0-9_.-]+\/[A-Za-z0-9_./-]+/g, ' '); + return (stripped.match(/[A-Za-z0-9][A-Za-z0-9'-]*/g) || []).length; +} + +export function countH2(markdown) { + return (String(markdown || '').match(/^##\s+/gm) || []).length; +} + +// Returns a list of human-readable issue strings (empty = passes). +// Checks run on the markdown with fenced code blocks removed: a fence that +// happens to contain "## " or "Sources:" must not satisfy (or pad) the checks. +export function getPageQualityIssues(markdown, opts = {}) { + const minWords = opts.minWords ?? 180; + const md = String(markdown || '').replace(/```[\s\S]*?```/g, ''); + const issues = []; + + if (!/^#\s+.+/m.test(md)) { + issues.push('Start with a single level-1 heading: "# Page Title".'); + } + const h2 = countH2(md); + if (h2 < 3) { + issues.push( + `Include at least 3 level-2 sections ("## ..."); found ${h2}. Use sections like Purpose and Scope, Relevant Source Files, System-to-Code Mapping, Execution Flow, Extension Points, Things to Watch.`, + ); + } + if (!/^##\s+Relevant Source Files\b/im.test(md)) { + issues.push('Add a "## Relevant Source Files" section with bullets naming the key files and why each one matters.'); + } + if (!/Sources:/i.test(md)) { + issues.push('Ground concrete claims with visible "Sources: path/a.ts, path/b.ts" lines in the prose.'); + } + const words = countWords(md); + if (words < minWords) { + issues.push( + `Add more explanation: only ${words} words of prose, need at least ${minWords}. Explain what the area does, why it exists, and what to watch when changing it.`, + ); + } + return issues; +} + +export function pageRepairFeedback(issues) { + return ( + 'Your previous draft did not meet the quality bar. Keep what was accurate and fix ALL of these:\n' + + issues.map((s) => `- ${s}`).join('\n') + ); +} diff --git a/openwiki/src/lib/schemas.mjs b/openwiki/src/lib/schemas.mjs new file mode 100644 index 000000000..f057feaac --- /dev/null +++ b/openwiki/src/lib/schemas.mjs @@ -0,0 +1,542 @@ +// Typed wire schemas for every openwiki function. +// +// SOP docs/sops/new-worker.md §5: every registered function must publish a +// typed request AND response schema — never the permissive AnyValue schema an +// untyped handler emits. The publish pipeline runs +// collect_worker_interface.py --assert-typed-schemas, so an untyped handler +// fails the release. These constants are the single source of truth; index.mjs +// attaches them at registration. + +const STRING = { type: 'string' }; +const BOOL = { type: 'boolean' }; +const NUM = { type: 'number' }; +const INT = { type: 'integer' }; + +// A wiki's stored metadata record. +export const WIKI_META = { + type: 'object', + additionalProperties: false, + required: ['id', 'repo_url', 'repo_name', 'page_count', 'category_count', 'generating'], + properties: { + id: STRING, + repo_url: STRING, + repo_name: STRING, + ref: STRING, + commit: STRING, + created_at: STRING, + updated_at: STRING, + page_count: INT, + category_count: INT, + categories: { + type: 'array', + items: { + type: 'object', + additionalProperties: false, + required: ['id', 'title'], + properties: { id: STRING, title: STRING, description: STRING }, + }, + }, + summary: STRING, + model: STRING, + generating: BOOL, + content_hash: STRING, + navigation: { type: 'array', items: { type: 'object', additionalProperties: true } }, + refresh_schedule: STRING, + last_refresh_at: STRING, + steer: { type: 'object', additionalProperties: true }, + }, +}; + +// A source citation pinned to a line range at a known commit. +export const CITATION = { + type: 'object', + additionalProperties: false, + required: ['path'], + properties: { + path: STRING, + start_line: INT, + end_line: INT, + note: STRING, + url: STRING, // host deep-link at the pinned commit + }, +}; + +// A page's frontmatter/metadata (no markdown body). +export const PAGE_META = { + type: 'object', + additionalProperties: false, + required: ['slug', 'title', 'category'], + properties: { + slug: STRING, + title: STRING, + category: STRING, + source_paths: { type: 'array', items: STRING }, + citations: { type: 'array', items: CITATION }, + last_updated: STRING, + confidence: { type: 'string', enum: ['low', 'medium', 'high'] }, + status: { type: 'string', enum: ['current', 'needs-review', 'stale'] }, + generator: { type: 'string', enum: ['harness', 'router', 'heuristic'] }, + quality_issues: INT, + }, +}; + +export const STATUS = { + type: 'object', + additionalProperties: true, + required: ['phase', 'progress'], + properties: { + phase: { + type: 'string', + enum: ['queued', 'cloning', 'inventorying', 'planning', 'generating', 'linting', 'ready', 'error', 'unknown'], + }, + progress: NUM, + message: STRING, + error: STRING, + updated_at: STRING, + }, +}; + +const ID_REQ = { + type: 'object', + additionalProperties: false, + required: ['id'], + properties: { id: STRING }, +}; + +// ---------- generate ---------- +export const GENERATE_REQ = { + type: 'object', + additionalProperties: false, + required: ['repo_url'], + properties: { + repo_url: { ...STRING, description: 'HTTPS git URL of a public repository.' }, + ref: { ...STRING, description: 'Optional branch/tag/commit to check out (default: default branch).' }, + model: { ...STRING, description: 'Optional LLM model id, routed via llm-router.' }, + steer: { + type: 'object', + description: 'Optional per-repo steering (repo_notes, explicit pages, caps). Mirrors openwiki.json.', + additionalProperties: true, + }, + }, +}; +export const GENERATE_RES = { + type: 'object', + additionalProperties: false, + required: ['wiki_id', 'status'], + properties: { wiki_id: STRING, status: STRING }, +}; + +// ---------- refresh ---------- +export const REFRESH_RES = { + type: 'object', + additionalProperties: false, + required: ['wiki_id', 'refresh'], + properties: { + wiki_id: STRING, + refresh: { type: 'string', enum: ['regenerating', 'up_to_date', 'in_progress', 'error'] }, + changed: { + type: 'array', + items: { + type: 'object', + additionalProperties: false, + required: ['status', 'path'], + properties: { status: STRING, path: STRING }, + }, + }, + pages_affected: { type: 'array', items: STRING }, + }, +}; + +// ---------- status / wikis / wiki / pages / page / search ---------- +export const STATUS_REQ = ID_REQ; +export const STATUS_RES = STATUS; + +export const WIKIS_REQ = { type: 'object', additionalProperties: false, properties: {} }; +export const WIKIS_RES = { + type: 'object', + additionalProperties: false, + required: ['wikis'], + properties: { wikis: { type: 'array', items: WIKI_META } }, +}; + +export const MODELS_REQ = { type: 'object', additionalProperties: false, properties: {} }; +export const MODELS_RES = { + type: 'object', + additionalProperties: false, + required: ['models', 'default_model'], + properties: { + models: { + type: 'array', + items: { + type: 'object', + additionalProperties: false, + required: ['id', 'provider'], + properties: { + id: { type: 'string' }, + provider: { type: 'string' }, + supports_structured_output: { type: 'boolean' }, + }, + }, + }, + default_model: { type: 'string' }, + }, +}; + +export const WIKI_REQ = ID_REQ; +export const WIKI_RES = WIKI_META; + +export const PAGES_REQ = ID_REQ; +export const PAGES_RES = { + type: 'object', + additionalProperties: false, + required: ['pages'], + properties: { pages: { type: 'array', items: PAGE_META } }, +}; + +export const PAGE_REQ = { + type: 'object', + additionalProperties: false, + required: ['id', 'slug'], + properties: { id: STRING, slug: STRING }, +}; + +// A page-writer sub-agent stores its finished page directly, so the parent never +// carries the markdown. Agent-exposed but guarded to a generating wiki. +export const WRITE_PAGE_REQ = { + type: 'object', + additionalProperties: false, + required: ['id', 'slug', 'markdown'], + properties: { + id: STRING, + slug: STRING, + title: STRING, + category: STRING, + markdown: STRING, + source_paths: { type: 'array', items: STRING }, + citations: { + type: 'array', + items: { + type: 'object', + additionalProperties: false, + required: ['path'], + properties: { path: STRING, start_line: INT, end_line: INT, note: STRING }, + }, + }, + }, +}; +export const WRITE_PAGE_RES = { + type: 'object', + additionalProperties: false, + required: ['slug', 'ok'], + properties: { slug: STRING, ok: { type: 'boolean' } }, +}; +export const SET_SCHEDULE_REQ = { + type: 'object', + additionalProperties: false, + required: ['id', 'schedule'], + properties: { id: STRING, schedule: STRING }, +}; +export const SET_SCHEDULE_RES = { + type: 'object', + additionalProperties: false, + required: ['id', 'schedule', 'ok'], + properties: { id: STRING, schedule: STRING, ok: { type: 'boolean' } }, +}; +export const PAGE_RES = { + type: 'object', + additionalProperties: false, + required: ['slug', 'markdown'], + properties: { + slug: STRING, + title: STRING, + category: STRING, + source_paths: { type: 'array', items: STRING }, + citations: { type: 'array', items: CITATION }, + last_updated: STRING, + confidence: { type: 'string', enum: ['low', 'medium', 'high'] }, + status: { type: 'string', enum: ['current', 'needs-review', 'stale'] }, + generator: { type: 'string', enum: ['harness', 'router', 'heuristic'] }, + quality_issues: INT, + markdown: STRING, + }, +}; + +export const SEARCH_REQ = { + type: 'object', + additionalProperties: false, + required: ['id', 'q'], + properties: { id: STRING, q: STRING }, +}; +export const SEARCH_RES = { + type: 'object', + additionalProperties: false, + required: ['results'], + properties: { + results: { + type: 'array', + items: { + type: 'object', + additionalProperties: false, + required: ['slug', 'score'], + properties: { + slug: STRING, + title: STRING, + category: STRING, + source_paths: { type: 'array', items: STRING }, + score: NUM, + snippet: STRING, + matched: { type: 'array', items: STRING }, + }, + }, + }, + }, +}; + +// ---------- ask ---------- +export const ASK_REQ = { + type: 'object', + additionalProperties: false, + required: ['id', 'q'], + properties: { + id: STRING, + q: STRING, + mode: { type: 'string', enum: ['fast', 'deep'], description: 'fast = router retrieval; deep = harness multi-hop.' }, + file_answer: { ...BOOL, description: 'File a good answer back into the wiki as a new page.' }, + }, +}; +export const ASK_RES = { + type: 'object', + additionalProperties: false, + required: ['answer'], + properties: { + answer: STRING, + citations: { type: 'array', items: CITATION }, + filed_slug: STRING, + }, +}; + +// ---------- lint ---------- +export const LINT_REQ = ID_REQ; +export const LINT_RES = { + type: 'object', + additionalProperties: false, + required: ['checked', 'issues'], + properties: { + checked: INT, + issues: { + type: 'array', + items: { + type: 'object', + additionalProperties: false, + required: ['slug', 'kind'], + properties: { + slug: STRING, + kind: { + type: 'string', + enum: ['broken-citation', 'orphan', 'missing-xref', 'stale', 'thin'], + }, + detail: STRING, + }, + }, + }, + }, +}; + +// ---------- diagram ---------- +export const DIAGRAM_REQ = { + type: 'object', + additionalProperties: false, + required: ['id'], + properties: { + id: STRING, + kind: { type: 'string', enum: ['architecture', 'dataflow', 'deps'] }, + }, +}; +export const DIAGRAM_RES = { + type: 'object', + additionalProperties: false, + required: ['mermaid'], + properties: { mermaid: STRING, kind: STRING }, +}; + +// ---------- export-agents-md ---------- +export const EXPORT_AGENTS_REQ = { + type: 'object', + additionalProperties: false, + required: ['id'], + properties: { + id: STRING, + targets: { type: 'array', items: { type: 'string', enum: ['AGENTS.md', 'CLAUDE.md'] } }, + base_url: STRING, + }, +}; +export const EXPORT_AGENTS_RES = { + type: 'object', + additionalProperties: false, + required: ['content'], + properties: { content: STRING, targets: { type: 'array', items: STRING } }, +}; + +// ---------- harness page output contract ---------- +// The JSON a harness turn returns for one page. The model supplies path + line +// range + note only; openwiki fills the host deep-link url itself, so the +// model-facing citation schema deliberately omits `url`. +const MODEL_CITATION = { + type: 'object', + additionalProperties: false, + required: ['path'], + properties: { + path: STRING, + start_line: INT, + end_line: INT, + note: STRING, + }, +}; + +export const PAGE_HARNESS_OUT = { + type: 'object', + additionalProperties: false, + required: ['title', 'markdown'], + properties: { + title: STRING, + markdown: STRING, + citations: { type: 'array', items: MODEL_CITATION }, + links: { type: 'array', items: STRING }, + confidence: { type: 'string', enum: ['low', 'medium', 'high'] }, + status: { type: 'string', enum: ['current', 'needs-review'] }, + }, +}; + +// ---------- harness plan output contract ---------- +// The JSON a harness turn returns when planning a wiki after exploring the repo. +// navigation is a nested tree (folder = title + children, no slug; leaf = title +// + slug); pages is the flat list of leaves to generate. +const NAV_LEAF = { + type: 'object', + additionalProperties: false, + required: ['title', 'slug'], + properties: { title: STRING, slug: STRING }, +}; +const NAV_L2 = { + type: 'object', + additionalProperties: false, + required: ['title'], + properties: { title: STRING, slug: STRING, children: { type: 'array', items: NAV_LEAF } }, +}; +const NAV_L1 = { + type: 'object', + additionalProperties: false, + required: ['title'], + properties: { title: STRING, slug: STRING, children: { type: 'array', items: NAV_L2 } }, +}; + +export const PLAN_HARNESS_OUT = { + type: 'object', + additionalProperties: false, + required: ['summary', 'pages', 'navigation'], + properties: { + summary: STRING, + pages: { + type: 'array', + items: { + type: 'object', + additionalProperties: false, + required: ['slug', 'title'], + properties: { slug: STRING, title: STRING, brief: STRING, source_paths: { type: 'array', items: STRING } }, + }, + }, + navigation: { type: 'array', items: NAV_L1 }, + }, +}; + +// Nav tree node as stored on the wiki and read by the UI. +export const NAV_NODE = NAV_L1; + +// ---------- MCP structure surface ---------- +export const MCP_STRUCTURE_RES = { + type: 'object', + additionalProperties: false, + required: ['pages'], + properties: { + repo: STRING, + summary: STRING, + categories: { + type: 'array', + items: { + type: 'object', + additionalProperties: false, + required: ['id'], + properties: { id: STRING, title: STRING, description: STRING }, + }, + }, + pages: { + type: 'array', + items: { + type: 'object', + additionalProperties: false, + required: ['slug'], + properties: { slug: STRING, title: STRING, category: STRING }, + }, + }, + }, +}; + +// ---------- scoped source-read functions (the harness's exploration tools) ---------- +export const SRC_READ_REQ = { + type: 'object', + additionalProperties: false, + required: ['id', 'path'], + properties: { id: STRING, path: STRING, from: INT, to: INT }, +}; +export const SRC_READ_RES = { + type: 'object', + additionalProperties: false, + required: ['path', 'content'], + properties: { path: STRING, content: STRING, from: INT, to: INT, total_lines: INT, truncated: BOOL }, +}; +export const SRC_LIST_REQ = { + type: 'object', + additionalProperties: false, + required: ['id'], + properties: { id: STRING, dir: STRING }, +}; +export const SRC_LIST_RES = { + type: 'object', + additionalProperties: false, + required: ['files'], + properties: { + files: { + type: 'array', + items: { + type: 'object', + additionalProperties: false, + required: ['path'], + properties: { path: STRING, language: STRING, size: INT, priority: INT }, + }, + }, + truncated: BOOL, + }, +}; +export const SRC_GREP_REQ = { + type: 'object', + additionalProperties: false, + required: ['id', 'pattern'], + properties: { id: STRING, pattern: STRING, max: INT }, +}; +export const SRC_GREP_RES = { + type: 'object', + additionalProperties: false, + required: ['matches'], + properties: { + matches: { + type: 'array', + items: { + type: 'object', + additionalProperties: false, + required: ['path', 'line', 'text'], + properties: { path: STRING, line: INT, text: STRING }, + }, + }, + truncated: BOOL, + }, +}; diff --git a/openwiki/src/lib/search.mjs b/openwiki/src/lib/search.mjs new file mode 100644 index 000000000..19112c1cd --- /dev/null +++ b/openwiki/src/lib/search.mjs @@ -0,0 +1,82 @@ +import { listPages, getPage } from './store.mjs'; + +const STOPWORDS = new Set(['a', 'an', 'the', 'is', 'of', 'to', 'and', 'or', 'in', 'on']); + +function tokenize(s) { + if (!s) return []; + return String(s) + .toLowerCase() + .split(/\W+/) + .filter((t) => t && !STOPWORDS.has(t)); +} + +function countOccurrences(tokens, target) { + let n = 0; + for (const t of tokens) if (t === target) n++; + return n; +} + +// Snippets are raw text: the API returns data, and the UI renders them via +// textContent, so payload-level HTML escaping would double-escape. +function makeSnippet(body, queryTokens) { + const lower = body.toLowerCase(); + let idx = -1; + for (const t of queryTokens) { + const i = lower.indexOf(t); + if (i !== -1 && (idx === -1 || i < idx)) idx = i; + } + if (idx === -1) { + const head = body.slice(0, 200); + return head + (body.length > 200 ? '…' : ''); + } + const start = Math.max(0, idx - 80); + const end = Math.min(body.length, idx + 120); + let snip = body.slice(start, end); + if (start > 0) snip = `…${snip}`; + if (end < body.length) snip = `${snip}…`; + return snip; +} + +export async function searchPages(wikiId, query, opts = { limit: 20 }) { + const limit = opts?.limit ?? 20; + const qTokens = tokenize(query); + if (qTokens.length === 0) return []; + const uniqQ = [...new Set(qTokens)]; + + const pages = await listPages(wikiId); + const results = []; + + for (const { slug, meta } of pages) { + const page = await getPage(wikiId, slug); + if (!page) continue; + const body = page.markdown || ''; + const titleTokens = tokenize(meta.title || ''); + const catTokens = tokenize(meta.category || ''); + const bodyTokens = tokenize(body); + + let score = 0; + const matched = []; + for (const q of uniqQ) { + const ct = countOccurrences(titleTokens, q); + const cc = countOccurrences(catTokens, q); + const cb = countOccurrences(bodyTokens, q); + const s = 5 * ct + 2 * cc + cb; + if (s > 0) matched.push(q); + score += s; + } + if (score <= 0) continue; + + results.push({ + slug, + title: meta.title, + category: meta.category, + source_paths: meta.source_paths, + score, + snippet: makeSnippet(body, uniqQ), + matched, + }); + } + + results.sort((a, b) => b.score - a.score); + return results.slice(0, limit); +} diff --git a/openwiki/src/lib/src.mjs b/openwiki/src/lib/src.mjs new file mode 100644 index 000000000..2f6833903 --- /dev/null +++ b/openwiki/src/lib/src.mjs @@ -0,0 +1,164 @@ +// Scoped source-read functions exposed to the harness. Each is jailed to one +// wiki's clone directory (repoDir(wikiId)); the harness calls them via +// agent_trigger to explore the repo and cite exact line ranges. Single-reader +// surface — the orchestrator remains the single writer. +import path from 'node:path'; +import fs from 'node:fs/promises'; +import { repoDir } from './store.mjs'; +import { inventoryRepo, readSourceFile } from './inventory.mjs'; +import { lineWindow } from './harness.mjs'; + +const MAX_GREP_FILES = 2000; + +// A harness turn calls src::list / src::grep many times per page; walking the +// clone each time is wasteful. Cache the inventory per wiki and invalidate when +// the clone changes (generation/refresh call invalidateInventory after cloning). +const invCache = new Map(); +async function inventory(wikiId) { + if (invCache.has(wikiId)) return invCache.get(wikiId); + const v = await inventoryRepo(repoDir(wikiId)); + invCache.set(wikiId, v); + return v; +} +export function invalidateInventory(wikiId) { + invCache.delete(wikiId); +} + +// Per-wiki read accounting: how much source material the agent actually pulled +// during a generation. Input token cost is dominated by this (plus per-turn +// context accumulation, which multiplies it). Used to measure real cost. +const readStats = new Map(); +function account(wikiId, field, n) { + const s = readStats.get(wikiId) || { + read_calls: 0, + read_bytes: 0, + list_calls: 0, + list_bytes: 0, + grep_calls: 0, + grep_bytes: 0, + }; + s[field] += n; + readStats.set(wikiId, s); +} +export function getReadStats(wikiId) { + return readStats.get(wikiId) || null; +} +export function resetReadStats(wikiId) { + readStats.delete(wikiId); +} + +function pathEscapeError() { + const e = new Error('path escapes repository'); + e.code = 'openwiki/path_escape'; + return e; +} +function contained(base, abs) { + return abs === base || abs.startsWith(base + path.sep); +} + +// Reject any path that escapes the wiki's clone directory, lexically AND after +// resolving symlinks (a symlink inside a clone can point outside it). +async function guard(root, rel) { + const base = path.resolve(root); + const abs = path.resolve(base, rel); + if (!contained(base, abs)) throw pathEscapeError(); + try { + const [realBase, realAbs] = await Promise.all([fs.realpath(base), fs.realpath(abs)]); + if (!contained(realBase, realAbs)) throw pathEscapeError(); + } catch (e) { + if (e.code === 'openwiki/path_escape') throw e; + // ENOENT: the file does not exist; readSourceFile will surface that. + } + return abs; +} + +export async function srcRead(wikiId, rel, from, to) { + const dir = repoDir(wikiId); + await guard(dir, rel); + const { content, truncated } = await readSourceFile(dir, rel, 200_000); + const w = lineWindow(content, from, to); + account(wikiId, 'read_calls', 1); + account(wikiId, 'read_bytes', w.text.length); + return { + path: rel, + content: w.text, + from: w.from, + to: w.to, + total_lines: w.total_lines, + truncated: truncated || w.truncated, + }; +} + +export async function srcList(wikiId, subdir) { + const inv = await inventory(wikiId); + let files = inv; + if (subdir) { + const pfx = `${String(subdir).replace(/\/+$/, '')}/`; + files = inv.filter((e) => e.relPath.startsWith(pfx)); + } + const truncated = files.length > 500; + const out = files + .slice(0, 500) + .map((e) => ({ path: e.relPath, language: e.language, size: e.size, priority: e.priority })); + account(wikiId, 'list_calls', 1); + account( + wikiId, + 'list_bytes', + out.reduce((n, f) => n + f.path.length + 20, 0), + ); + return { files: out, truncated }; +} + +const MAX_GREP_PATTERN = 256; +const GREP_DEADLINE_MS = 5_000; + +export async function srcGrep(wikiId, pattern, max = 200) { + let re; + if (String(pattern ?? '').length > MAX_GREP_PATTERN) return { matches: [], truncated: false }; + try { + re = new RegExp(pattern, 'i'); + } catch { + return { matches: [], truncated: false }; + } + const dir = repoDir(wikiId); + const inv = await inventory(wikiId); + const matches = []; + let truncated = false; + let scanned = 0; + const deadline = Date.now() + GREP_DEADLINE_MS; + for (const e of inv) { + if (Date.now() > deadline) { + truncated = true; + break; + } + if (matches.length >= max || scanned >= MAX_GREP_FILES) { + truncated = true; + break; + } + if (e.language === 'text' && (e.priority ?? 0) <= 0) continue; // skip binary-ish + scanned += 1; + let content; + try { + ({ content } = await readSourceFile(dir, e.relPath, 200_000)); + } catch { + continue; + } + const lines = content.split(/\r?\n/); + for (let i = 0; i < lines.length; i++) { + if (re.test(lines[i])) { + matches.push({ path: e.relPath, line: i + 1, text: lines[i].slice(0, 300) }); + if (matches.length >= max) { + truncated = true; + break; + } + } + } + } + account(wikiId, 'grep_calls', 1); + account( + wikiId, + 'grep_bytes', + matches.reduce((n, m) => n + m.text.length + m.path.length, 0), + ); + return { matches, truncated }; +} diff --git a/openwiki/src/lib/store.mjs b/openwiki/src/lib/store.mjs new file mode 100644 index 000000000..062c9df36 --- /dev/null +++ b/openwiki/src/lib/store.mjs @@ -0,0 +1,236 @@ +// iii-state backed store for openwiki. Wiki content (metadata, pages, status, +// outline, logs) lives in iii-state under `openwiki:*` scopes; cloned repos are +// ephemeral working dirs on the local filesystem (a git clone cannot live in a +// key/value store). +// +// Scaling note: `state::list` returns every value in a scope. Enumerating the +// pages scope pulls every markdown body — a large wiki can produce a multi-MB +// response that blocks the worker event loop. So pages are indexed by a single +// lightweight side-record per wiki (`openwiki:page-index` -> [{slug, meta, hash}]) +// maintained on write; the hot paths (listPages, refresh mapping, content hash) +// read the index and never enumerate bodies. +import fs from 'node:fs/promises'; +import path from 'node:path'; +import crypto from 'node:crypto'; + +const REPO_ROOT = process.env.OPENWIKI_DATA || '/tmp/openwiki-data'; +export const repoDir = (id) => path.join(REPO_ROOT, 'repos', id); + +let worker = null; +/** Wire the iii worker used for all state calls. Call once at startup. */ +export function setWorker(c) { + worker = c; +} + +const S_WIKIS = 'openwiki:wikis'; +const S_STATUS = 'openwiki:status'; +const S_OUTLINE = 'openwiki:outline'; +const S_LOG = 'openwiki:log'; +const S_PAGE_INDEX = 'openwiki:page-index'; // wikiId -> [{ slug, meta, hash }] +const pagesScope = (id) => `openwiki:pages:${id}`; + +async function sget(scope, key) { + const res = await worker.trigger({ function_id: 'state::get', payload: { scope, key } }); + return res == null ? null : res; +} +async function sset(scope, key, value) { + await worker.trigger({ function_id: 'state::set', payload: { scope, key, value } }); +} +async function slist(scope) { + const res = await worker.trigger({ function_id: 'state::list', payload: { scope } }); + if (Array.isArray(res)) return res; + if (res && Array.isArray(res.values)) return res.values; + return []; +} +async function sdel(scope, key) { + try { + await worker.trigger({ function_id: 'state::delete', payload: { scope, key } }); + } catch { + /* best effort */ + } +} + +function sha256(s) { + return crypto + .createHash('sha256') + .update(String(s ?? ''), 'utf8') + .digest('hex'); +} + +/** Ensure the local working area for repo clones exists (fs, ephemeral). */ +export async function ensureRoot() { + await fs.mkdir(path.join(REPO_ROOT, 'repos'), { recursive: true }); + await fs.mkdir(path.join(REPO_ROOT, 'tmp'), { recursive: true }); +} + +// ---------- wikis ---------- + +export async function saveWiki(id, meta) { + await sset(S_WIKIS, id, meta); +} +export async function getWiki(id) { + return sget(S_WIKIS, id); +} +export async function listWikis() { + const all = await slist(S_WIKIS); + return all.sort((a, b) => String(b.updated_at || '').localeCompare(String(a.updated_at || ''))); +} + +// ---------- page index (side-record; never enumerates bodies) ---------- + +async function getIndex(wikiId) { + const idx = await sget(S_PAGE_INDEX, wikiId); + return Array.isArray(idx) ? idx : []; +} +async function setIndex(wikiId, idx) { + await sset(S_PAGE_INDEX, wikiId, idx); +} + +// Serialize per-wiki read-modify-write cycles. Pages generate concurrently +// (batch writers, spawned sub-agents), so an unguarded +// get -> mutate -> set would lose updates. One promise chain per key per map. +function withLock(locks, key, fn) { + const prev = locks.get(key) || Promise.resolve(); + const run = prev.then(fn, fn); + locks.set( + key, + run.catch(() => {}), + ); + return run; +} +const indexLocks = new Map(); +const logLocks = new Map(); +const withIndexLock = (wikiId, fn) => withLock(indexLocks, wikiId, fn); +const withLogLock = (wikiId, fn) => withLock(logLocks, wikiId, fn); + +// Rebuild the index from the pages scope. Migration / self-heal path only — +// runs once when a wiki has pages but no index (pre-index wikis). +async function rebuildIndex(wikiId) { + const all = await slist(pagesScope(wikiId)); + const idx = all.map((p) => ({ slug: p.slug, meta: p.meta, hash: sha256(p.markdown || '') })); + await setIndex(wikiId, idx); + return idx; +} + +// ---------- pages ---------- + +export async function savePage(wikiId, slug, markdown, meta) { + await sset(pagesScope(wikiId), slug, { slug, markdown, meta }); + await withIndexLock(wikiId, async () => { + const idx = await getIndex(wikiId); + const entry = { slug, meta, hash: sha256(markdown || '') }; + const i = idx.findIndex((e) => e.slug === slug); + if (i >= 0) idx[i] = entry; + else idx.push(entry); + await setIndex(wikiId, idx); + }); +} + +export async function getPage(wikiId, slug) { + const p = await sget(pagesScope(wikiId), slug); + return p ? { markdown: p.markdown, meta: p.meta } : null; +} + +export async function listPages(wikiId) { + let idx = await getIndex(wikiId); + if (idx.length === 0) { + // Self-heal a pre-index wiki without enumerating bodies on every call. + const all = await slist(pagesScope(wikiId)); + if (all.length > 0) idx = await rebuildIndex(wikiId); + } + return idx.map((e) => ({ slug: e.slug, meta: e.meta })); +} + +export async function deletePage(wikiId, slug) { + await sdel(pagesScope(wikiId), slug); + await withIndexLock(wikiId, async () => { + const idx = await getIndex(wikiId); + const next = idx.filter((e) => e.slug !== slug); + if (next.length !== idx.length) await setIndex(wikiId, next); + }); +} + +// Remove a wiki entirely: every page body, the page index, all per-wiki side +// records, the wiki meta, and the ephemeral clone. Best-effort per key so a +// partial store still clears the wiki from the list. +export async function deleteWiki(wikiId) { + // sdel already swallows per-key errors, so the deletions below are each + // best-effort on their own. Guard the page enumeration too: if getIndex or + // slist throws we still want the side records and the wiki meta removed, so a + // transient read error cannot strand the wiki in the list. + let slugs = []; + try { + const idx = await getIndex(wikiId); + slugs = idx.length ? idx.map((e) => e.slug) : (await slist(pagesScope(wikiId))).map((p) => p.slug); + } catch { + /* enumeration failed; still clear the records below */ + } + for (const slug of slugs) await sdel(pagesScope(wikiId), slug); + await sdel(S_PAGE_INDEX, wikiId); + await sdel(S_OUTLINE, wikiId); + await sdel(S_STATUS, wikiId); + await sdel(S_LOG, wikiId); + await sdel(S_WIKIS, wikiId); + try { + await fs.rm(repoDir(wikiId), { recursive: true, force: true }); + } catch { + /* ephemeral */ + } +} + +// Slugs whose source_paths or citations touch any of `changedPaths`. Drives +// incremental refresh — reads only the lightweight index, never page bodies. +export async function pagesForPaths(wikiId, changedPaths) { + const want = new Set((changedPaths || []).map(String)); + if (want.size === 0) return []; + const idx = await getIndex(wikiId); + const hit = []; + for (const e of idx) { + const paths = new Set(); + for (const p of e.meta?.source_paths || []) paths.add(String(p)); + for (const c of e.meta?.citations || []) if (c?.path) paths.add(String(c.path)); + for (const p of paths) + if (want.has(p)) { + hit.push(e.slug); + break; + } + } + return hit; +} + +// Anti-churn digest over the ordered page bodies, derived from the index hashes +// (no body enumeration). Stable for identical content regardless of write order. +export async function computeContentHash(wikiId) { + const idx = await getIndex(wikiId); + const parts = idx.map((e) => `${e.slug}:${e.hash}`).sort(); + return sha256(parts.join('\n')); +} + +// ---------- outline / status / log ---------- + +export async function saveOutline(wikiId, outline) { + await sset(S_OUTLINE, wikiId, outline); +} +export async function getOutline(wikiId) { + return sget(S_OUTLINE, wikiId); +} + +export async function updateStatus(wikiId, status) { + await sset(S_STATUS, wikiId, { ...status, updated_at: status.updated_at || new Date().toISOString() }); +} +export async function getStatus(wikiId) { + return sget(S_STATUS, wikiId); +} + +export async function appendLog(wikiId, line) { + const ts = new Date().toISOString().replace(/\.\d{3}Z$/, 'Z'); + await withLogLock(wikiId, async () => { + const prev = (await sget(S_LOG, wikiId)) || []; + prev.push(`${ts} ${line}`); + if (prev.length > 500) prev.splice(0, prev.length - 500); + await sset(S_LOG, wikiId, prev); + }); +} +export async function getLog(wikiId) { + return (await sget(S_LOG, wikiId)) || []; +} diff --git a/openwiki/src/lib/turnbus.mjs b/openwiki/src/lib/turnbus.mjs new file mode 100644 index 000000000..842df05b4 --- /dev/null +++ b/openwiki/src/lib/turnbus.mjs @@ -0,0 +1,94 @@ +// Real-time collection of harness turns via `harness::turn-completed` events, +// replacing per-turn `harness::status` polling. openwiki registers ONE trigger +// on the harness's emitted turn-completed type (see index.mjs) and routes each +// event here. A generation registers its root session; the router delivers the +// plan turn (matched by session_id == root) and each page child (matched by +// event.parent_session_id == root, which only harness::spawn stamps — a plain +// send leaves it null, harness send.rs sets display_parent_session_id: None). +// +// One in-process map keyed by root session id. Single-process worker; if +// openwiki ever runs multiple instances, the subscription is per-engine so each +// instance sees every event and ignores roots it does not own (cheap map miss). + +const active = new Map(); // root session id -> collector + +// Register a collector for one generation. Returns an unregister handle. +// onPlan(result) called once when the root/plan turn completes +// onPage(childId, result) called per page child completion (ok) +// onPageError(childId, err) called per page child that failed/cancelled +// onSpawn(childId) called when a page-writer sub-agent STARTS +export function register(rootSessionId, { onPlan, onPage, onPageError, onSpawn } = {}) { + active.set(rootSessionId, { onPlan, onPage, onPageError, onSpawn }); + return () => active.delete(rootSessionId); +} + +export function unregister(rootSessionId) { + active.delete(rootSessionId); +} + +export function isActive(rootSessionId) { + return active.has(rootSessionId); +} + +// Route a harness::turn-started event: a child sub-agent under a root we own +// just began, so signal its spawn (for live progress). The parent's own start +// (session_id === root, no parent) is ignored. +export function deliverStarted(evt) { + if (!evt || typeof evt !== 'object') return false; + const pid = evt.parent_session_id; + if (pid && active.has(pid)) { + try { + active.get(pid).onSpawn?.(evt.session_id); + } catch { + /* isolate */ + } + return true; + } + return false; +} + +// Route one harness::turn-completed event payload to its generation, if any. +// Event shape (harness events.rs): { session_id, turn_id, status, result?, +// result_error?, parent_session_id? }. Returns true when it matched a root. +export function deliver(evt) { + if (!evt || typeof evt !== 'object') return false; + const sid = evt.session_id; + const pid = evt.parent_session_id; + // Page child: parent_session_id points at a root we own. + if (pid && active.has(pid)) { + const c = active.get(pid); + if (evt.status === 'completed') { + try { + c.onPage?.(sid, evt.result); + } catch { + /* isolate */ + } + } else { + try { + c.onPageError?.(sid, evt.result_error || evt.status); + } catch { + /* isolate */ + } + } + return true; + } + // Plan/root turn: the root session itself completed. + if (sid && active.has(sid)) { + const c = active.get(sid); + if (evt.status === 'completed') { + try { + c.onPlan?.(evt.result); + } catch { + /* isolate */ + } + } else { + try { + c.onPageError?.(sid, evt.result_error || evt.status); + } catch { + /* isolate */ + } + } + return true; + } + return false; +} diff --git a/openwiki/src/lib/ui.mjs b/openwiki/src/lib/ui.mjs new file mode 100644 index 000000000..9ebbcdd9a --- /dev/null +++ b/openwiki/src/lib/ui.mjs @@ -0,0 +1,1432 @@ +// OpenWiki browser UI. Single-page app served inline by the worker. +// No npm deps; vanilla HTML/CSS/JS. Mermaid loads on demand from a CDN for the +// diagram view and falls back to rendering the source when blocked. + +export const INDEX_HTML = String.raw` + + + + +openwiki + + + + + + +
+
+
OpenWiki
+ +
+
+
+ +
+ +
+ +
+ +
+ +
+ + + +`; diff --git a/openwiki/tests/config.test.mjs b/openwiki/tests/config.test.mjs new file mode 100644 index 000000000..ccb4c672a --- /dev/null +++ b/openwiki/tests/config.test.mjs @@ -0,0 +1,31 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import * as configuration from '../src/lib/configuration.mjs'; + +test('config has sane defaults', () => { + const d = configuration.defaults(); + assert.equal(typeof d.model, 'string'); + assert.ok(d.max_parallel >= 1); +}); + +test('fetchConfig merges the stored value over defaults', async () => { + const worker = { + async trigger({ function_id }) { + if (function_id === 'configuration::get') return { value: { max_parallel: 7 } }; + return {}; + }, + }; + const c = await configuration.fetchConfig(worker); + assert.equal(c.max_parallel, 7); + assert.ok(c.model, 'default model preserved when not overridden'); +}); + +test('fetchConfig falls back to defaults when the config worker errors', async () => { + const worker = { + async trigger() { + throw new Error('no configuration worker'); + }, + }; + const c = await configuration.fetchConfig(worker); + assert.equal(c.max_parallel, configuration.defaults().max_parallel); +}); diff --git a/openwiki/tests/docs.test.mjs b/openwiki/tests/docs.test.mjs new file mode 100644 index 000000000..36e4fa44a --- /dev/null +++ b/openwiki/tests/docs.test.mjs @@ -0,0 +1,38 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { parseLlmsTxt, candidateOrigins, docsBudget, docsHint } from '../src/lib/docs_oracle.mjs'; + +test('parseLlmsTxt groups links under section headings', () => { + const txt = + '# Docs\n\n## Get Started\n- [Install](https://x.com/install)\n- [Quickstart](https://x.com/qs)\n\n## Reference\n- [API](https://x.com/api)'; + const p = parseLlmsTxt(txt); + assert.equal(p.links.length, 3); + assert.equal(p.sections.length, 2); + assert.equal(p.sections[0].title, 'Get Started'); + assert.equal(p.sections[0].links.length, 2); +}); + +test('candidateOrigins keeps doc hosts, skips github/badge hosts', () => { + const readme = 'see https://docs.example.com/guide and https://github.com/o/r and https://shields.io/x'; + assert.deepEqual(candidateOrigins('', readme), ['https://docs.example.com']); +}); + +test('docsBudget scales with link count', () => { + assert.equal(docsBudget(5), 18); + assert.equal(docsBudget(50), 26); + assert.equal(docsBudget(300), 48); +}); + +test('docsHint mentions sections and a page budget, empty for null', () => { + const h = docsHint({ + source: 'x/llms.txt', + linkCount: 100, + sections: [ + { title: 'Learn', links: [1] }, + { title: 'Reference', links: [1] }, + ], + }); + assert.match(h, /Learn, Reference/); + assert.match(h, /Target about 34 pages/); + assert.equal(docsHint(null), ''); +}); diff --git a/openwiki/tests/git.test.mjs b/openwiki/tests/git.test.mjs new file mode 100644 index 000000000..bedcc89c7 --- /dev/null +++ b/openwiki/tests/git.test.mjs @@ -0,0 +1,21 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { parseNameStatus, repoName } from '../src/lib/git.mjs'; + +test('repoName extracts owner/repo from common url shapes', () => { + assert.equal(repoName('https://github.com/owner/repo.git'), 'owner/repo'); + assert.equal(repoName('git@github.com:owner/repo.git'), 'owner/repo'); + assert.equal(repoName('https://github.com/owner/repo/'), 'owner/repo'); + assert.equal(repoName('https://example.com/deep/path/owner/repo'), 'owner/repo'); +}); + +test('parseNameStatus handles added/modified/deleted/renamed and blanks', () => { + const out = parseNameStatus('M\tsrc/a.ts\nA\tsrc/b.ts\nD\tsrc/c.ts\nR100\told.ts\tnew.ts\n\n'); + assert.deepEqual(out, [ + { status: 'M', path: 'src/a.ts' }, + { status: 'A', path: 'src/b.ts' }, + { status: 'D', path: 'src/c.ts' }, + { status: 'R', path: 'new.ts' }, + ]); + assert.deepEqual(parseNameStatus(''), []); +}); diff --git a/openwiki/tests/harness.test.mjs b/openwiki/tests/harness.test.mjs new file mode 100644 index 000000000..7743b8661 --- /dev/null +++ b/openwiki/tests/harness.test.mjs @@ -0,0 +1,44 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { citationUrl, lineWindow, mapResult } from '../src/lib/harness.mjs'; + +test('citationUrl builds a GitHub blob permalink at the pinned commit', () => { + assert.equal( + citationUrl('https://github.com/owner/repo', 'abc', 'src/a.ts', 10, 24), + 'https://github.com/owner/repo/blob/abc/src/a.ts#L10-L24', + ); + assert.equal( + citationUrl('git@github.com:owner/repo.git', 'abc', 'x.ts', 5), + 'https://github.com/owner/repo/blob/abc/x.ts#L5', + ); + assert.equal(citationUrl('https://gitlab.com/o/r', 'abc', 'x.ts', 5), null); + assert.equal(citationUrl('https://github.com/o/r', null, 'x.ts', 5), null); +}); + +test('lineWindow slices inclusive 1-indexed ranges', () => { + const c = 'l1\nl2\nl3\nl4'; + assert.deepEqual(lineWindow(c, 2, 3), { text: 'l2\nl3', from: 2, to: 3, total_lines: 4, truncated: true }); + assert.deepEqual(lineWindow(c), { text: c, from: 1, to: 4, total_lines: 4, truncated: false }); +}); + +test('mapResult attaches citation urls, source paths, and defaults', () => { + const out = mapResult( + { title: 'T', markdown: '# T\nbody', citations: [{ path: 'a.ts', start_line: 1, end_line: 2 }] }, + { + outlineItem: { slug: 's', category: 'c', source_paths: ['a.ts'] }, + repoUrl: 'https://github.com/o/r', + commit: 'sha', + }, + ); + assert.equal(out.frontmatter.generator, 'harness'); + assert.equal(out.frontmatter.citations[0].url, 'https://github.com/o/r/blob/sha/a.ts#L1-L2'); + assert.equal(out.frontmatter.confidence, 'medium'); + assert.equal(out.frontmatter.status, 'current'); + assert.ok(out.frontmatter.source_paths.includes('a.ts')); +}); + +test('mapResult throws on empty markdown', () => { + assert.throws(() => + mapResult({ title: 'T', markdown: ' ' }, { outlineItem: { slug: 's' }, repoUrl: '', commit: '' }), + ); +}); diff --git a/openwiki/tests/model.test.mjs b/openwiki/tests/model.test.mjs new file mode 100644 index 000000000..820bbbda4 --- /dev/null +++ b/openwiki/tests/model.test.mjs @@ -0,0 +1,27 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { pickModel } from '../src/lib/model.mjs'; + +const MODELS = [ + { id: 'a', provider: 'p1', supports_tools: true }, + { id: 'b', provider: 'p2', supports_structured_output: true, supports_tools: true }, +]; + +test('pickModel prefers the requested id when present', () => { + const r = pickModel(MODELS, 'a'); + assert.equal(r.model, 'a'); + assert.equal(r.provider, 'p1'); + assert.equal(r.resolved, true); +}); + +test('pickModel falls back to a structured-output + tools model', () => { + const r = pickModel(MODELS, 'missing'); + assert.equal(r.model, 'b'); + assert.equal(r.supports_structured_output, true); +}); + +test('pickModel returns unresolved when the catalog is empty', () => { + const r = pickModel([], 'x'); + assert.equal(r.resolved, false); + assert.equal(r.model, 'x'); +}); diff --git a/openwiki/tests/nav.test.mjs b/openwiki/tests/nav.test.mjs new file mode 100644 index 000000000..9fa0c636a --- /dev/null +++ b/openwiki/tests/nav.test.mjs @@ -0,0 +1,63 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { normalizePlan, navSlugs, slugToSection, navFromCategories } from '../src/lib/nav.mjs'; + +test('normalizePlan (harness shape) keeps nav and derives page sections', () => { + const planned = { + summary: 's', + pages: [ + { slug: 'overview', title: 'Overview' }, + { slug: 'api', title: 'API' }, + ], + navigation: [ + { title: 'Start Here', children: [{ title: 'Overview', slug: 'overview' }] }, + { title: 'Reference', children: [{ title: 'API', slug: 'api' }] }, + ], + }; + const n = normalizePlan(planned); + assert.equal(n.outline.length, 2); + assert.equal(n.outline.find((o) => o.slug === 'api').category, 'Reference'); + assert.equal(n.navigation.length, 2); +}); + +test('normalizePlan (fallback shape) builds nav from categories', () => { + const planned = { + summary: 's', + categories: [{ id: 'c1', title: 'Cat1' }], + outline: [{ slug: 'a', title: 'A', category: 'c1' }], + }; + const n = normalizePlan(planned); + assert.equal(n.navigation[0].title, 'Cat1'); + assert.equal(n.navigation[0].children[0].slug, 'a'); + assert.equal(n.outline[0].category, 'Cat1'); +}); + +test('navSlugs flattens nested leaves', () => { + const nav = [ + { + title: 'F', + children: [ + { title: 'G', children: [{ title: 'L', slug: 'l' }] }, + { title: 'M', slug: 'm' }, + ], + }, + ]; + assert.deepEqual(navSlugs(nav).sort(), ['l', 'm']); +}); + +test('slugToSection maps deep leaves to their top folder', () => { + const nav = [{ title: 'Top', children: [{ title: 'Sub', children: [{ title: 'Deep', slug: 'd' }] }] }]; + assert.equal(slugToSection(nav).d, 'Top'); +}); + +test('navFromCategories puts uncategorized pages under Other', () => { + const nav = navFromCategories( + [{ id: 'c', title: 'C' }], + [ + { slug: 'a', title: 'A', category: 'c' }, + { slug: 'b', title: 'B' }, + ], + ); + assert.equal(nav.find((f) => f.title === 'C').children[0].slug, 'a'); + assert.equal(nav.find((f) => f.title === 'Other').children[0].slug, 'b'); +}); diff --git a/openwiki/tests/quality.test.mjs b/openwiki/tests/quality.test.mjs new file mode 100644 index 000000000..ceeda4b7b --- /dev/null +++ b/openwiki/tests/quality.test.mjs @@ -0,0 +1,32 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { countWords, countH2, getPageQualityIssues } from '../src/lib/quality.mjs'; + +test('countWords excludes code blocks, paths, urls, and Sources lines', () => { + const md = + '# T\nHello world here.\n```js\nlots of code words do not count\n```\nSee src/a.ts and http://x.com\nSources: src/b.ts, src/c.ts'; + const n = countWords(md); + assert.ok(n >= 4 && n <= 8, `expected ~6 prose words, got ${n}`); +}); + +test('countH2 counts only level-2 headings', () => { + assert.equal(countH2('## A\n### b\n## C\ntext'), 2); +}); + +test('getPageQualityIssues flags a thin, unstructured page', () => { + const issues = getPageQualityIssues('# T\nshort', { minWords: 50 }); + assert.ok(issues.some((i) => /level-2/.test(i))); + assert.ok(issues.some((i) => /Relevant Source Files/.test(i))); + assert.ok(issues.some((i) => /Sources:/.test(i))); + assert.ok(issues.some((i) => /words of prose/.test(i))); +}); + +test('getPageQualityIssues passes a rich, grounded page', () => { + const good = + '# Title\n\n## Purpose and Scope\n' + + 'word '.repeat(60) + + '\n\n## Relevant Source Files\n- `src/a.ts` matters.\n\n## Execution Flow\n' + + 'word '.repeat(60) + + '\nSources: src/a.ts'; + assert.deepEqual(getPageQualityIssues(good, { minWords: 50 }), []); +}); diff --git a/openwiki/tests/src.test.mjs b/openwiki/tests/src.test.mjs new file mode 100644 index 000000000..78aedcaae --- /dev/null +++ b/openwiki/tests/src.test.mjs @@ -0,0 +1,65 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import os from 'node:os'; + +// Build a throwaway clone under OPENWIKI_DATA before importing src.mjs (store.mjs +// reads OPENWIKI_DATA at module load to compute repoDir). +const root = await fs.mkdtemp(path.join(os.tmpdir(), 'ow-src-')); +process.env.OPENWIKI_DATA = root; +const wikiId = 'w-test'; +const repo = path.join(root, 'repos', wikiId); +await fs.mkdir(path.join(repo, 'src'), { recursive: true }); +await fs.writeFile(path.join(repo, 'README.md'), '# Demo\nhello world\n'); +await fs.writeFile(path.join(repo, 'src', 'a.ts'), 'export const x = 1;\nexport const y = 2;\nconsole.log(x);\n'); + +const src = await import('../src/lib/src.mjs'); + +test('srcList lists files with metadata', async () => { + const { files } = await src.srcList(wikiId); + const paths = files.map((f) => f.path); + assert.ok(paths.includes('README.md')); + assert.ok(paths.includes('src/a.ts')); +}); + +test('srcList honors the subdir filter', async () => { + const { files } = await src.srcList(wikiId, 'src'); + assert.ok(files.length >= 1); + assert.ok(files.every((f) => f.path.startsWith('src/'))); +}); + +test('srcRead returns an inclusive line window', async () => { + const r = await src.srcRead(wikiId, 'src/a.ts', 2, 3); + assert.equal(r.from, 2); + assert.equal(r.to, 3); + assert.match(r.content, /y = 2/); + assert.ok(r.total_lines >= 3); +}); + +test('srcRead rejects path traversal', async () => { + await assert.rejects(() => src.srcRead(wikiId, '../../etc/passwd'), /escapes/); +}); + +test('srcGrep finds matches with line numbers', async () => { + const { matches } = await src.srcGrep(wikiId, 'export const'); + assert.ok(matches.length >= 2); + assert.ok(matches.every((m) => m.line > 0 && m.path && typeof m.text === 'string')); +}); + +test('invalidateInventory forces a re-walk so new files appear', async () => { + await src.srcList(wikiId); // populate cache + await fs.writeFile(path.join(repo, 'NEW.md'), 'new file'); + let { files } = await src.srcList(wikiId); + assert.equal( + files.some((f) => f.path === 'NEW.md'), + false, + ); // still cached + + src.invalidateInventory(wikiId); + ({ files } = await src.srcList(wikiId)); + assert.equal( + files.some((f) => f.path === 'NEW.md'), + true, + ); // fresh walk +}); diff --git a/openwiki/tests/store.test.mjs b/openwiki/tests/store.test.mjs new file mode 100644 index 000000000..9c920d850 --- /dev/null +++ b/openwiki/tests/store.test.mjs @@ -0,0 +1,119 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import * as store from '../src/lib/store.mjs'; + +// In-memory mock of the iii worker's state:: functions. +function mockWorker() { + const db = new Map(); // scope -> Map(key -> value) + const scoped = (s) => { + if (!db.has(s)) db.set(s, new Map()); + return db.get(s); + }; + return { + async trigger({ function_id, payload }) { + const { scope, key, value } = payload || {}; + if (function_id === 'state::set') { + scoped(scope).set(key, value); + return {}; + } + if (function_id === 'state::get') { + return scoped(scope).has(key) ? scoped(scope).get(key) : null; + } + if (function_id === 'state::list') { + return [...scoped(scope).values()]; + } + if (function_id === 'state::delete') { + scoped(scope).delete(key); + return {}; + } + throw new Error(`unexpected ${function_id}`); + }, + }; +} + +test('wiki round-trip via iii-state, listed newest first', async () => { + store.setWorker(mockWorker()); + await store.saveWiki('w1', { id: 'w1', repo_name: 'a/b', updated_at: '2026-01-02' }); + await store.saveWiki('w2', { id: 'w2', repo_name: 'c/d', updated_at: '2026-01-03' }); + assert.equal((await store.getWiki('w1')).repo_name, 'a/b'); + const all = await store.listWikis(); + assert.equal(all.length, 2); + assert.equal(all[0].id, 'w2'); +}); + +test('page save / get / list / delete', async () => { + store.setWorker(mockWorker()); + await store.savePage('w1', 'overview', '# Overview\n\nbody', { title: 'Overview', category: 'overview' }); + const p = await store.getPage('w1', 'overview'); + assert.equal(p.meta.title, 'Overview'); + assert.match(p.markdown, /# Overview/); + const pages = await store.listPages('w1'); + assert.equal(pages.length, 1); + assert.equal(pages[0].slug, 'overview'); + await store.deletePage('w1', 'overview'); + assert.equal(await store.getPage('w1', 'overview'), null); +}); + +test('status stamps updated_at; log appends and caps', async () => { + store.setWorker(mockWorker()); + await store.updateStatus('w1', { phase: 'ready', progress: 1 }); + const s = await store.getStatus('w1'); + assert.equal(s.phase, 'ready'); + assert.ok(s.updated_at); + await store.appendLog('w1', 'started'); + await store.appendLog('w1', 'done'); + const log = await store.getLog('w1'); + assert.equal(log.length, 2); + assert.match(log[1], /done/); +}); + +test('pagesForPaths maps changed files to affected pages (source_paths + citations)', async () => { + store.setWorker(mockWorker()); + await store.savePage('w1', 'a', '# A', { slug: 'a', title: 'A', category: 'c', source_paths: ['src/a.ts'] }); + await store.savePage('w1', 'b', '# B', { + slug: 'b', + title: 'B', + category: 'c', + citations: [{ path: 'src/b.ts', start_line: 1 }], + }); + await store.savePage('w1', 'c', '# C', { slug: 'c', title: 'C', category: 'c', source_paths: ['src/shared.ts'] }); + await store.savePage('w1', 'd', '# D', { slug: 'd', title: 'D', category: 'c', source_paths: ['src/shared.ts'] }); + assert.deepEqual((await store.pagesForPaths('w1', ['src/a.ts'])).sort(), ['a']); + assert.deepEqual((await store.pagesForPaths('w1', ['src/b.ts'])).sort(), ['b']); + assert.deepEqual((await store.pagesForPaths('w1', ['src/shared.ts'])).sort(), ['c', 'd']); + assert.deepEqual(await store.pagesForPaths('w1', ['nope.ts']), []); + assert.deepEqual(await store.pagesForPaths('w1', []), []); +}); + +test('computeContentHash is order-independent and content-sensitive', async () => { + store.setWorker(mockWorker()); + await store.savePage('w1', 'a', 'A body', { slug: 'a', title: 'A', category: 'c' }); + await store.savePage('w1', 'b', 'B body', { slug: 'b', title: 'B', category: 'c' }); + const h1 = await store.computeContentHash('w1'); + + store.setWorker(mockWorker()); // fresh store, pages written in the other order + await store.savePage('w1', 'b', 'B body', { slug: 'b', title: 'B', category: 'c' }); + await store.savePage('w1', 'a', 'A body', { slug: 'a', title: 'A', category: 'c' }); + assert.equal(await store.computeContentHash('w1'), h1); + + await store.savePage('w1', 'a', 'A body CHANGED', { slug: 'a', title: 'A', category: 'c' }); + assert.notEqual(await store.computeContentHash('w1'), h1); +}); + +test('listPages reads the side-index, never enumerates page bodies', async () => { + const base = mockWorker(); + let pageListCalls = 0; + const spy = { + async trigger(req) { + if (req.function_id === 'state::list' && String(req.payload?.scope || '').startsWith('openwiki:pages:')) + pageListCalls++; + return base.trigger(req); + }, + }; + store.setWorker(spy); + await store.savePage('w1', 'a', '# A body', { slug: 'a', title: 'A', category: 'c' }); + await store.savePage('w1', 'b', '# B body', { slug: 'b', title: 'B', category: 'c' }); + const pages = await store.listPages('w1'); + assert.equal(pages.length, 2); + assert.equal(pageListCalls, 0); +}); diff --git a/openwiki/tests/surfaces.test.mjs b/openwiki/tests/surfaces.test.mjs new file mode 100644 index 000000000..8747a18fa --- /dev/null +++ b/openwiki/tests/surfaces.test.mjs @@ -0,0 +1,54 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { heuristicAnswer, firstMeaningful, slugify } from '../src/lib/ask.mjs'; +import { escLabel, heuristicMermaid } from '../src/lib/diagram.mjs'; +import { buildAgentsBlock } from '../src/lib/agents_md.mjs'; + +test('firstMeaningful skips headings/metadata and returns the first paragraph', () => { + assert.equal(firstMeaningful('# Title\n\n_meta_\n\nThe body here.\n\nmore'), 'The body here.'); +}); + +test('slugify produces safe slugs', () => { + assert.equal(slugify('What is the auth flow?'), 'what-is-the-auth-flow'); + assert.equal(slugify(''), 'answer'); +}); + +test('heuristicAnswer stitches page excerpts', () => { + const a = heuristicAnswer('auth', [{ slug: 'overview', title: 'Overview', excerpt: 'It authenticates.' }]); + assert.match(a, /Overview/); + assert.match(a, /authenticates/); +}); + +test('heuristicAnswer handles no matches', () => { + assert.match(heuristicAnswer('x', []), /No wiki pages/); +}); + +test('escLabel neutralizes mermaid-breaking chars', () => { + assert.equal(escLabel('a "b" [c] {d}'), "a 'b' c d"); +}); + +test('heuristicMermaid builds a category -> pages flowchart', () => { + const meta = { repo_name: 'o/r', categories: [{ id: 'overview', title: 'Overview' }] }; + const pages = [ + { slug: 'p1', title: 'P1', category: 'overview' }, + { slug: 'p2', title: 'P2', category: 'overview' }, + ]; + const m = heuristicMermaid(meta, pages); + assert.match(m, /^flowchart TD/); + assert.match(m, /ROOT\["o\/r"\]/); + assert.match(m, /Overview/); + assert.match(m, /P1/); + assert.match(m, /P2/); +}); + +test('buildAgentsBlock lists pages, browse url, and ask hint', () => { + const b = buildAgentsBlock( + { id: 'w1', repo_name: 'o/r' }, + [{ slug: 'overview', title: 'Overview', category: 'overview' }], + 'http://x', + ); + assert.match(b, /## OpenWiki/); + assert.match(b, /Overview/); + assert.match(b, /openwiki::ask/); + assert.match(b, /http:\/\/x\/#\/wiki\/w1/); +});