diff --git a/README.md b/README.md index aa531238..3e4aa539 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,7 @@ Configuration is optional — middle ships with working defaults. To override, d ```toml [global] -dispatcher_port = 8822 # the hook server agents report to +dispatcher_port = 4120 # the hook server + dashboard agents report to max_concurrent = 4 # how many agents in the office at once default_adapter = "claude" worktree_root = "~/.middle/worktrees" @@ -122,7 +122,7 @@ mm stop # everybody go home One Bun process. No build step. The dispatcher is `bunqueue` + a hook receiver + a watchdog ticker, all writing to a local SQLite (WAL) and serving a read-only dashboard over HTTP+SSE. GitHub is the source of truth for the work; the dispatcher spawns `tmux` sessions for the agents and `git` worktrees to isolate them.

- middle dispatcher (single process) — bunqueue engine, hook receiver, and watchdog ticker write to SQLite (WAL); Bun.serve fronts HTTP + SSE on :8822 to a read-only dashboard; GitHub on the left is the source of truth; tmux sessions and git worktrees are spawned below. + middle dispatcher (single process) — bunqueue engine, hook receiver, and watchdog ticker write to SQLite (WAL); Bun.serve fronts HTTP + SSE on :4120 to a read-only dashboard; GitHub on the left is the source of truth; tmux sessions and git worktrees are spawned below.

--- @@ -139,6 +139,12 @@ If you want to see it: browse the open [Epics](https://github.com/thejustinwalsh ## Going deeper +- **[`docs/operator.md`](docs/operator.md)** — the operator how-to: every `mm` command, the daily run loop, `mm doctor`, backups, and resetting state. +- **[`docs/architecture.md`](docs/architecture.md)** — how the pieces fit: the daemon, the dispatch lifecycle, the crons, and why SQLite is operational state while GitHub is the system of record. +- **[`docs/adapters.md`](docs/adapters.md)** — the `AgentAdapter` interface every coding-agent CLI implements. +- **[`docs/bootstrap.md`](docs/bootstrap.md)** — what `mm init` stamps into a target repo, and how to remove it. +- **[`docs/skill-enforcement.md`](docs/skill-enforcement.md)** — the gates that hold a dispatched agent to the workflow. +- **[`docs/dogfooding.md`](docs/dogfooding.md)** — how middle builds itself. - **`planning/middle-management-build-spec.md`** — the authoritative design: architecture, the adapter interface, the dispatch lifecycle, the state-issue schema, and the full build sequence (phases 0–11). - **`CLAUDE.md`** — the working conventions every contributor and every dispatched agent follows (Conventional Commits, the Epic/PR workflow, the byte-identical state-issue round-trip invariant). diff --git a/docs/adapters.md b/docs/adapters.md new file mode 100644 index 00000000..b6360d41 --- /dev/null +++ b/docs/adapters.md @@ -0,0 +1,104 @@ +# Adapters + +An adapter is the interface every coding-agent CLI sits behind. middle dispatches every agent as an interactive `tmux` session — there is no headless mode — so the adapter abstracts the per-CLI launch command, the text to send into the session, how to enter auto mode, and how to read and classify the on-disk transcript. This is a reference for that contract. The source of truth is `packages/core/src/adapter.ts`. + +## The `AgentAdapter` interface + +```ts +export interface AgentAdapter { + readonly name: string; // 'claude' | 'codex' | ... + + /** Write hook config + any per-CLI setup into the worktree. */ + installHooks(opts: InstallHookOpts): Promise; + + /** Build the INTERACTIVE launch command. tmux runs this; it takes no prompt. */ + buildLaunchCommand(opts: LaunchOpts): { argv: string[]; env: Record }; + + /** The literal text to send-keys into the session to start or continue the agent. */ + buildPromptText(opts: BuildPromptOpts): string; + + /** Put the ready session into auto mode — a launch flag or post-ready keystrokes. */ + enterAutoMode(opts: { sessionName: string }): Promise; + + /** The normalized event that signals the CLI is ready for input. */ + readonly readyEvent: NormalizedEvent; + + /** Locate the on-disk session transcript from the ready/session hook payload. */ + resolveTranscriptPath(payload: HookPayload): string; + + /** Read activity, state, and context/token usage from the transcript. */ + readTranscriptState(transcriptPath: string): TranscriptState; + + /** Classify the agent's state at a Stop hook. */ + classifyStop(opts: { + payload: HookPayload; + transcriptPath: string; + sentinelPresent: boolean; + worktree: string; + }): StopClassification; + + /** Optional: detect a rate-limit message in a Stop-hook payload or transcript. */ + detectRateLimit?(opts: { payload: HookPayload; transcriptPath: string }): RateLimitDetection | null; +} +``` + +## Methods + +| Member | Contract | +|---|---| +| `name` | The adapter's identifier (`'claude'`), matched against `default_adapter` and the per-Epic adapter choice. | +| `installHooks` | Writes the hook config and any per-CLI setup into the worktree before launch. | +| `buildLaunchCommand` | Returns the `argv` + `env` for the **interactive** CLI. `tmux` runs it; it takes no prompt. | +| `buildPromptText` | Returns the literal text to `send-keys` into the session to start or continue the agent. | +| `enterAutoMode` | Puts a ready session into auto mode — a launch flag or post-ready keystrokes. | +| `readyEvent` | The normalized hook event that means the CLI is ready for input. | +| `resolveTranscriptPath` | Locates the on-disk transcript from the ready/session hook payload. | +| `readTranscriptState` | Reads activity, turn count, and token usage from the transcript. | +| `classifyStop` | Classifies a `Stop` hook into one of the `StopClassification` outcomes. | +| `detectRateLimit` | Optional. Detects a rate-limit message and returns when the limit resets. | + +## Prompt kinds + +`buildPromptText` takes a discriminated union on `kind`, so the `kind`/`epicNumber` coupling is enforced at compile time: + +```ts +export type BuildPromptOpts = + | { promptFile: string; kind: "initial" | "resume" | "answer"; epicNumber: number } + | { promptFile: string; kind: "recommender" | "docs"; epicNumber?: never }; +``` + +The dispatched-issue kinds (`initial`, `resume`, `answer`) carry an `epicNumber`. The repo-level kinds (`recommender`, `docs`) run against no Epic and must omit it — the union makes `kind: "initial"` without an Epic a compile error rather than a malformed `implement #undefined` prompt. + +## Stop classification + +`classifyStop` resolves the agent's state at each turn boundary. `worktree` is the workstream root where `.middle/` lives — sentinel files resolve from there, never from `payload.cwd`, which may be a subdirectory the agent has `cd`'d into. + +```ts +export type StopClassification = + | { kind: "done" } // agent marked the PR ready + | { kind: "asked-question"; sentinelPath: string; sentinel: BlockedSentinel | null } + | { kind: "rate-limited"; resetAt: string /* ISO */ } + | { kind: "bare-stop" } // stopped, no sentinel, not done + | { kind: "failed"; reason: string }; +``` + +The `asked-question` sentinel is `.middle/blocked.json`, parsed tolerantly — a missing or malformed file yields `null` rather than failing the Stop: + +```ts +export type BlockedSentinel = { + question: string; + context?: string; + kind?: "question" | "complexity"; // "complexity" marks a complexity pause; absent = plain question +}; +``` + +See [skill-enforcement.md](skill-enforcement.md) for how the dispatcher acts on each classification. + +## Adapter selection + +The dispatcher resolves an adapter by name through a registry in `packages/dispatcher/src/main.ts`. A repo's default comes from `global.default_adapter`; an Epic can be dispatched against a specific adapter. + +## Shipped adapters + +- **`@middle/adapter-claude`** — shipped. It implements the full interface for Claude Code: `installHooks` writes `.claude/settings.json`, `readyEvent` is the `SessionStart` hook, `classifyStop` reads the `.middle/` sentinels, and `detectRateLimit` matches Claude's usage-limit message. +- **`@middle/adapter-codex`** — a stub on the roadmap. Its bootstrap hook config is written by `mm init`, but the `AgentAdapter` implementation is not yet complete; the dispatcher's registry currently accepts only `claude`. diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 00000000..fe176530 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,58 @@ +# Architecture + +middle is one long-running daemon that dispatches coding agents at GitHub Epics, plus a CLI to drive it. This document explains how the pieces fit and why the design is shaped the way it is. For how to operate it, see [operator.md](operator.md); for the agent-facing contract, see [adapters.md](adapters.md). + +## The packages + +middle is a Bun monorepo. Each package owns one concern. + +- **`@middle/dispatcher`** — the long-running daemon. It owns the workflow engine, the SQLite store, the hook receiver, the SSE control feed, and every cron. `mm start` spawns it. The runnable entry is `packages/dispatcher/src/main.ts`. +- **`@middle/cli`** — the `mm` binary. Each subcommand delegates to a `run*` function and exits with its return code. +- **`@middle/core`** — shared types: the `AgentAdapter` interface, the config loader, the normalized hook-event taxonomy. No process-level side effects. +- **`@middle/state-issue`** — parse, render, and validate the GitHub **state issue**. The dispatcher edits it one section at a time, so the parser and renderer guarantee a byte-identical round-trip (see the root `CLAUDE.md`). +- **`@middle/adapter-claude`** — the shipped adapter; it launches and drives Claude Code. `@middle/adapter-codex` is a stub on the roadmap. +- **`@middle/dashboard`** — a React SPA plus `Bun.serve` route handlers, mounted on the dispatcher's port. + +## One daemon, one port + +The dispatcher composes the hook receiver and the dashboard onto a single port (default `4120`). `main.ts` starts the hook server with the dashboard's routes merged in, so an operator runs one process and visits one URL. + +The server binds to `127.0.0.1` only. The hook receiver has no cryptographic auth and uses predictable session names, so a `0.0.0.0` bind would let any host on the network hijack a running workflow. Localhost-only is the security boundary. + +## The dispatch lifecycle + +Every dispatch is **launch → drive → observe**. There is no headless mode and no exit code to read — the agent runs as an interactive process that does not exit between turns. + +1. **Launch.** `mm dispatch` (or the auto-dispatch loop) POSTs to `/control/dispatch`. The dispatcher checks slot limits and in-flight collisions, then starts an `implementation` workflow. The workflow creates a worktree and launches the agent in a detached `tmux` session running the interactive CLI with no prompt. State: `launching`. +2. **Drive.** The agent's `SessionStart` hook fires; its payload yields the session id and the transcript path, which the dispatcher records on the workflow row. The adapter answers the CLI's boot dialogs (`enterAutoMode`), then sends the dispatch prompt with `send-keys`. State: `running`. +3. **Observe.** The agent works. Each `Stop` hook is a turn boundary; the dispatcher classifies it (`classifyStop`) against the transcript, the `.middle/blocked.json` sentinel, and PR state — done, asked-a-question, rate-limited, or a bare stop. + +### The transcript is the state channel + +An interactive `tmux` session gives no captured stdout. The CLI's on-disk JSONL **transcript** replaces it: the adapter reads activity, turn boundaries, tool use, and token usage from the transcript file. Hooks are the fast-path notification; the transcript is the source of truth. A reconciler cron corrects any drift between what the hooks reported and what the transcript shows. The transcript is retained after the `tmux` session ends so `--resume` stays available. + +## The crons + +The daemon runs several recurring passes, each a bunqueue cron: + +- **Watchdog** (every 30s) — the liveness safety net. It reconciles `launching`/`running` workflows: launch timeout, `tmux` liveness, activity freshness, and re-arming a `waitFor` signal when a blocked sentinel appeared after the workflow advanced. It acts on staleness only and never overrides an in-progress hook decision. Freshness checks are skipped while a session is human-controlled. +- **Poller** (every 60s) — for each parked workflow with an armed `waitFor`, it fires the resume signal when the unblocking event appears (a human reply, or a PR review verdict), and finalizes parked workflows whose Epic PR has merged or closed. +- **Recommender** (60s due-check) — runs the recommender for each managed repo whose configured interval has elapsed, ranking the backlog into the state issue. +- **Retention** (daily) — prunes old events and archives old completed workflows (see [operator.md](operator.md#retention)). +- **Epic-cache refresh** (every 60s) — refreshes the Epic browse cache the dashboard reads. + +## SQLite is operational state; GitHub is the system of record + +middle keeps two kinds of state in two places, deliberately. + +**GitHub holds the work.** Epics, sub-issues, PRs, and the state issue live on GitHub. They survive a database reset because they were never middle's to lose. + +**SQLite holds the bookkeeping.** `~/.middle/db.sqlite3` (WAL mode) tracks workflow rows, the event log, rate-limit state, the managed-repo registry, and the Epic cache. It is operational state — losing it loses in-flight tracking, not work. That is why `scripts/reset-db.sh` is safe by design and retention can prune freely: the durable record is always GitHub. + +The schema is a sequence of numbered migrations under `packages/dispatcher/src/db/migrations/`, applied on daemon start. The core tables are `workflows`, `events`, `rate_limit_state`, `repo_config`, `waitfor_signals`, and `retention_runs`. + +## Workflow states + +A workflow row moves through a fixed set of states, enforced by a `CHECK` constraint: + +`pending` → `launching` → `running`, then to a terminal state (`completed`, `compensated`, `failed`, `cancelled`) — or sideways into `waiting-human` (parked on a question or review) or `rate-limited` (the adapter hit a usage limit). The poller and watchdog move parked workflows back to `running` when the blocking condition clears. diff --git a/docs/bootstrap.md b/docs/bootstrap.md new file mode 100644 index 00000000..b06fee86 --- /dev/null +++ b/docs/bootstrap.md @@ -0,0 +1,72 @@ +# Bootstrap + +`mm init ` stamps middle into a target repo: the skills a dispatched agent runs, the hook config that reports its activity, the per-repo config, and the dispatch state issue. This is a reference for what gets written and where. `mm uninit` reverses it. + +## What `mm init` writes + +`mm init` takes a path to a local checkout. Run it with `--dry-run` first to print the planned actions without touching anything. + +```bash +mm init --dry-run # print the plan +mm init # apply it +``` + +Before writing, `mm init` validates the target: it must be a git repo with a clean worktree and an `origin` remote, and `gh` must be authenticated. + +It then performs these actions: + +1. **Stage skills** into `.claude/skills/` and `.codex/skills/` — copied from middle's canonical `packages/skills/`. These are the skills a dispatched agent invokes (`implementing-github-issues`, `recommending-github-issues`, `creating-github-issues`, `documenting-the-repo`). +2. **Stage the hook script** to `.middle/hooks/hook.sh` — the universal POST script that reports agent activity to the dispatcher. +3. **Write hook config** — `.claude/settings.json` (Claude Code hook entries) and a sentinel-delimited block appended to `.codex/config.toml`. +4. **Resolve the state issue** — trust the number in an existing local config, otherwise find or create the dispatch state issue on GitHub and label it. +5. **Write `.middle/config.toml`** — the per-repo config, merged from global defaults plus the resolved state-issue number and bootstrap version. +6. **Add `.middle/` to `.gitignore`** — middle's operational directory is local, never committed. + +## What lands in the target repo + +| Path | Purpose | Committed? | +|---|---|---| +| `.middle/config.toml` | Per-repo config (limits, recommender, state-issue number) | No (`.gitignore`) | +| `.middle/hooks/hook.sh` | Universal hook POST script | No | +| `.claude/settings.json` | Claude Code hook entries | Per the repo's own policy | +| `.claude/skills/` | Stamped skill copies | Per the repo's own policy | +| `.codex/config.toml` | Codex hook block (sentinel-delimited) | Per the repo's own policy | +| `.codex/skills/` | Stamped skill copies | Per the repo's own policy | + +`.middle/prompt.md` — the dispatch brief — is written per dispatch by the workflow, not by `mm init`. + +## The hook script + +`.middle/hooks/hook.sh` is a small POSIX script that POSTs each hook payload to the dispatcher and never blocks the agent: + +```sh +#!/bin/sh +EVENT="$1" +curl -sS -X POST "${MIDDLE_DISPATCHER_URL}/hooks/${EVENT}" \ + -H "X-Middle-Session: ${MIDDLE_SESSION}" \ + -H "X-Middle-Token: ${MIDDLE_SESSION_TOKEN}" \ + -H "X-Middle-Epic: ${MIDDLE_EPIC}" \ + -H "Content-Type: application/json" \ + --data-binary @- --max-time 3 || true +exit 0 +``` + +It reads the hook payload on stdin, posts it to `${MIDDLE_DISPATCHER_URL}/hooks/`, and exits 0 regardless — a dispatcher that is slow or down (the `--max-time 3` cap, the `|| true`) never stalls the agent. The session token authenticates the post. + +## The two-copy skills invariant + +Skill text exists in two places: + +- `packages/skills//` — the **canonical** source. +- `packages/cli/src/bootstrap-assets/skills//` — the **mirror** `mm init` stamps from. + +The two must stay byte-identical. `bun run sync-skills` regenerates the mirror; a pre-commit hook (`scripts/hooks/pre-commit`) runs `sync-skills --check` and fails the commit on drift. `mm doctor` surfaces the same drift as a `skills` warning. Edit the canonical copy, then re-sync — never edit the mirror directly. + +## Removing middle + +`mm uninit ` reverses `mm init`: it strips middle's hook entries from `.claude/settings.json` (preserving any other hooks), removes the sentinel-delimited block from `.codex/config.toml`, deletes the `.middle/` directory, and removes the `.gitignore` entry. + +```bash +mm uninit --dry-run # print the plan +mm uninit # apply it +``` diff --git a/docs/diagrams/architecture.excalidraw b/docs/diagrams/architecture.excalidraw index a6c78936..2ad2bc56 100644 --- a/docs/diagrams/architecture.excalidraw +++ b/docs/diagrams/architecture.excalidraw @@ -106,7 +106,7 @@ }, { "type": "text", "id": "serve-txt", "x": 467, "y": 422, "width": 190, "height": 36, - "text": "Bun.serve\nHTTP + SSE · :8822", "originalText": "Bun.serve\nHTTP + SSE · :8822", + "text": "Bun.serve\nHTTP + SSE · :4120", "originalText": "Bun.serve\nHTTP + SSE · :4120", "fontSize": 14, "fontFamily": 3, "textAlign": "center", "verticalAlign": "middle", "strokeColor": "#212529", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "angle": 0, diff --git a/docs/diagrams/architecture.svg b/docs/diagrams/architecture.svg index 19b55d20..79b7d5ee 100644 --- a/docs/diagrams/architecture.svg +++ b/docs/diagrams/architecture.svg @@ -1,2 +1,2 @@ middle dispatcher — single processbunqueueenginehookreceiverwatchdogtickerSQLitebun:sqlite · WALBun.serveHTTP + SSE · :8822GitHubsource of truthread-onlydashboardtmux sessions(agents)git worktrees \ No newline at end of file + @font-face { font-family: 'JetBrains Mono'; src: url(data:font/woff2;base64,d09GMgABAAAAACNAABEAAAAARwQAACLhAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGhwbHhyXaAZgAIFICIIWCZ8DERAKwny4EwuBQgABNgIkA4FCBCAFgxwHIAyFMxuwPyXsJnhwHoBIyT8UiKJUr5IoSvZoO/v/r8nJGAIWbKXmr53I6qeoqIN2UQV5Pd3o5SCOKEvvcJPv+Kp1wUdmi0ZsnsLDj5poOh4LfZwbHWRyKJYeMPXJpT/e2Dj8jkQZEaYWTR4NvOXvL231LSqsQllGr/GpjtHQSGIOkZv9zxaKgIiApYKINUYRe4uoERURS4i1EIwJomeJJc1U0wq2GkvFElNeYpqXs31/juS80r3uJf3wwzX7CWzC3U2BDohVq+TZeXKE/dKyMZ+Hvr3v7szO7L6fBDCGg+zC0BUI35RBYwwKW42PRIPK8bG/8fD/p96MKRTSEqc4lmnQewpe4Kz+9i2uYWmbpnApH9CYnJk0exbOE/ype31Pks0hdHKnnCuVRwJdHCYXDQUCF0jFscuoX6B9sYJ611Gk9MD8B+c4nJonZd9JPwF5wgpYAfLKXnRZxE63R0vAA9KJywqLbDkD5WdU+Z+60r7joC/VzijwNb/UXmpCL4YW5vLlC0iLk6LjEn+65inoTYcBmB89QEmTk/WTLt/akbTrr4Q3gFB0+b8KUFWnmnlvZGlmJO9YJsn71xiw1gHt2j8LIcuhDQBWSUdwtJ/XAaDqEDvA8q67a9Ne0Vzf5so7nDZ7lzS/j+F+OHNlLEXdLyUyWkbCnhphwFHXOtBsifE4rPyfY/fJVSxzpOJp5kgtLWbvk+ELFvHqq+3AZkJ4fWgamx2MVapb24x25KMfNR9AACAZCbg6OMQCxNaA9gaAwPidQK/fRgoAHgoAoFYjACL7ijM6IG/VfSyXFgL3U9r+YkBnDXr1QsAEBKuVxmDIR81Z8DQOVIxcULcQPn4s3A8GR//YkmNxSRHI/48HCvy+oVMhSLmUFUmB8CrtPS/gP/OKswHv7w9oEAwiUb6HUjGwoi2+lWDFGpGzMIH4woj72TQBw9BApEkbnS59xpjYLFnbYScHXM5cufPkzVeAYKF24QEABgAkse+8oLrnu7u7sZ2v4ziWWx3A327r2+Im2niBmbFN3xKKLuv3tbnm19DyX1zD5sFocCrm2b2MiYwF333MTW/0fA+1uPk7lnbV1pqMAeiPUuzGqrFkJS5+uaI1klGA3IyN0TgKh6Ei4dhSmf3zbLo3Vp/5mZLe0l6i0OsYiUtRFZKbJD0EESpv1FiQwbIxEAAgeDAShA3gaRcI16zJCBYAMBbCxirBA6FSiV4cyii0Iq6EE92Dzocw4z/YFUS7odUSlhxxJ3d2wEm58wZ05dSPYU89C3/gIBga8BijcKF2D1nGt/T5tZlsnM2lsweh963dAbZQhgFlCCtKefh2vQwIa9FrXJhgRSg27rKD3UNzHObHXosqnLMFb+PgnEKFKzR7C2GWlRYLpLToLKRX+bBPUYG4mMjYBCEvVMcQlApcquDtwPgZF2A14kH0tFgsgEOPnHigedDbcBD+tblM/U3kV+vnGkYcT3uBz3AOI+BSLOAhfduFz2ngSHNKpQ9k00xVPpC5N3e7DFsCfTl98Qsa75Efzto0pbHFvH50J7ql1q+pLzGPwfydFixRbjK3wR5kCJ3UCu6StAdzrFJM7vS2GPwQeZ46BMdU9FXu2xyApVYjKXd0mcNA/UevVq+p/izaW5JVGLikJNzubSHyyxMDZWCwTcWTirNtAzTdNPUpdsmayzW5Gx1DlT+3SloCE02VPrN4pf6F0HMA5i8MsUju2Ch51aQaDakFmp9D4+liO5KPAWBBwGOqvgF5CiKze2wILd1VHaISYMNpAKCyeTZx6nxxkhH3lKarLZXlwpiwyL74k80Y8Q1yLZ5qajoKo1p2H8QUHYThXp4rNCjaI4NanNZqmfIwqgiwkRL40F4aW2dwhjrXT+Ner8g9mT/TFBjWKV2pS0kp6lzT7vYXqDLd/EXFd67vaantWI91iuec+p4xyEOCNzZveAtBqcdpPCaPCa1OVEalOqDImO7TXYbAV4A1cxwIVzBoh72wp7AukR68ZbT7A4VQSSi7vBTj1vPVEvjlegWu3wUXFgbLXPvwmPV4M3mAMREOP83F9Xbq+oheIozdCeSjLyCHDD5+W5xDwMf+x2tUSSqb0G7ObIPBvBngMBIXrrwS/htGQsWKk0yMxI+/ADEEhBKIJEmxhxjCSjgxnGAhduGJl0gMI1CQUGHEUPY4uJyhdnLgyAnMmvhLs0yHa+wpy0aX2FsHuEafYjfH3o43tIbiNAyAeYbPAHBlcjD4GjevnLzEasDlY0O/AIjRlxV34AWABcyh0yk8MQH8n9j8AKBv/zIA0Pu6CQBMGGm8FnSvICmgxmJrTBF4zrWxgz1uo1sFXq2KfEBePkGeQuUmB3qtRoDZSHdMRoARVuCL2z9NgaIPeoPGDFC/i+34GAvneXOUpkwa169Pw0M/MXx0klgu7MOlUqFXTbRL9h6Aemvq5By/h/hqFViyaN5tveoWWxDCZLx3so4/pwjEI/ExkpRZML3SfMy6KVc8k4PfN66bqrrdXFZpFMkdT/wQ/miLVgyZNE/upA7DZjzwuXUKYzGxmpS0GDAW3/ueMn3JtLueIt50rB8/tmR9qVeGTLntvBsWPEpPuo27sxDLKLTpbZ51P6t0U5eoyQy8NfmdKlhoTHfr0rAl3kyfKAX+vMuXLl443NhQX3eotqa66rPKCnm57OCBsv2l0n2SkuKiwoL8vNyc7KzMjPS01L3iPSnJSYmiBGG8IC6GH707KjKCF74rLNSaAQcUR7wdeSPwE749P7KzmTUwmHtHyzm5sRM4Wz4M5h1BDcT2h1LeMdZ6wxYKTJR6ZpxpudXzRpElhLDvvrb+iPuDSqjUDGyjbbzixooXws4sAkkkxpiThULGM6UxkYExiVgtE2BB9ekEVISmlUk4XdpKPqeYd55ROLYsc1tTciEcGofb3Z5wdTlvgyRUt0Kh55SPuEY0VQKeLXzHOA1E4L4O80Y17Ipf3N5RAC+JYAuePBQdMpixeSwMhXVuCNrXkvd1fWcA8wtiu99MBIZTjO8Wk8rfLVeyagXg71bkC0GShFs+baJpINxSJgEEdybfO5vOp4emO+HNSumsSrEcoul+jVDscW2EHPFbuZiSkQbJsT6FabCcm6vyWwzXbJ3zCwgCzPkuTOCUF7txxFI7alcdI00kSpgXX2LH/wjgIfwQ2IQYqazhxF+sGrbErBAgOCujguVIBZtNCSyk4qUP3um2VeUvaFbyNYQmjDImylu759hvlonMOfZpChGIQAhOBW/4nlBBgXLhSPAmg7E+a3+nDALrTZLw3LBPnZ16vUytrbf7mgyqMMPt/JRvyeDU38DLIi4uhSZQ0Z1yK/CTDCay6G6j6QFFrzCyqUJFS8cCQkaoofv+nqlIyER+ODvcQGKHxo/ntxDEtyq3O5IdMW3irAR2G/y7UXJJjeda8oQI6qEszcJH8BKKI1h8yyQGw9b6Rr6zSWWCOXdnaO4PknrgHh7KjuBepHqEKhsyTuNklTvWsNXyB8cjdzy2VwZtlEUE0S490hKMLDnKkWZQAzBBgGWewqrCT48r0fA0P/VIlukbmtM0Gpj1RtOpsSfu7DfzTLK2zYHJQZLEeyymi2Z+zaEIPMndPXg2v9olEAtDBobc44UnCxOd4nImQuLFdCGaDMXSzPtLCAtDxmVscUycJlBBeUxbDzzkkUV2vBs159Cy87KV7B3AxHlEoSSql7C4aHDZu/o1RVQFKW/lS5zik0abtIU1jFZl7JmkHXZsY/Q2SDIx1oPJVtBgo5GtXSF3xVVHj3YtwBWxIKIrKVQLFHwWgSCL2presEZpET0L+lz58FVYV6iE4qjLZSGk8TMJZ0VU0ZztCorKmxScAnyX6gGWBRBpHWuD6U60lE0ES8KW3Q7mLdcdxzl4JsrjDsiYxlfDAKMZU7L1KKvAjKC1qZpAkU6UGueWj34iOFhiT9s136cbv+BEkQGT04Pg2FJBLbKzjmYFGpBdoP1lx1vZnkBSPFxC4opLJ7ZFgYPBa66lchmenwBLgoV65YyIVXXDgfRcYKDjES5VB8WsFILW6W4oISq0hnCaQFZgwNay+1xgCoTlbOl3pu1V1Qocjso+WCHxiKuvOkdfceqk05VednADiTk+of4N6WEEfAKqNK0g3I0yMV/z90KQnoymoCInKVPXCwvOGAYGofe8pcsFsZGu1AnqCpbX6OoBK92DGciu/Nky5UPpKIXVvDtXsbTuGm5qVS+A0XDizKGyTMWHiHRijpwh5/0ZQbA1eMdE02OJpoMvGk2HYi3b7QgxyPxIWHZavpb/waB+b2Bg48UoH+NsDBZqgW4wEE/k6UTGDaZ+kKV7aeEvBNCNL8yIV5sIOga7e6JUcTzUJefBMHuXrn4ogL3RFRw0QTzDW7PEXceAFdmMNtLpObbEPRfAkEFtL2Ike7HPMaJ0QY/iAgGxiQ0ZgGveIbgdtMuJuCo4SA/gQHQb5M7KX7dOqQd29NeYih4grDA5Vd2vjaYTJ6IzDuUeVX0KvrgIB0BBNEvlsQMs8FVuAWfhWDoRBDbrF0dRUdjUb79mS8MhtoowYD61qtVURkyuAYCwhQOvcwgxMAEKn2QrEeEuW4BHEfMkhEN2wHdpEw/gIjVFpgTv8kVmBg86YXipsWxEmyl8SQWouGTKSXnUOGut+9ktHGlahIypDoZ+QLMrAJV9srxbSDX3zcATt99U7TZVXu1OhTo+dhxcbDpBifNWqPgJguSri4gW6Vm/5jws8fb/XFoGyL42z98M6HysAXfII/7wNUeWl+Rs/vzisDCWt2Ck+L/J3sDRljYR5n9DQRjeQ9fcp1z51rHixXtBdlROzyNYksdhFm4xz8nIjtjDxapzRMYr/w7jNVLYH34xXseoEi4oP5gd/CYU8E+MlBCYEKT1ogdmdf3zwFAFo1C0k6qxcl7s4bSf0w7M8Vq3TrXiIm3LsCUGG9ltUGscULzlGNHTIzD5Y8w7ksSR5H67utXfttdJIm0BxjjFL6weEqzxS4scjcH5Y7G2avy0V82fMv4RNXggGZTbuAM+iwxXBR6J2vmehL+nGsxrR6hHMx8S1FNBpHju+7q8ZZMBIcBCKDnDA8o+3XAF5qLZxuGYOpbyJenM7BkaE65cBbgnZKZAJcxL6wUIQgMBl9b/sCUAUjNj4SMWANj3gMZ3AHoIANwH8P8BAENA9rSArfIwXNwMATY61TjN7BEuwB5Bzjtc9yFTxEBMnEikkAFAdHOIJVELV4G9/aIBAG8oxWmQjBlL1NC2bEvZQi2SYfV1jc7q05hAq7wjJFE91etf9fa4FKY+M87qkUi6a0iizy6eRCKhJH02Rmplu4SE2iBu7+Fd2tgT8PSepTdAoluTtWUnQUpusSYc1ZCtuSVMwUsMSI8vgS5yCjPm64bgckrF2WsX/DJJll8L717VzaahNQZ7eM+/oTdKMJn/nZCi/GWBtli4yM9R+RLadcr4DN6TQK4wT/P1bKMceyh8uPyWf9+BHZExi1e2iLosA8tOU4qZb1la++GMeEQJeKNn0h1hJyuC3RJlcv0AqWT+Rri46QatPbP1dOGbt81zCtyEjoVJYoVZxtg/jREoJ0gzc+C7J6GfbvM/qUSMOWsZveReCvlBedo2k2QZHGPriUKTCrQ16M3MdhSzvScMQADV5zuAdxdTbM+9dTQufS0Lgkn2QRZJg3crBwkiX7X6qnhoKCLLyKaiH+m4Ky4j2GQdeSz4GgOeV51DJ4UbxnTysihB5RX3S2SZTXtjKmoJIl86URLCsicLoFmqkEmaDOn+eGQoYsnHgDdlBfHuookG95Z20BhhVbXjVX8fBtrCvIo492SY83W3K2D/rI4iYrSpMFmlEw8faUVcwllpdwP9gf9v196jmF1AZ6e2gWbh1Z8j+4jgTgGHlwqUf2RDUHT2oHS9YlEP2SDNZ1St02SKI6c5nSSJYVj2ZS32pi0crCOjpECO3JYA/z2KBfrI3r1KMjB44rtnItOVdUEtbIgEGV46kJumNuWjgEhKgBjGdxYfggFYOaP3CWKPrwsWqXatWUYx9546cAlLNGizWMnEcLtL1t68jSr2SING/y8ujSslKJhBiMJa2K4M1VP7Uq2sNy68PWrs0sZ5pBdlC0al/Udxd0kAmq8N6XwG68Jpb0KxjbCs5rVpF3mv+9oXRIvi50ieaLNbsPU11z0Zsn0lQvTlWxB1Gz9Ynl6Jci0GLzXhzLg+tm8LR+q+5wHbTrYVsMzJTsjKic/MEvQfCSqFpFA5qJBDFUAC9kkIFURsBZBnIBSYkHAyXxsguS/HL2sDyoVZAxkLUyhWH/1scHnyoOPR2lq8vHgnokxWtZ02oIiKbcYV15ROFsjU9OUYUdSztbYGgmiAsvqGVktBUgsJHePEExpelXRsjC3seqjlN3jsQXaSSJicnpCWRBRlxKcm7jToaGLBxHJK4tt4EB108/bDs58nQwEwTZ0bvyoHDg1daaWQvoC/n+5SfvVkiXB/qb9WWX8mU5xyMc3Kl/KF6VwjK5JclJtjfy/657/r5sxzer+bZ3xJq1Qlh1fanv8qxVlwrnln0LFw/uOvdNVtniskhvFZ85s3ri3bmzcVVBbz8fmLp0omPtNIMa4sfTevTR5/vbQihu+//90Qg7yY0mxfeiF6cWPg4rFzAy1GnmZPR6bwbYEN35FI3zUN+ODhrWza6NUST307T087fYYl7e/RKINHRfZPOWn0B9W1nv+WnA64Mn1M+ceTLxufxYnFx09Ehx3zjyirSU8LCReGmX2edj28bfrsIcvtpfZ5ynz7Em2MgGlaxtVv0fFO9Yt4Wj29Fsv7xAXbLCbB/5HGn9NuLdxo7t4e/VqWqO+Rfy79LPIJS5tZg7HIfy+WlaM6vXEaGj/rEMr7yvN69siDfPLc/auKUQ9w45pjN8CoVmV08sq/qz8awEQHtm7SAKE8ZfBeCmR/o+/B+ECVsozM6KFhjI2JqC4DuaqS2ue2yu12HY+Oa24d6xofn+/mX9PkEfWmhdqTRAMM7ZFKF0N8yX78yXCFLlsFBdWchvaAvkbSxZQIo9zXpDHAIZdV/c98jqZlaUxE3zG++79fkLfh4qdG9feOKAKryQEkg+k92lc1NMzPVpy5TDpheOrwJbOzjbsHd1+wfC3No+lM/rgXs9NYrBgZHGlkjL/UmInTUuBOt7dlUzm0y0T/fnqU2wsPjCCiAYHYphIb2HT4fWmvYLsnnyhHVkeyMgqT3nMERjbspz5CoY+vMB7eVHy8r0+80Bx9OnjugWjI7kDooJmC5Ke3ybO+Jyn1mW6vitNw95obwG63hlJiC8+a/UqbanuouuE+SAoUMPJSHJPjaiz0pvk652g6ZAl7VrOOQrzYGySKIEdoR4kSBcUZZfW+q7bFsbmn8WO+SYLs+2Pt6GYvUG7rqbR1akYOsXyzCTTVPwws7h9T1cu/TXHYvxmqubKqmX7L0D5SDvTPNTpdT0DhT8KvIMOVT09UbCJGpftIdiGFkESXUL7nkzlgjPQ6d6LqfxafaMZhJXe7KqFb53+SBX9wa/H+oMRcS1/bnNrePLO8vD84K+pw/Q9rhuigBYYm69UM1dK7naul0NrVcto3hGMf+Y/PNauzZbJBwlWbsw00nUk+rgLjNXLTO8Q+wDqIcs+h0+pKQ03fuysszib9r/WHqh8IgwT/Vf9vBr9RVcn3neCWEMLwetMhxHqiyTta18SD8dnV2eScdPH+srMDnquZZ7z4CyKv0fQLzpSaPddH2d9QzSVU7TrJN3qZbJ15XD17EYPUU3Er9XchRcW01S9roAmKu8cjuUZGvpwo1/vzk7Tq/Ge1VjXtT7jkbYnhS0NRq3RMn5MWxaleRTN/4DV37sfHpSe0Rmkacx20KnJ7hfshRtrCkJ0exiRnKEftvaN16YvezbsAzW8yYkq+ev+VFEAXdrUWub66lb0RwqGN4n8DSpQBGn/Da57p495FLR0w4NL170nN36NdLekizfnrJRb2O9XXRddoI6kIQSQ64du1TayjwNZGkP8tZLKg28SiXbFY/bQlMeIwOfsyU3KPSqqzQrFw0SA8hRDOtIBR0H/yyLt8kuQtBpUaHU/ydjosSvBjc7vFkTwLizr4yszkxUszE1euKicuXVROXjkfFp8VJwAR1tuCuN8Onf7NXePBx5e3/465tcc36uiR4PRPB48v9G9QmnQnTDOVJgm0Hz+u3zvVPnnJVKvB5FTLZ30/za6s/jjbV3mypd5U6yKjbeLUvY/r5NL04oPZybkHS4qLZCW5ybLswp7apgqJrKmhttZpmWT+rrB1MljXrXMq92TzoncGHB1qYEWXNRUmXvvnkCa5Sc+i1ODQ6VRRUF5MKkFESI3wCT8cLa9wOtzHsbVHoHSQSKLHLkH+fkGBfv4BhgJBjCAu351ZAi2eH45kUUfJ3x3yLRv9UXX+fQj6S8/B/dKeL8u/kv95mPqgLacl82hERmyKeHyPX9TRkyHp8MGL99aHWl6+PFkdWxZLcIuW/wXO4vnYThsNz+PmlboYkv1qPIS2FoNaOpd0Hj3M7eOLRaCiOyc0a+PWeNarUB2rHGWqknSwnWEqSwEYB5SUkTUveO7ujJodFHoqWlyUkUodsSKYPaFiH80Is/8kSHM+8sx8NfsF1ShrmvN8MBkXmzauvgYvin4z13dOea5noaft0meQkaP8SUl6O1Mtyyx/7lQtF5dSh7cOMrHU+MyaakaDmltVXl8fe2wRsLl+Yl6rviVoWL2ZmcsiKL1dVCQrgDIdWidors+7b4Gz970BinlXbh41ElVzCbW5SOWu/frW7KhFjH+uk0fA65gaMBuzABajU0x7terKuZiSW7++fFrqz5aQ5/EVOwK1HO3E/mAW/jR2zO/K7VfiOvzNIlIlGEbPcj3lwpZFf7kHlzsi3PEXsy/w/lihAdjZowowq4Dl1rgljyBpbCmTgQvG3Jh/op/G5kRqc7K9b+0Ef6q642RUQ+i/ZyROXvEZdLay1KGhS3DZ0d/yUFqom5tjQHiSWc4CbdQ5jUazrRWLbCto2hSENtrGNyo8tXFr/NSrQpNhh4dfuz87d7j1eEJSPoc/iWg2yNtItKMTXS79EbGXloAFoAU8XMikMad3lMxb2Eks2E3gxAn4eOpN58XXJpa1hnNhARfxNpAoG1glXxUAiMHYPxpaOescf7O/JHDFybvmXfOuedeVbgt62uSbOYXlMq3uyT35P1iuKz2/fxLvTNdYECkkH1mOTqFNlmuZn04+5jyHvtu4JPI/AgTU5LjN134KNh8oAPRdNxHFVllrlXKzpRu2h8//T0VSVkTwebsoSDmwFctuhT0MGvpm3E+u3Kpfh8w8tyIH668itjI16D+3eztSONy83KmzfyweKo/0hYNqmeKN5wJIYpoO/uaLhXp62KAoWLh1+L03LIWAIpe+0Of33warrIsgIqUFIQjcPd+0r4ebNv56fNr2No7XqJVUA1uCDVP2EOOz/IKpqgO0riy7vAPiztMc00SxK7DavJFiOe/56YzWGFWCUuSUFksI5blWcmBo4oxt/uE/VVGa/tkr3L1uU4gIgVRp4VB8K83MYJREgIqsCjoVmbIH1L23eWhL2NOUD/ZHO4GmzQiQXNF48XVkhglPRnuXLuQ3nNBXUzqXlBZWVGzUzL3JgzabYhySHNMJLDkaVUQyWGmGjAXrsL20R0pvoPIEwFkSrYDkRcCRN6BWEu8qHCVA09cZDDSacEq0eMGw54wwRWKN+3CzeXXtClfDTG6Op/MJTYPbVy9ymEDlgXyFEQLecOqlrsPSOznKR6i5sOrQ78L4+kwBSDL0SnhmKq6XvNRBgoBgAvpcc/kuGYWkcQXXwXh0Pfwnq9SYTXN7GNy2J7iedUmmDgQCLDu4/7DjotQ//ZP6Ncxm+24XPlSjwNkkdVlN+9tLDRmgrJzLE3CxoaiqJxc0bMehLJB4I3rcIbkJxM5DC493XkeBI/JuuciZvKtIqcF4LxqhSQZQAE0wwDda19Gf1tHRYdsLNF2jNzUx+Q696zVKcB08BSYSKDXcWWFLWp7wEtA/kiGiKYC1iUdy7vs7XHl0p7Kx1aQtVvqsETCa34Q1Rq+8tdHp2OdqHDXAlQ+bxCiK6utlfwAPF0dHe3ur1exea+dSGl2cnBwc7OwsFpPJaGaGq+bkHqwNoZQh0eal7TltvF06BB29s3G6JrSjNdMRLDsOjFAcAghdWcOEyjwvKrCm7KqDD51UEU0mF59duHqaTrxd32KFegjFOXAJCIYf+LqZrj+IRtCFO41OrVj1TvpjwvkxV1qeYp+Zd3AD14T8FQOyFVCpg47BTQiWmQb2htEDqRGRZ+906k03tv0+8XyIuUmXAjg1i9Lotx5n6XVuNPZmen+QUenAPB+DRlHL3JPSVHpoLzGAGQhrKD1MYeNc8QtoYFDwwB6xPEq+nyKIt9o/1BZCP3CNNQURY0chYIioc2yExkeneNQW32Djw9khpjOeFhd9kcftJ4uTYYgFL7fX4UKp0MHZGGyrgwCOAw0k72XpnjittUeviZWEUR0rAeIlL6Luu9tmEXPQnfEhJUtMSA8vHpyMSjuvpOChbwVqEII+PgU78GSw7QwaSJS3Bog5mrSCRJTp1ZnAssO+iQbI9+CSf8U0mfR9nrWWKwPaFHiV2rmc0OkJMkj4xFHGiIbrcSt6azgqRZs6OFEknd+NRUrFOA6e3whODL+NvhG4sRS5ABU6ZAn4C1zu2o9+fkq8n2JvMiHyxKT7YVgtYQ2S5bT88FsQ5/YBjhaArtjSa4lhvZprlwsmNJjpg5zwbovTxsGPETzj3E4ghz0PZYSOM+92tF7u7Vzsnx8cnR6elWNMNHV62hYWGnHx0vblEw6u+QekfoAA3N8H2XcDEi7vTd5f8UQ8AACAjdfXWQAA8OIzKP/e3C937jYCACwYACABf9Tdj/6KkFHYyI6aoRpSYydFtU+qJoomkWrx0Zl0OyJVZfACw0LkpCTEKESG6BpvZZN/dqh8bsqXqfmHMBYwqLI/8xHBkPdramTsJMsI4LFM1inXXJM2joptG5NvFw+qmuTuDIDM5nJnpf50k7/ByxXq2az6dPr9dvMgqF9aH/gYggCQ6jS2uKv+5BQOrgDw9b06CAQAOEkQUGkAADK9WuwPMXR/fxjZxP6IYNL9UbZ27Y/BZC2wRTMAQdL8xlIZvpApLw5SgiTLTrFIcpQpDMRSwlpITSIvVl2MiYvjEhf2HP3Vcw5RZMoKmIsnVXuJE4Cn149XJtmVy3pavydL1IQpSp5BSMOilOVs+00lKYkXaRv82oS9RAEKHWZu5InccFB6Har0YRAHvMg0DSBGp62eYAGAOqrYHVarLQc1TXT1r6mqt9lnL3Tlh1TF68GQP9Dl96Ytu+2D50EOXH3KLr3/YIfS5cOQrhr71kJLNtgle6p4SFE0kCoczxTMic7vsPP6QXJbs5yms5v1IFm1RmYVGeySXoVIqwxGalHbm6nFGQfZk6VIiTo5QSeleCoxOjBEAakE/4zQFzreQgscvobirA8j1tSIMWp8I0S0cWbsNkxGlFaLVM0ilBpPqoXLaOySOxlh0pQRKtRChDMjmKsFQf1bIGoEMDV/quZnXMPHOwfenjx4eUrh7trMzdWZ4ewUDSduCBw50dhh48ywtalhY4KVkUGKtRGVYWVgaJBi+UxYYKoubAM1bUBnsMxqYKbLpYW+LoPRx+NN9dRYjnG88eaaiLOBQbK+t2568toz9X7WB4Ng3YSk5kyN13amxFMTKYmaWEMeT8WON0iAuJ+GHavIKnfStbTIqLqxif/wK6y4fu3UsdNwFw6B74xY9vBjXg27+4ZhtTqUhPg5fffpEwwO9/F1Wb5QmHueEUkGO2xGR29l2z8eg8z52xeub8/p7ApyI87gFrq8dgpemHlmPSuX3nML2QAA); }middle dispatcher — single processbunqueueenginehookreceiverwatchdogtickerSQLitebun:sqlite · WALBun.serveHTTP + SSE · :4120GitHubsource of truthread-onlydashboardtmux sessions(agents)git worktrees \ No newline at end of file diff --git a/docs/dogfooding.md b/docs/dogfooding.md index 399cd71a..8073f328 100644 --- a/docs/dogfooding.md +++ b/docs/dogfooding.md @@ -15,7 +15,7 @@ the hook script, session bookkeeping). The crossover commands conflict with that in two load-bearing ways: 1. **`mm dispatch . ` nests agents.** It needs the dispatcher process - (`mm start`, port 8822), creates a *second* worktree, and spawns a *second* + (`mm start`, port 4120), creates a *second* worktree, and spawns a *second* `claude`/tmux session. Run from inside a live dispatch it collides with the running dispatcher and consumes a slot against itself. It also needs a **manually created** Epic to target. @@ -56,7 +56,7 @@ gh issue list --repo thejustinwalsh/middle --label agent-queue:state # 5. Dispatch a manually created Epic against middle. `mm dispatch` is # self-contained — it runs its OWN hook server + workflow engine inline — so -# do NOT run `mm start` first: both bind the dispatcher port (8822) and a +# do NOT run `mm start` first: both bind the dispatcher port (4120) and a # running `mm start` makes `mm dispatch` fail with EADDRINUSE. (`mm start` is # the long-running daemon for Phase 8 auto-dispatch, not manual dispatch.) mm dispatch . # the Epic (see "Creating a dispatchable Epic" below) diff --git a/docs/operator.md b/docs/operator.md new file mode 100644 index 00000000..eb2bae62 --- /dev/null +++ b/docs/operator.md @@ -0,0 +1,123 @@ +# Operator guide + +Run middle day to day: start the dispatcher, put work on the board, watch it, and keep its state healthy. This guide assumes you have installed the prerequisites and linked the `mm` CLI — see the [README](../README.md) if you have not. + +## The daily loop + +```bash +mm doctor # check the toolchain and middle's own state +mm start # start the dispatcher (hook server + workflow engine) +mm dispatch # hand an Epic to an agent +mm status # see who is working, blocked, or parked +mm stop # shut the dispatcher down +``` + +`` is a path to a **local checkout** of the target repo, not an `owner/name` slug. `` is an Epic (or standalone issue) number in that repo. + +## Start and stop the dispatcher + +`mm start` spawns the dispatcher as a background process and records its pid in `~/.middle/dispatcher.pid`. The dispatcher serves the hook receiver and the dashboard on one port (default `4120`, set by `global.dispatcher_port`). + +```bash +mm start # start in the background +mm start --window # also open the queue observability page once it is up +mm stop # SIGTERM the recorded pid and clear the pidfile +``` + +To run the dispatcher in the foreground during development, use `scripts/dev.sh` instead of `mm start`. Set `MIDDLE_CONFIG` to point at a non-default config file. + +## Dispatch work + +```bash +mm dispatch # force-dispatch one Epic now +``` + +A dispatch creates a fresh worktree, launches the agent in a `tmux` session, hands it the dispatch brief (`.middle/prompt.md`), and drives it through the Epic's sub-issues. The agent pushes commits to one draft PR and, when every phase passes its gates, flips the PR to ready-for-review and posts a reviewer's brief. middle never merges — that is yours. + +To let middle pick work itself instead of dispatching by hand, turn on auto-dispatch and the recommender for a repo (both default off): + +```bash +mm config auto_dispatch true # let the recommender's ranked work auto-dispatch +mm run-recommender # rank the backlog now (rewrites the state issue; dispatches nothing) +``` + +The recommender rewrites the repo's **state issue** — a single GitHub issue holding the ranked dispatch plan and a needs-human digest. `mm run-recommender` is read-only with respect to dispatch: it ranks, it does not launch. + +## Pause and resume a repo + +```bash +mm pause # stop auto-dispatching this repo (in-flight work continues) +mm resume # clear the pause +``` + +Pausing sets `repo_config.paused_until`; the auto-dispatch loop skips paused repos. It does not touch work already in flight. + +## Read `mm status` + +`mm status` prints a one-screen summary of every managed repo and the state of its workflows. Workflow states are: `pending`, `launching`, `running`, `waiting-human`, `rate-limited`, `completed`, `compensated`, `failed`, `cancelled`. A workflow in `waiting-human` is parked on a question or a review — it needs you. + +## Run the health check + +`mm doctor` is the command to run when something feels off. It checks: + +- the external tools every dispatch shells out to — `bun`, `tmux` (≥ 3.5), `claude`, `git`, `gh`, and `gh` auth; +- whether the `mm` symlink's directory is on your `PATH` (`--fix` writes the export to your shell rc); +- that your config files parse; +- that the dispatcher is reachable on its port; +- that the state-issue parser still round-trips against `schemas/state-issue.v1.md`; +- SQLite row counts and the most recent retention run; +- repo-convention drift (skills mirror, module-index frontmatter, TSDoc coverage). + +```bash +mm doctor # report; exit non-zero if any check fails +mm doctor --fix # also append the bun PATH export to ~/.zshrc / ~/.bashrc +``` + +Each check is pass (`✓`), warn (`!`), or fail (`✗`). Warnings mean degraded-but-functional; the command exits non-zero only on a failure. + +## Back up and restore state + +middle's SQLite database holds operational bookkeeping — workflow rows, the event log, rate-limit state. GitHub holds the work itself (issues, sub-issues, PRs), so a backup captures middle's state, never GitHub's. + +```bash +scripts/backup.sh # write middle-backup-.tar.gz to the current dir +scripts/backup.sh --out ~/backups # choose where the archive lands +scripts/backup.sh --restore # restore (refuses while the dispatcher is up) +``` + +The backup snapshots the live database with SQLite's `VACUUM INTO`, so it is consistent even while the dispatcher runs — you do not have to stop it to back up. Restoring overwrites the database and refuses to run while the dispatcher is up; stop it with `mm stop` first, then `mm start` after (the dispatcher migrates the restored db if needed). + +## Reset the database + +```bash +scripts/reset-db.sh # delete the db (+ -wal/-shm) after a confirmation prompt +scripts/reset-db.sh --yes # skip the prompt +``` + +`reset-db.sh` refuses while the dispatcher is running, lists exactly what it will delete, and confirms first. The dispatcher recreates an empty, migrated database on the next `mm start`. It never touches GitHub — a reset loses in-flight workflow rows and the event log, not work. Back up first with `scripts/backup.sh`. + +## Retention + +The dispatcher runs a daily retention cron so operational state does not grow without bound: + +- `events` rows older than 14 days are deleted; +- `completed` workflows older than 30 days are archived — their events are dropped, while the row, its final state, and its config snapshot are preserved. + +Retention touches only middle's SQLite. `mm doctor`'s `database` line reports the most recent retention run. + +## Command reference + +| Command | What it does | +|---|---| +| `mm init ` | Bootstrap middle into a repo (skills, hooks, config, state issue) | +| `mm uninit ` | Remove middle from a repo | +| `mm start [--window]` | Start the dispatcher | +| `mm stop` | Stop the dispatcher | +| `mm status` | One-screen summary of repos and workflow states | +| `mm doctor [--fix]` | Full health check | +| `mm dispatch ` | Force-dispatch an Epic (or standalone issue) | +| `mm run-recommender ` | Rank the backlog now (rewrites the state issue) | +| `mm pause ` / `mm resume ` | Pause / resume auto-dispatch for a repo | +| `mm config ` | Set a per-repo config value | +| `mm docs ` | Trigger a docs-harvester audit run (read-only) | +| `mm version` | Print the `mm` version | diff --git a/docs/skill-enforcement.md b/docs/skill-enforcement.md new file mode 100644 index 00000000..d6af9ca3 --- /dev/null +++ b/docs/skill-enforcement.md @@ -0,0 +1,45 @@ +# Skill enforcement + +A dispatched agent runs a skill (`implementing-github-issues`) that describes the workflow — open a draft PR, work the phases, verify each, mark ready. The skill is instructions, and instructions can be skipped. middle's job is to make the load-bearing steps **mechanical**: gates that pass or fail on evidence, not on the agent's say-so. This document explains how those gates work and why they exist. + +The principle: the skill says what good work looks like; the gates check that it happened. An agent that forgets to post its plan, ticks a phase whose tests fail, or marks a PR ready with unmet acceptance criteria is corrected by the system, not by a human noticing later. + +## Hooks are the observation channel + +`mm init` installs a hook script that POSTs every agent event to the dispatcher. Two of those events carry the workflow: + +- **`SessionStart`** establishes the session — its payload yields the transcript path the dispatcher reads for the rest of the run. +- **`Stop`** is a turn boundary. At each Stop the dispatcher classifies what the agent did (`classifyStop` in [adapters.md](adapters.md#stop-classification)): it finished, it asked a question, it hit a rate limit, or it stopped without a clear outcome. + +The hooks are the fast path; the on-disk transcript is the source of truth. The watchdog cron reconciles the two so a dropped hook does not strand a workflow (see [architecture.md](architecture.md#the-crons)). + +## The plan-comment guard + +The skill requires the agent to post its plan as a comment on the Epic before writing code. The guard (`packages/dispatcher/src/gates/plan-comment.ts`) verifies a plan comment by the agent's account exists on the issue. No plan comment, no progress — the public commitment the skill calls "non-negotiable" is enforced rather than trusted. + +## The PR-ready gate + +The strongest gate intercepts `gh pr ready`. A `PreToolUse` hook matches the command and calls the dispatcher's `/gates/pr-ready` endpoint before the tool runs (`packages/dispatcher/src/gates/pr-ready.ts`). + +The dispatcher walks the Epic PR's acceptance criteria — the union of every sub-issue's criteria, all rendered into the one PR body — and requires each to carry **either**: + +- an evidence link (a URL or `#`-reference proving delivery), **or** +- a `(deferred: )` annotation whose linked comment is by a non-bot user. + +An empty criteria section denies, so the gate can't be bypassed by deleting it. A 200 response lets the tool run; a 403 with a reason blocks it and prints why. This is what stops an agent from marking a PR ready with work still undone or with scope unilaterally cut. + +## Phase-verification gates + +When the agent ticks a sub-issue's Status checkbox `[ ] → [x]` and pushes, the dispatcher runs that sub-issue's verification gates — the lint, typecheck, test, and acceptance commands declared in the repo's `verify.toml` (`packages/dispatcher/src/gates/verify.ts`). It posts an evidence comment for the phase. If a gate fails, the dispatcher reverts the checkbox and comments naming the failed gate (`checkbox-revert.ts`). A phase is "done" only when its gates actually pass. + +## Parking instead of guessing + +When the agent hits a question it can't resolve — ambiguous acceptance criteria, or a decision needing more candidate forks than the configured complexity ceiling — the skill writes `.middle/blocked.json` and exits rather than guessing. `classifyStop` detects the sentinel and classifies the Stop as `asked-question`; the dispatcher arms a `waitFor` signal, parks the workflow as `waiting-human`, and surfaces the question on the Epic. + +The poller watches for the unblocking event — a human reply on the issue, or a PR review verdict — and fires the resume signal, which re-enters the parked workflow where it left off. A complexity pause (`kind: "complexity"` in the sentinel) is surfaced distinctly so a human can reduce scope or approve a best-judgment call. + +This is the design's bright line: an agent that is stuck parks and escalates; it never guesses past a real blocker, and it never sits idle (the watchdog kills a stalled session). + +## What middle never does + +middle stops at "PR ready for review." The final review and the merge are the human's gate — no gate, hook, or cron merges a PR. Enforcement holds the agent to the workflow; it does not sign off on the work. diff --git a/packages/cli/src/checks/state-issue.ts b/packages/cli/src/checks/state-issue.ts new file mode 100644 index 00000000..6313aa42 --- /dev/null +++ b/packages/cli/src/checks/state-issue.ts @@ -0,0 +1,112 @@ +import { existsSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { + isParseError, + type ParsedState, + parseStateIssue, + renderStateIssue, + validate, +} from "@middle/state-issue"; + +/** + * @packageDocumentation + * @module @middle/cli/checks/state-issue + * + * The `mm doctor` state-issue self-check: re-validate the parser/renderer/validate + * machinery against `schemas/state-issue.v1.md` (the schema source of truth) using + * the package's canonical conforming fixture. A parse failure, a broken + * byte-identical round-trip, or a failed `validate()` means the parser has drifted + * from the schema doc and the dispatcher's read/write of state issues is unsafe. + * + * Public surface: + * - `checkStateIssue` — resolve the schema doc + fixture from middle's source tree + * and run the round-trip, returning a doctor check status + * - `checkStateIssueRoundTrip` — the pure parse → render → validate check (testable) + * - `SCHEMA_DOC_PATH`, `STATE_ISSUE_FIXTURE_PATH` — the resolved source-tree paths + * + * Where things live: + * - this file — the whole check (resolves paths like the module-index check does) + * + * Gotchas: + * - Paths resolve from this module's location so the check inspects middle's own + * source tree, not the cwd's repo (matches the skills-drift / module-index checks). + * - The fixture is a fixed conformance artifact; the check uses the same adapter + * set the fixture's own round-trip test does (`claude`, `codex`) — it is a + * self-test of the machinery, independent of the operator's configured adapters. + * + * claude-md: false + */ + +/** middle's repo root, resolved from this file (`packages/cli/src/checks` → up 4). */ +const REPO_ROOT = join(import.meta.dir, "..", "..", "..", ".."); + +/** The schema source of truth the parser/renderer/validate conform to. */ +export const SCHEMA_DOC_PATH = join(REPO_ROOT, "schemas", "state-issue.v1.md"); + +/** The canonical conforming state-issue body the round-trip is checked against. */ +export const STATE_ISSUE_FIXTURE_PATH = join( + REPO_ROOT, + "packages", + "state-issue", + "test", + "fixtures", + "state-issue.example.md", +); + +/** + * Adapter set the fixture is authored against — kept in lockstep with the + * fixture's own round-trip test (`packages/state-issue/test/fixture.test.ts`). + * The check is a self-test of the parse/render/validate machinery, so the + * adapter set comes from the fixture, not the operator's config. + */ +const FIXTURE_ADAPTERS = ["claude", "codex"] as const; + +/** Outcome of the pure round-trip check: did the machinery hold, and why/why not. */ +export type StateIssueCheckResult = { ok: boolean; detail: string }; + +/** + * The pure check: `parseStateIssue` the body, assert the parse succeeded, assert + * `renderStateIssue` reproduces the body **byte-identically** (the hard round-trip + * invariant the dispatcher relies on to edit one section without disturbing + * others), and assert `validate` passes. Returns the first failure it hits. + */ +export function checkStateIssueRoundTrip(body: string, adapters: string[]): StateIssueCheckResult { + const parsed = parseStateIssue(body); + if (isParseError(parsed)) return { ok: false, detail: `parse failed — ${parsed.message}` }; + const rendered = renderStateIssue(parsed); + if (rendered !== body) { + return { ok: false, detail: "round-trip broken — render is not byte-identical to the fixture" }; + } + const result = validate(parsed as ParsedState, { adapters }); + if (!result.ok) return { ok: false, detail: `validate failed — ${result.errors.join("; ")}` }; + return { ok: true, detail: "parser ↔ renderer round-trip + validate OK against v1 schema" }; +} + +/** A doctor check status: pass when the machinery holds, fail when it drifted. */ +export type StateIssueCheckStatus = { status: "pass" | "warn" | "fail"; detail: string }; + +/** + * Resolve the schema doc + canonical fixture from middle's source tree and run + * {@link checkStateIssueRoundTrip}. Degrades to `warn` (not `fail`) when either + * artifact is absent — that's an unusual install layout, not a parser defect. + */ +export function checkStateIssue(): StateIssueCheckStatus { + if (!existsSync(SCHEMA_DOC_PATH)) { + return { status: "warn", detail: "schemas/state-issue.v1.md not found — skipped" }; + } + if (!existsSync(STATE_ISSUE_FIXTURE_PATH)) { + return { status: "warn", detail: "canonical state-issue fixture not found — skipped" }; + } + try { + const body = readFileSync(STATE_ISSUE_FIXTURE_PATH, "utf8"); + const result = checkStateIssueRoundTrip(body, [...FIXTURE_ADAPTERS]); + return result.ok + ? { status: "pass", detail: result.detail } + : { status: "fail", detail: result.detail }; + } catch (error) { + return { + status: "fail", + detail: `state-issue fixture unreadable — ${(error as Error).message}`, + }; + } +} diff --git a/packages/cli/src/commands/doctor.ts b/packages/cli/src/commands/doctor.ts index 03f19fbe..5f1472eb 100644 --- a/packages/cli/src/commands/doctor.ts +++ b/packages/cli/src/commands/doctor.ts @@ -1,10 +1,15 @@ +import { existsSync, readFileSync } from "node:fs"; import { homedir } from "node:os"; -import { loadConfig } from "@middle/core"; +import { join } from "node:path"; +import { loadConfig, type MiddleConfig } from "@middle/core"; +import { currentSchemaVersion, openDb } from "@middle/dispatcher/src/db.ts"; +import { collectRetentionStatus, type RetentionStatus } from "@middle/dispatcher/src/retention.ts"; import { getTmuxVersion, MIN_TMUX_VERSION, tmuxVersionAtLeast, } from "@middle/dispatcher/src/tmux.ts"; +import { defaultPidFile } from "../paths.ts"; import { BOOTSTRAP_SKILLS_DIR, CANONICAL_SKILLS_DIR, @@ -18,8 +23,13 @@ import { resolveShellRc, } from "../checks/bun-path.ts"; import { checkModuleIndex } from "../checks/module-index.ts"; +import { checkStateIssue } from "../checks/state-issue.ts"; import { checkTsdocCoverage } from "../checks/tsdoc-coverage.ts"; +/** Schema version migration 007 (retention) brings the db to — `mm doctor` + * reports retention status only once the db is at least here. */ +const RETENTION_SCHEMA_VERSION = 7; + type CheckStatus = "pass" | "warn" | "fail"; type Check = { name: string; status: CheckStatus; detail: string }; @@ -122,21 +132,17 @@ async function runBunPathFix(): Promise { * Check the binary of every configured + enabled adapter (not just `claude`). * A missing binary is a **warning**, not a failure: middle can still dispatch * with whichever adapters are installed — the recommender / `mm dispatch` simply - * won't pick the absent one. When the config file is absent, `loadConfig`'s - * defaults still describe both `claude` and `codex`, so a default environment is - * checked for both. The check `name` is the adapter name so each gets its own row. + * won't pick the absent one. Takes the **same** repo-aware config `runDoctor` + * already resolved (via {@link loadDoctorConfig}) rather than reloading global-only + * config — so a repo's `.middle/config.toml` adapter set is honored here too. A + * `null` config means it failed to parse (already a hard `fail` on the config row), + * so adapter checks are skipped with a warning. The check `name` is the adapter + * name so each gets its own row. */ -async function checkAdapterBinaries(): Promise { - let config: ReturnType; - try { - config = loadConfig({ globalPath: process.env.MIDDLE_CONFIG }); - } catch (error) { +export async function checkAdapterBinaries(config: MiddleConfig | null): Promise { + if (!config) { return [ - { - name: "adapters", - status: "warn", - detail: `config unreadable: ${(error as Error).message}`, - }, + { name: "adapters", status: "warn", detail: "config unreadable — adapter checks skipped" }, ]; } const enabled = Object.entries(config.adapters).filter(([, a]) => a.enabled); @@ -237,21 +243,188 @@ function checkTsdocCoverageWarn(): Check { } /** - * `mm doctor` — run a system check for every external tool the dispatcher - * shells out to: `bun`, `tmux` (≥ 3.5), each configured adapter's binary (e.g. - * `claude`, `codex`), `git`, `gh`, and `gh` auth. Exits 0 when no check fails; + * Load middle's config (global, plus the cwd's `.middle/config.toml` when the + * operator runs `mm doctor` from inside a managed repo) and report whether it + * parses. A malformed TOML throws out of `loadConfig` — that's a hard fail (the + * dispatcher can't start). The parsed config is handed to the downstream + * dispatcher/database checks so they read the operator's real port and db path. + */ +function loadDoctorConfig(): { check: Check; config: MiddleConfig | null } { + const globalPath = process.env.MIDDLE_CONFIG ?? join(homedir(), ".middle", "config.toml"); + const repoConfigPath = join(process.cwd(), ".middle", "config.toml"); + const hasRepoConfig = existsSync(repoConfigPath); + try { + const config = loadConfig({ + globalPath: process.env.MIDDLE_CONFIG, + repoPath: hasRepoConfig ? repoConfigPath : undefined, + }); + const sources = [existsSync(globalPath) ? globalPath : `${globalPath} (defaults)`]; + if (hasRepoConfig) sources.push(repoConfigPath); + return { + check: { name: "config", status: "pass", detail: `parsed — ${sources.join(", ")}` }, + config, + }; + } catch (error) { + return { + check: { + name: "config", + status: "fail", + detail: `failed to parse — ${(error as Error).message}`, + }, + config: null, + }; + } +} + +/** Is the pid recorded in the pidfile a live process? Best-effort (`kill -0`). */ +function dispatcherPidAlive(): boolean { + const pidFile = defaultPidFile(); + if (!existsSync(pidFile)) return false; + const pid = Number(readFileSync(pidFile, "utf8").trim()); + if (!Number.isInteger(pid) || pid <= 0) return false; + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +/** Probe the dispatcher's `/health` endpoint with a short timeout. */ +async function probeHealth(port: number): Promise<{ ok: boolean; version: string }> { + try { + const res = await fetch(`http://127.0.0.1:${port}/health`, { + signal: AbortSignal.timeout(2000), + }); + if (!res.ok) return { ok: false, version: "" }; + const body = (await res.json().catch(() => null)) as { ok?: unknown; version?: unknown } | null; + return { + ok: body?.ok === true, + version: typeof body?.version === "string" ? body.version : "", + }; + } catch { + return { ok: false, version: "" }; + } +} + +/** + * Check the dispatcher is reachable. Reachable `/health` → pass. A live pidfile + * but an unreachable `/health` → **fail** (the daemon is wedged). No pidfile / + * dead process → **warn**: the dispatcher simply isn't started, which is normal + * when an operator runs `mm doctor` before `mm start`. + */ +async function checkDispatcher(config: MiddleConfig | null): Promise { + const port = config?.global.dispatcherPort ?? 4120; + const health = await probeHealth(port); + if (health.ok) { + const v = health.version ? ` (v${health.version})` : ""; + return { name: "dispatcher", status: "pass", detail: `reachable on :${port}${v}` }; + } + if (dispatcherPidAlive()) { + return { + name: "dispatcher", + status: "fail", + detail: `pidfile live but /health on :${port} unreachable — dispatcher may be wedged`, + }; + } + return { name: "dispatcher", status: "warn", detail: `not running — run \`mm start\`` }; +} + +/** Render a unix-ms timestamp as a coarse "Ns/Nm/Nh/Nd ago" relative to `now`. */ +export function formatAgo(then: number, now: number): string { + const sec = Math.max(0, Math.round((now - then) / 1000)); + if (sec < 60) return `${sec}s ago`; + const min = Math.round(sec / 60); + if (min < 60) return `${min}m ago`; + const hr = Math.round(min / 60); + if (hr < 48) return `${hr}h ago`; + return `${Math.round(hr / 24)}d ago`; +} + +/** + * Format the db row counts + last retention run into a doctor detail line, and + * decide the status: a *failed* last retention run degrades to `warn` (retention + * is broken but dispatch still works); otherwise `pass`. Pure so it unit-tests + * without a real db. + */ +export function summarizeRetention( + status: RetentionStatus, + now: number, +): { status: CheckStatus; detail: string } { + const { workflows, archivedWorkflows, events } = status.rowCounts; + const counts = `${workflows} workflows (${archivedWorkflows} archived), ${events} events`; + const last = status.lastRun; + if (!last) { + return { status: "pass", detail: `${counts} · retention never run` }; + } + const verdict = last.ok ? "ok" : "FAILED"; + const retention = `retention ${verdict} ${formatAgo(last.ranAt, now)} (−${last.eventsDeleted} events, ${last.workflowsArchived} archived)`; + return { status: last.ok ? "pass" : "warn", detail: `${counts} · ${retention}` }; +} + +/** + * Report SQLite row counts and recent retention-run status. No db file yet + * (dispatcher never started) → `warn`. A db below the retention schema version → + * `warn` (start the dispatcher to migrate). A db that can't be opened → `fail`. + */ +function checkDatabase(config: MiddleConfig | null): Check { + const dbPath = config?.global.dbPath ?? join(homedir(), ".middle", "db.sqlite3"); + if (!existsSync(dbPath)) { + return { + name: "database", + status: "warn", + detail: `${dbPath} not created yet — run \`mm start\` once`, + }; + } + let db: ReturnType | null = null; + try { + db = openDb(dbPath); + const version = currentSchemaVersion(db); + if (version < RETENTION_SCHEMA_VERSION) { + return { + name: "database", + status: "warn", + detail: `schema v${version} (pre-retention, needs ≥ v${RETENTION_SCHEMA_VERSION}) — run \`mm start\` to migrate`, + }; + } + const summary = summarizeRetention(collectRetentionStatus(db), Date.now()); + return { name: "database", status: summary.status, detail: summary.detail }; + } catch (error) { + return { + name: "database", + status: "fail", + detail: `cannot read ${dbPath} — ${(error as Error).message}`, + }; + } finally { + db?.close(); + } +} + +/** + * `mm doctor` — full operator health check. Validates the toolchain every + * dispatch shells out to (`bun`, `tmux` ≥ 3.5, each configured adapter's binary + * — e.g. `claude`, `codex` — `git`, `gh`, and `gh` auth), that config parses, + * the dispatcher is reachable, the state-issue parser still round-trips against + * its v1 schema, and reports SQLite row counts + recent retention status — plus + * the repo's skills/docs-convention drift warnings. Exits 0 when no check fails; * 1 if anything is missing or broken. Warnings (degraded but functional — a * missing adapter binary among others present) do not fail the run. */ export async function runDoctor({ fix }: { fix?: boolean } = {}): Promise { + const { check: configCheck, config } = loadDoctorConfig(); + const stateIssue = checkStateIssue(); const checks: Check[] = [ await checkBinary("bun", ["bun", "--version"]), await checkBunPath(), await checkTmux(), - ...(await checkAdapterBinaries()), + ...(await checkAdapterBinaries(config)), await checkBinary("git", ["git", "--version"]), await checkBinary("gh", ["gh", "--version"]), await checkGhAuth(), + configCheck, + await checkDispatcher(config), + { name: "state-issue", status: stateIssue.status, detail: stateIssue.detail }, + checkDatabase(config), checkSkillsDrift(), checkModuleIndexFrontmatter(), checkTsdocCoverageWarn(), @@ -259,7 +432,7 @@ export async function runDoctor({ fix }: { fix?: boolean } = {}): Promise, +): Promise<{ code: number; stdout: string; stderr: string }> { + const proc = Bun.spawn(["bash", script, ...args], { + stdout: "pipe", + stderr: "pipe", + env: env ? { ...process.env, ...env } : process.env, + }); + const [stdout, stderr] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + ]); + return { code: await proc.exited, stdout, stderr }; +} + +beforeEach(() => { + home = mkdtempSync(join(tmpdir(), "middle-home-")); + out = mkdtempSync(join(tmpdir(), "middle-out-")); +}); + +afterEach(() => { + rmSync(home, { recursive: true, force: true }); + rmSync(out, { recursive: true, force: true }); +}); + +describe("backup.sh + reset-db.sh round-trip", () => { + test("backup → reset → restore preserves the db and its rows", async () => { + seedDb(); + + const backup = await run(BACKUP, ["--home", home, "--out", out]); + expect(backup.code).toBe(0); + const archives = readdirSync(out).filter((f) => f.endsWith(".tar.gz")); + expect(archives.length).toBe(1); + const archive = join(out, archives[0]!); + + const reset = await run(RESET, ["--home", home, "--yes"]); + expect(reset.code).toBe(0); + expect(existsSync(join(home, "db.sqlite3"))).toBe(false); + expect(reset.stdout).toContain("GitHub was not touched"); + + const restore = await run(BACKUP, ["--restore", archive, "--home", home, "--yes"]); + expect(restore.code).toBe(0); + expect(existsSync(join(home, "db.sqlite3"))).toBe(true); + + // The restored db is intact: schema migrated, and the seeded row survived. + const db = openAndMigrate(join(home, "db.sqlite3")); + expect(currentSchemaVersion(db)).toBe(7); + const row = db.query("SELECT id FROM workflows WHERE id = 'wf-keep'").get(); + expect(row).toEqual({ id: "wf-keep" }); + db.close(); + }); +}); + +describe("safety guards", () => { + test("backup.sh fails when there is no database", async () => { + const r = await run(BACKUP, ["--home", home, "--out", out]); + expect(r.code).toBe(1); + expect(r.stderr).toContain("nothing to back up"); + }); + + test("reset-db.sh is a no-op (exit 0) when there is no database", async () => { + const r = await run(RESET, ["--home", home, "--yes"]); + expect(r.code).toBe(0); + expect(r.stdout).toContain("nothing to reset"); + }); + + test("reset-db.sh refuses while the dispatcher pidfile is live", async () => { + seedDb(); + // This test process is, by definition, alive — use its own pid as the sentinel. + writeFileSync(join(home, "dispatcher.pid"), String(process.pid)); + const r = await run(RESET, ["--home", home, "--yes"]); + expect(r.code).toBe(1); + expect(r.stderr).toContain("dispatcher is running"); + expect(existsSync(join(home, "db.sqlite3"))).toBe(true); // untouched + }); + + test("--db points both scripts at a relocated database", async () => { + // A db that does NOT live at /db.sqlite3 — the relocated-db case the + // config's db_path covers. --home stays a different, empty dir. + const relocated = join(out, "elsewhere.sqlite3"); + const dbA = openAndMigrate(relocated); + dbA.run( + `INSERT INTO workflows (id, kind, repo, adapter, state, created_at, updated_at) + VALUES ('wf-reloc', 'implementation', 'o/r', 'claude', 'completed', 1, 1)`, + ); + dbA.close(); + + const backup = await run(BACKUP, ["--db", relocated, "--home", home, "--out", out]); + expect(backup.code).toBe(0); + + const reset = await run(RESET, ["--db", relocated, "--home", home, "--yes"]); + expect(reset.code).toBe(0); + expect(existsSync(relocated)).toBe(false); // the relocated db, not /db.sqlite3 + }); + + test("restore creates missing parent dirs for a relocated db and config", async () => { + // Back up a db + config from , then restore into destinations whose + // parent directories do NOT exist yet (a relocated --db / MIDDLE_CONFIG). + // Bare `cp` would abort on the missing parents; restore must mkdir -p first. + seedDb(); + writeFileSync(join(home, "config.toml"), 'default_adapter = "claude"\n'); + const backup = await run(BACKUP, ["--home", home, "--out", out]); + expect(backup.code).toBe(0); + const archive = join(out, readdirSync(out).find((f) => f.endsWith(".tar.gz"))!); + + const dbDest = join(out, "no", "such", "dir", "relocated.sqlite3"); + const cfgDest = join(out, "other", "missing", "config.toml"); + const restore = await run( + BACKUP, + ["--restore", archive, "--db", dbDest, "--home", join(out, "fresh-home"), "--yes"], + { MIDDLE_CONFIG: cfgDest }, + ); + expect(restore.code).toBe(0); + expect(existsSync(dbDest)).toBe(true); + expect(existsSync(cfgDest)).toBe(true); + expect(restore.stdout).toContain("restored: config.toml"); + + // The relocated db is the real, migrated db with the seeded row intact. + const db = openAndMigrate(dbDest); + const row = db.query("SELECT id FROM workflows WHERE id = 'wf-keep'").get(); + expect(row).toEqual({ id: "wf-keep" }); + db.close(); + }); + + test("restore refuses while the dispatcher pidfile is live", async () => { + seedDb(); + await run(BACKUP, ["--home", home, "--out", out]); + const archive = join(out, readdirSync(out).find((f) => f.endsWith(".tar.gz"))!); + writeFileSync(join(home, "dispatcher.pid"), String(process.pid)); + const r = await run(BACKUP, ["--restore", archive, "--home", home, "--yes"]); + expect(r.code).toBe(1); + expect(r.stderr).toContain("dispatcher is running"); + }); +}); diff --git a/packages/cli/test/doctor.test.ts b/packages/cli/test/doctor.test.ts index 0c444ddf..a6cb5ccb 100644 --- a/packages/cli/test/doctor.test.ts +++ b/packages/cli/test/doctor.test.ts @@ -1,13 +1,24 @@ import { describe, expect, spyOn, test } from "bun:test"; -import { runDoctor } from "../src/commands/doctor.ts"; +import type { AdapterConfig, MiddleConfig } from "@middle/core"; +import type { RetentionStatus } from "@middle/dispatcher/src/retention.ts"; +import { + checkAdapterBinaries, + formatAgo, + runDoctor, + summarizeRetention, +} from "../src/commands/doctor.ts"; // runDoctor shells out to bun/tmux/claude/git/gh — these all exist on the // machine middle is built for, so the happy path is verifiable. We don't fake // out missing binaries here (that's interactive operator territory); the unit // behavior of the version checks is covered by the tmux helpers' unit tests. +// The config/dispatcher/database checks degrade to pass-or-warn off the happy +// path (no config → defaults; no daemon → "not running"; no db → "not created"), +// never fail, so the run still returns 0; their formatting logic is unit-tested +// below against fabricated inputs. describe("runDoctor — happy path", () => { - test("returns 0 and prints a check per tool when the toolchain is healthy", async () => { + test("returns 0 and prints every check when the toolchain is healthy", async () => { const lines: string[] = []; const spy = spyOn(console, "log").mockImplementation((...args: unknown[]) => { lines.push(args.join(" ")); @@ -29,6 +40,10 @@ describe("runDoctor — happy path", () => { "git", "gh", "gh auth", + "config", + "dispatcher", + "state-issue", + "database", "skills", "docs", "tsdoc", @@ -37,3 +52,114 @@ describe("runDoctor — happy path", () => { } }); }); + +describe("checkAdapterBinaries", () => { + const adapter = (enabled: boolean, binary: string): AdapterConfig => ({ + enabled, + binary, + extraArgs: [], + }); + const withAdapters = (adapters: Record): MiddleConfig => + ({ adapters }) as MiddleConfig; + + test("null config (unparseable) → single warn, no throw", async () => { + expect(await checkAdapterBinaries(null)).toEqual([ + { name: "adapters", status: "warn", detail: "config unreadable — adapter checks skipped" }, + ]); + }); + + test("no enabled adapters → warn", async () => { + expect(await checkAdapterBinaries(withAdapters({}))).toEqual([ + { name: "adapters", status: "warn", detail: "no adapters enabled in config" }, + ]); + }); + + test("reports a row per ENABLED adapter from the passed config — not a reloaded global one", async () => { + // `bun` is the runtime, so it's always on PATH: an adapter whose binary is + // `bun` reliably passes, proving the rows came from THIS config object (the + // repo-aware one runDoctor resolved) rather than a reloaded global default. + const checks = await checkAdapterBinaries( + withAdapters({ + repoonly: adapter(true, "bun"), + disabled: adapter(false, "bun"), + }), + ); + expect(checks.map((c) => c.name)).toEqual(["repoonly"]); + expect(checks[0]!.status).toBe("pass"); + expect(checks[0]!.detail).toContain("on PATH"); + }); + + test("enabled adapter with a missing binary → warn (never fail)", async () => { + const checks = await checkAdapterBinaries( + withAdapters({ ghost: adapter(true, "middle-no-such-binary-xyz") }), + ); + expect(checks).toHaveLength(1); + expect(checks[0]!.status).toBe("warn"); + expect(checks[0]!.detail).toContain("not installed"); + }); +}); + +describe("formatAgo", () => { + const now = 1_000_000_000_000; + test("renders sub-minute as seconds", () => { + expect(formatAgo(now - 5_000, now)).toBe("5s ago"); + }); + test("renders minutes, hours, and days at the boundaries", () => { + expect(formatAgo(now - 5 * 60_000, now)).toBe("5m ago"); + expect(formatAgo(now - 3 * 3_600_000, now)).toBe("3h ago"); + expect(formatAgo(now - 3 * 86_400_000, now)).toBe("3d ago"); + }); + test("clamps a future timestamp to 0s (never negative)", () => { + expect(formatAgo(now + 10_000, now)).toBe("0s ago"); + }); +}); + +describe("summarizeRetention", () => { + const now = 1_000_000_000_000; + const counts: RetentionStatus["rowCounts"] = { workflows: 12, archivedWorkflows: 3, events: 40 }; + + test("never-run → pass, reports counts", () => { + const r = summarizeRetention({ rowCounts: counts, lastRun: null }, now); + expect(r.status).toBe("pass"); + expect(r.detail).toContain("12 workflows (3 archived), 40 events"); + expect(r.detail).toContain("retention never run"); + }); + + test("clean last run → pass, reports the run", () => { + const r = summarizeRetention( + { + rowCounts: counts, + lastRun: { + id: 1, + ranAt: now - 3_600_000, + eventsDeleted: 7, + workflowsArchived: 2, + ok: true, + detail: null, + }, + }, + now, + ); + expect(r.status).toBe("pass"); + expect(r.detail).toContain("retention ok 1h ago (−7 events, 2 archived)"); + }); + + test("failed last run → warn, surfaces FAILED", () => { + const r = summarizeRetention( + { + rowCounts: counts, + lastRun: { + id: 2, + ranAt: now - 60_000, + eventsDeleted: 0, + workflowsArchived: 0, + ok: false, + detail: "disk full", + }, + }, + now, + ); + expect(r.status).toBe("warn"); + expect(r.detail).toContain("retention FAILED 1m ago"); + }); +}); diff --git a/packages/cli/test/state-issue-check.test.ts b/packages/cli/test/state-issue-check.test.ts new file mode 100644 index 00000000..af1ec04d --- /dev/null +++ b/packages/cli/test/state-issue-check.test.ts @@ -0,0 +1,59 @@ +import { describe, expect, spyOn, test } from "bun:test"; +import * as fs from "node:fs"; +import { readFileSync } from "node:fs"; +import { + checkStateIssue, + checkStateIssueRoundTrip, + STATE_ISSUE_FIXTURE_PATH, +} from "../src/checks/state-issue.ts"; + +const FIXTURE = readFileSync(STATE_ISSUE_FIXTURE_PATH, "utf8"); +const ADAPTERS = ["claude", "codex"]; + +describe("checkStateIssueRoundTrip", () => { + test("passes for the canonical conforming fixture", () => { + const result = checkStateIssueRoundTrip(FIXTURE, ADAPTERS); + expect(result.ok).toBe(true); + }); + + test("fails when the body does not parse", () => { + const result = checkStateIssueRoundTrip("not a state issue at all", ADAPTERS); + expect(result.ok).toBe(false); + expect(result.detail).toContain("parse failed"); + }); + + // The byte-identical render(parse(body)) === body invariant itself is owned and + // exhaustively tested by packages/state-issue (fixture.test.ts); here we cover + // the parse-fail and validate-fail branches of the doctor wrapper. + + test("fails validate when a Ready row uses an unconfigured adapter", () => { + // The fixture's Ready rows use claude + codex; drop codex from the config. + const result = checkStateIssueRoundTrip(FIXTURE, ["claude"]); + expect(result.ok).toBe(false); + expect(result.detail).toContain("validate failed"); + }); +}); + +describe("checkStateIssue", () => { + test("passes against middle's own source tree", () => { + expect(checkStateIssue().status).toBe("pass"); + }); + + test("returns a structured fail (never throws) when the fixture is unreadable", () => { + // Schema doc + fixture still exist (existsSync untouched), so the check + // reaches the read; force that read to throw the way a permission/I/O error + // would. checkStateIssue must catch it and return a fail status rather than + // letting the exception propagate and abort `mm doctor`. + const spy = spyOn(fs, "readFileSync").mockImplementation(() => { + throw new Error("EACCES: permission denied"); + }); + try { + const result = checkStateIssue(); + expect(result.status).toBe("fail"); + expect(result.detail).toContain("unreadable"); + expect(result.detail).toContain("EACCES"); + } finally { + spy.mockRestore(); + } + }); +}); diff --git a/packages/dispatcher/src/db/migrations/007_retention.sql b/packages/dispatcher/src/db/migrations/007_retention.sql new file mode 100644 index 00000000..bd7e7825 --- /dev/null +++ b/packages/dispatcher/src/db/migrations/007_retention.sql @@ -0,0 +1,23 @@ +-- 007_retention.sql +-- Retention bookkeeping. A daily cron deletes `events` older than 14 days and +-- archives `completed` workflows older than 30 days: their events are dropped +-- while the row itself — final state plus the config snapshot in meta_json — is +-- preserved. `archived_at` both marks an archived workflow (so the pass is +-- idempotent) and lets `mm doctor` distinguish live from archived rows. +-- `retention_runs` records every pass so `mm doctor` can report recent status. +-- Retention is SQLite-only — GitHub is the system of record and is never touched. + +ALTER TABLE workflows ADD COLUMN archived_at INTEGER; -- epoch ms when archived; null = live + +CREATE INDEX idx_workflows_archived ON workflows(archived_at); + +CREATE TABLE retention_runs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + ran_at INTEGER NOT NULL, -- epoch ms when the pass ran + events_deleted INTEGER NOT NULL, -- event rows pruned (older than 14d) + workflows_archived INTEGER NOT NULL, -- completed workflows archived (older than 30d) + ok INTEGER NOT NULL DEFAULT 1, -- 1 = clean pass, 0 = errored + detail TEXT -- error message when ok=0, else null +); + +CREATE INDEX idx_retention_runs_ran_at ON retention_runs(ran_at); diff --git a/packages/dispatcher/src/index.ts b/packages/dispatcher/src/index.ts index 4c1f1f3a..d2736cfc 100644 --- a/packages/dispatcher/src/index.ts +++ b/packages/dispatcher/src/index.ts @@ -26,6 +26,8 @@ * hook receiver + `/control` + `/health` surface * - `startPoller` / `POLLER_INTERVAL_MS`, `startWatchdog` / `WATCHDOG_INTERVAL_MS` * — the reconciliation crons + * - `startRetentionCron` / `runRetentionPass` / `collectRetentionStatus` — the + * daily SQLite retention cron + the status `mm doctor` reads * - `createDurableEngine` / `recoverEngine` / `reconcileOrphanedSignals` — the * durable workflow engine (persistent store, transient queue) and its boot * recovery: re-arm parked `waiting` executions, reconcile orphaned signals (#116) @@ -47,6 +49,8 @@ * - `metrics.ts` — queue observability: the `/metrics` (Prometheus) + * `/control/metrics` (JSON) snapshot exports * - `poller*.ts` / `watchdog*.ts` — the GitHub-poll + liveness crons + * - `retention.ts` / `retention-cron.ts` — the daily events-prune + + * workflow-archival pass and its cron wrapper * - `recovery.ts` — the durable engine factory + boot recovery/reconciliation * - `db.ts`, `db/` — SQLite open/migrate + migrations * - `tmux.ts`, `worktree.ts` — session + worktree lifecycle @@ -78,6 +82,17 @@ export { EventHub } from "./event-hub.ts"; export type { Event, WorkflowEventData } from "./event-hub.ts"; export { POLLER_INTERVAL_MS, startPoller } from "./poller-cron.ts"; export { startWatchdog, WATCHDOG_INTERVAL_MS } from "./watchdog-cron.ts"; +export { startRetentionCron } from "./retention-cron.ts"; +export { + collectRetentionStatus, + EVENTS_MAX_AGE_MS, + getLatestRetentionRun, + RETENTION_CRON_INTERVAL_MS, + type RetentionRun, + type RetentionStatus, + runRetentionPass, + WORKFLOWS_MAX_AGE_MS, +} from "./retention.ts"; export { createDurableEngine, recoverEngine, reconcileOrphanedSignals } from "./recovery.ts"; export type { EngineRecoveryResult, diff --git a/packages/dispatcher/src/main.ts b/packages/dispatcher/src/main.ts index 1d5c6ad2..7ae8f330 100644 --- a/packages/dispatcher/src/main.ts +++ b/packages/dispatcher/src/main.ts @@ -31,6 +31,8 @@ import { startPoller } from "./poller-cron.ts"; import { ghReconcilerGateway, gitOps, reconcileOpenPRs } from "./reconcilers/pr-divergence.ts"; import { createDurableEngine, recoverEngine, reconcileOrphanedSignals } from "./recovery.ts"; import { runRecommenderCronPass, startRecommenderCron } from "./recommender-cron.ts"; +import { startRetentionCron } from "./retention-cron.ts"; +import { runRetentionPass } from "./retention.ts"; import { startAuditCron } from "./audit-cron.ts"; import { startStalenessCron } from "./staleness-cron.ts"; import { isPaused, listManagedRepos, registerManagedRepo } from "./repo-config.ts"; @@ -811,6 +813,19 @@ export async function runDaemon(opts: RunDaemonOptions = {}): Promise { console.error(`[recommender-cron] startup pass failed: ${(error as Error).message}`); }); + // Retention cron: daily, prune `events` older than 14d and archive `completed` + // workflows older than 30d (events dropped, row + final state preserved). + // SQLite-only — never touches GitHub. Run one pass at startup too, so a long + // downtime doesn't leave stale state unpruned until the first daily tick (and + // so `mm doctor` has a recent run to report). Guarded: a failed pass logs and + // records itself in `retention_runs` but never blocks startup. + const stopRetentionCron = await startRetentionCron({ db }); + try { + runRetentionPass(db); + } catch (error) { + console.error(`[retention] startup pass failed: ${(error as Error).message}`); + } + // Epic-cache refresh: an initial pass + a fixed-cadence sweep over every known // repo. Best-effort — a GitHub hiccup logs and the next tick retries. // Ticks are fire-and-forget per repo: a GitHub call slower than the interval @@ -858,6 +873,11 @@ export async function runDaemon(opts: RunDaemonOptions = {}): Promise { } catch (error) { console.error(`shutdown: stopRecommenderCron failed — ${(error as Error).message}`); } + try { + await stopRetentionCron(); + } catch (error) { + console.error(`shutdown: stopRetentionCron failed — ${(error as Error).message}`); + } try { await stopAuditCron(); } catch (error) { diff --git a/packages/dispatcher/src/retention-cron.ts b/packages/dispatcher/src/retention-cron.ts new file mode 100644 index 00000000..58ccb459 --- /dev/null +++ b/packages/dispatcher/src/retention-cron.ts @@ -0,0 +1,38 @@ +import type { Database } from "bun:sqlite"; +import { Bunqueue } from "bunqueue/client"; +import { RETENTION_CRON_INTERVAL_MS, runRetentionPass } from "./retention.ts"; + +/** + * Collaborators the retention cron needs, as an injectable seam so the cron + * wrapper unit-tests without a real engine. The daemon wires the live `db`; the + * pass logic ({@link runRetentionPass}) is tested directly against an in-memory db. + */ +export type RetentionCronDeps = { db: Database }; + +/** + * Stand up the retention cron as a bunqueue cron (mirrors `startRecommenderCron` + * / `startPoller` / `startWatchdog`): every `intervalMs` (default daily, + * {@link RETENTION_CRON_INTERVAL_MS}) it runs one {@link runRetentionPass}. Returns + * a stop function that tears the cron down. `runRetentionPass` records its own + * outcome (success or failure) in `retention_runs`; this wrapper additionally + * guards the pass so a thrown pass logs and never crashes the cron worker. + */ +export async function startRetentionCron( + deps: RetentionCronDeps, + intervalMs: number = RETENTION_CRON_INTERVAL_MS, +): Promise<() => Promise> { + const queue = new Bunqueue("middle-retention-cron", { + embedded: true, + processor: async () => { + try { + runRetentionPass(deps.db); + } catch (error) { + console.error(`[retention-cron] pass failed: ${(error as Error).message}`); + } + }, + }); + await queue.every("retention-cron-tick", intervalMs); + return async () => { + await queue.close(true); + }; +} diff --git a/packages/dispatcher/src/retention.ts b/packages/dispatcher/src/retention.ts new file mode 100644 index 00000000..3994eff7 --- /dev/null +++ b/packages/dispatcher/src/retention.ts @@ -0,0 +1,154 @@ +import type { Database } from "bun:sqlite"; + +/** + * Delete `events` rows older than this. The spec's retention window: operational + * activity is a 14-day rolling log, not a permanent record (GitHub is the system + * of record). Indexed by `idx_events_ts` so the scan is cheap. + */ +export const EVENTS_MAX_AGE_MS = 14 * 24 * 60 * 60 * 1000; + +/** + * Archive `completed` workflows older than this (by `updated_at` — the time the + * row reached its terminal state). Archival drops the workflow's events but + * preserves the row (final state + `meta_json` config snapshot), so history + * stays auditable while the events table stays bounded. + */ +export const WORKFLOWS_MAX_AGE_MS = 30 * 24 * 60 * 60 * 1000; + +/** Daily cadence for the retention cron — retention is not latency-sensitive. */ +export const RETENTION_CRON_INTERVAL_MS = 24 * 60 * 60 * 1000; + +/** What one retention pass pruned/archived. */ +export type RetentionResult = { eventsDeleted: number; workflowsArchived: number }; + +/** A recorded retention pass, as `mm doctor` reads it back. */ +export type RetentionRun = { + id: number; + ranAt: number; + eventsDeleted: number; + workflowsArchived: number; + ok: boolean; + detail: string | null; +}; + +/** Row counts + last-run status — the shape `mm doctor` reports. */ +export type RetentionStatus = { + rowCounts: { workflows: number; archivedWorkflows: number; events: number }; + lastRun: RetentionRun | null; +}; + +/** Overridable knobs; `now` and the cutoffs are injected so tests pin them. */ +export type RetentionOptions = { + now?: number; + eventsMaxAgeMs?: number; + workflowsMaxAgeMs?: number; +}; + +/** + * Record one retention pass in `retention_runs`. Called by {@link runRetentionPass} + * on both success (`ok=true`) and failure (`ok=false`, with `detail`), so + * `mm doctor` can surface a recent failure rather than silently showing the last + * good run. Best-effort: a write that itself throws is swallowed (the pass error + * is the one that matters), so this never masks the original failure. + */ +export function recordRetentionRun( + db: Database, + run: { ranAt: number; eventsDeleted: number; workflowsArchived: number; detail?: string | null }, +): void { + try { + db.run( + `INSERT INTO retention_runs (ran_at, events_deleted, workflows_archived, ok, detail) + VALUES (?, ?, ?, ?, ?)`, + // ok is keyed on the *presence* of a detail, not its truthiness: a failure + // whose Error.message is "" still records ok=0 (an empty `detail ? …` would + // mis-record it as a clean run). + [ + run.ranAt, + run.eventsDeleted, + run.workflowsArchived, + run.detail == null ? 1 : 0, + run.detail ?? null, + ], + ); + } catch (error) { + console.error(`[retention] failed to record run: ${(error as Error).message}`); + } +} + +/** The most recent retention pass, or null if retention has never run. */ +export function getLatestRetentionRun(db: Database): RetentionRun | null { + const row = db + .query( + `SELECT id, ran_at AS ranAt, events_deleted AS eventsDeleted, + workflows_archived AS workflowsArchived, ok, detail + FROM retention_runs ORDER BY ran_at DESC, id DESC LIMIT 1`, + ) + .get() as (Omit & { ok: number }) | null; + if (!row) return null; + return { ...row, ok: row.ok === 1 }; +} + +/** Row counts + last retention run — what `mm doctor` reports for the db. */ +export function collectRetentionStatus(db: Database): RetentionStatus { + const workflows = (db.query("SELECT count(*) AS c FROM workflows").get() as { c: number }).c; + const archivedWorkflows = ( + db.query("SELECT count(*) AS c FROM workflows WHERE archived_at IS NOT NULL").get() as { + c: number; + } + ).c; + const events = (db.query("SELECT count(*) AS c FROM events").get() as { c: number }).c; + return { + rowCounts: { workflows, archivedWorkflows, events }, + lastRun: getLatestRetentionRun(db), + }; +} + +/** + * Run one retention pass against middle's SQLite — and **only** SQLite; GitHub + * is the system of record and is never touched. Two cutoffs, both relative to + * `now`: + * + * 1. Delete every `events` row older than `eventsMaxAgeMs` (default 14d). + * 2. Archive every `completed` workflow whose `updated_at` is older than + * `workflowsMaxAgeMs` (default 30d) and that isn't already archived: drop its + * events and stamp `archived_at`. The row, its final state, and its + * `meta_json` config snapshot are preserved. `archived_at IS NULL` in the + * predicate makes the pass idempotent — re-running archives nothing new. + * + * The mutations run in one transaction. The run is recorded in `retention_runs` + * either way: on success with `ok=1`, on failure with `ok=0` and the error + * detail (then the error is rethrown for the cron to log). + */ +export function runRetentionPass(db: Database, opts: RetentionOptions = {}): RetentionResult { + const now = opts.now ?? Date.now(); + const eventsCutoff = now - (opts.eventsMaxAgeMs ?? EVENTS_MAX_AGE_MS); + const workflowsCutoff = now - (opts.workflowsMaxAgeMs ?? WORKFLOWS_MAX_AGE_MS); + + try { + const result = db.transaction(() => { + const eventsDeleted = db.run("DELETE FROM events WHERE ts < ?", [eventsCutoff]).changes; + // Drop events of the workflows about to be archived, then stamp them. Both + // statements share the same predicate so the counts stay consistent. + const archivePredicate = "state = 'completed' AND updated_at < ? AND archived_at IS NULL"; + db.run( + `DELETE FROM events WHERE workflow_id IN (SELECT id FROM workflows WHERE ${archivePredicate})`, + [workflowsCutoff], + ); + const workflowsArchived = db.run( + `UPDATE workflows SET archived_at = ? WHERE ${archivePredicate}`, + [now, workflowsCutoff], + ).changes; + return { eventsDeleted, workflowsArchived }; + })(); + recordRetentionRun(db, { ranAt: now, ...result }); + return result; + } catch (error) { + recordRetentionRun(db, { + ranAt: now, + eventsDeleted: 0, + workflowsArchived: 0, + detail: (error as Error).message, + }); + throw error; + } +} diff --git a/packages/dispatcher/test/db.test.ts b/packages/dispatcher/test/db.test.ts index 3f727a58..d87c4d28 100644 --- a/packages/dispatcher/test/db.test.ts +++ b/packages/dispatcher/test/db.test.ts @@ -22,6 +22,7 @@ const EXPECTED_TABLES = [ "events", "rate_limit_state", "repo_config", + "retention_runs", "schema_version", "waitfor_signals", "workflows", @@ -31,8 +32,10 @@ const EXPECTED_INDEXES = [ "idx_workflows_state", "idx_workflows_repo", "idx_workflows_heartbeat", + "idx_workflows_archived", "idx_events_workflow_ts", "idx_events_ts", + "idx_retention_runs_ran_at", ]; function names(db: Database, type: "table" | "index"): string[] { @@ -61,8 +64,8 @@ describe("runMigrations", () => { test("applies every migration and reports the latest version", () => { const db = openDb(dbPath); - expect(runMigrations(db)).toBe(6); - expect(currentSchemaVersion(db)).toBe(6); + expect(runMigrations(db)).toBe(7); + expect(currentSchemaVersion(db)).toBe(7); db.close(); }); @@ -85,8 +88,8 @@ describe("runMigrations", () => { test("is idempotent — running twice leaves version at the latest and does not throw", () => { const db = openDb(dbPath); runMigrations(db); - expect(runMigrations(db)).toBe(6); - expect(currentSchemaVersion(db)).toBe(6); + expect(runMigrations(db)).toBe(7); + expect(currentSchemaVersion(db)).toBe(7); db.close(); }); @@ -159,8 +162,8 @@ describe("runMigrations", () => { ); db.run(`INSERT INTO events (workflow_id, ts, type) VALUES ('w1', 2, 'session.started')`); - // Now apply the remaining migrations (003 rebuild, then 004, 005, 006) over the seeded data. - expect(runMigrations(db, realDir)).toBe(6); + // Now apply the remaining migrations (003 rebuild, then 004, 005, 006, 007) over the seeded data. + expect(runMigrations(db, realDir)).toBe(7); // The row survived the rebuild... expect( @@ -183,7 +186,7 @@ describe("runMigrations", () => { describe("openAndMigrate", () => { test("opens, migrates, and returns a ready database", () => { const db = openAndMigrate(dbPath); - expect(currentSchemaVersion(db)).toBe(6); + expect(currentSchemaVersion(db)).toBe(7); db.close(); }); }); diff --git a/packages/dispatcher/test/retention.test.ts b/packages/dispatcher/test/retention.test.ts new file mode 100644 index 00000000..599ce79d --- /dev/null +++ b/packages/dispatcher/test/retention.test.ts @@ -0,0 +1,179 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import type { Database } from "bun:sqlite"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { openAndMigrate } from "../src/db.ts"; +import { + collectRetentionStatus, + EVENTS_MAX_AGE_MS, + getLatestRetentionRun, + recordRetentionRun, + runRetentionPass, + WORKFLOWS_MAX_AGE_MS, +} from "../src/retention.ts"; + +const NOW = 1_900_000_000_000; // fixed clock; all ages are relative to this +const DAY = 24 * 60 * 60 * 1000; + +let dir: string; +let db: Database; + +beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "middle-retention-")); + db = openAndMigrate(join(dir, "db.sqlite3")); +}); + +afterEach(() => { + db.close(); + rmSync(dir, { recursive: true, force: true }); +}); + +function insertWorkflow( + id: string, + opts: { state?: string; updatedAt?: number; meta?: string } = {}, +): void { + db.run( + `INSERT INTO workflows (id, kind, repo, adapter, state, created_at, updated_at, meta_json) + VALUES (?, 'implementation', 'o/r', 'claude', ?, ?, ?, ?)`, + [id, opts.state ?? "completed", 1, opts.updatedAt ?? NOW, opts.meta ?? null], + ); +} + +function insertEvent(workflowId: string, ts: number): void { + db.run(`INSERT INTO events (workflow_id, ts, type) VALUES (?, ?, 'session.started')`, [ + workflowId, + ts, + ]); +} + +function eventCount(): number { + return (db.query("SELECT count(*) AS c FROM events").get() as { c: number }).c; +} + +describe("runRetentionPass — events cutoff (14d)", () => { + test("deletes events older than 14 days, keeps newer ones", () => { + insertWorkflow("w1", { state: "running", updatedAt: NOW }); + insertEvent("w1", NOW - EVENTS_MAX_AGE_MS - 1); // just over the line → deleted + insertEvent("w1", NOW - EVENTS_MAX_AGE_MS + DAY); // inside the window → kept + insertEvent("w1", NOW); // fresh → kept + + const result = runRetentionPass(db, { now: NOW }); + + expect(result.eventsDeleted).toBe(1); + expect(eventCount()).toBe(2); + }); + + test("an event exactly at the cutoff age is kept (strict `< cutoff`)", () => { + insertWorkflow("w1", { state: "running" }); + insertEvent("w1", NOW - EVENTS_MAX_AGE_MS); // ts === cutoff → kept + const result = runRetentionPass(db, { now: NOW }); + expect(result.eventsDeleted).toBe(0); + expect(eventCount()).toBe(1); + }); +}); + +describe("runRetentionPass — workflow archival (30d, completed only)", () => { + test("archives completed workflows older than 30 days; drops their events, preserves the row", () => { + insertWorkflow("old", { + state: "completed", + updatedAt: NOW - WORKFLOWS_MAX_AGE_MS - 1, + meta: '{"x":1}', + }); + insertEvent("old", NOW); // fresh event, but its workflow is archived → dropped + + const result = runRetentionPass(db, { now: NOW }); + + expect(result.workflowsArchived).toBe(1); + // The row survives with its state + config snapshot… + const row = db + .query("SELECT state, meta_json AS meta, archived_at AS a FROM workflows WHERE id='old'") + .get() as { + state: string; + meta: string; + a: number; + }; + expect(row.state).toBe("completed"); + expect(row.meta).toBe('{"x":1}'); + expect(row.a).toBe(NOW); // stamped + // …but its events are gone. + expect(eventCount()).toBe(0); + }); + + test("does not archive completed workflows inside the 30-day window", () => { + insertWorkflow("recent", { state: "completed", updatedAt: NOW - WORKFLOWS_MAX_AGE_MS + DAY }); + const result = runRetentionPass(db, { now: NOW }); + expect(result.workflowsArchived).toBe(0); + expect(db.query("SELECT archived_at FROM workflows WHERE id='recent'").get()).toEqual({ + archived_at: null, + }); + }); + + test("does not archive old non-completed workflows (failed/running/etc.)", () => { + insertWorkflow("failed", { state: "failed", updatedAt: NOW - WORKFLOWS_MAX_AGE_MS - DAY }); + insertWorkflow("running", { state: "running", updatedAt: NOW - WORKFLOWS_MAX_AGE_MS - DAY }); + const result = runRetentionPass(db, { now: NOW }); + expect(result.workflowsArchived).toBe(0); + }); + + test("is idempotent — a second pass archives nothing new", () => { + insertWorkflow("old", { state: "completed", updatedAt: NOW - WORKFLOWS_MAX_AGE_MS - DAY }); + expect(runRetentionPass(db, { now: NOW }).workflowsArchived).toBe(1); + expect(runRetentionPass(db, { now: NOW }).workflowsArchived).toBe(0); + }); +}); + +describe("retention_runs recording", () => { + test("records each pass (even a no-op) with ok=true", () => { + runRetentionPass(db, { now: NOW }); + const last = getLatestRetentionRun(db); + expect(last).not.toBeNull(); + expect(last!.ok).toBe(true); + expect(last!.ranAt).toBe(NOW); + expect(last!.eventsDeleted).toBe(0); + expect(last!.workflowsArchived).toBe(0); + }); + + test("recordRetentionRun with a detail marks ok=false", () => { + recordRetentionRun(db, { ranAt: NOW, eventsDeleted: 0, workflowsArchived: 0, detail: "boom" }); + const last = getLatestRetentionRun(db); + expect(last!.ok).toBe(false); + expect(last!.detail).toBe("boom"); + }); + + test("an empty-string detail still marks ok=false (failure presence, not truthiness)", () => { + // A thrown Error whose .message is "" must not be recorded as a clean run. + recordRetentionRun(db, { ranAt: NOW, eventsDeleted: 0, workflowsArchived: 0, detail: "" }); + const last = getLatestRetentionRun(db); + expect(last!.ok).toBe(false); + expect(last!.detail).toBe(""); + }); + + test("getLatestRetentionRun returns the most recent by ran_at", () => { + recordRetentionRun(db, { ranAt: NOW - DAY, eventsDeleted: 1, workflowsArchived: 0 }); + recordRetentionRun(db, { ranAt: NOW, eventsDeleted: 5, workflowsArchived: 2 }); + const last = getLatestRetentionRun(db); + expect(last!.ranAt).toBe(NOW); + expect(last!.eventsDeleted).toBe(5); + expect(last!.workflowsArchived).toBe(2); + }); +}); + +describe("collectRetentionStatus", () => { + test("reports row counts (incl. archived) and the last run", () => { + insertWorkflow("a", { state: "completed", updatedAt: NOW - WORKFLOWS_MAX_AGE_MS - DAY }); + insertWorkflow("b", { state: "running" }); + insertEvent("b", NOW); + runRetentionPass(db, { now: NOW }); // archives 'a' + + const status = collectRetentionStatus(db); + expect(status.rowCounts.workflows).toBe(2); + expect(status.rowCounts.archivedWorkflows).toBe(1); + expect(status.rowCounts.events).toBe(1); // b's event survived + expect(status.lastRun?.ok).toBe(true); + }); + + test("lastRun is null before any retention has run", () => { + expect(collectRetentionStatus(db).lastRun).toBeNull(); + }); +}); diff --git a/planning/issues/64/decisions.md b/planning/issues/64/decisions.md new file mode 100644 index 00000000..c23556b6 --- /dev/null +++ b/planning/issues/64/decisions.md @@ -0,0 +1,37 @@ +# Decisions — Issue #64 (Operator polish) + +## Implement sub-issues in dependency order, not numeric order +**File(s):** whole workstream +**Date:** 2026-05-29 + +**Decision:** Land #66 (retention) before #65 (doctor), then #67, then #68. +**Why:** Doctor's "recent retention-run status" reporting reads the `retention_runs` table, which retention creates. Docs (#68) go last so they describe shipped behavior, not the spec's intentions. Numeric order would force doctor to read a table that doesn't exist yet or stub it. +**Evidence:** #65 acceptance "reports SQLite row counts and recent retention-run status"; #66 acceptance "Retention runs are recorded so `mm doctor` can report recent retention status". + +## doctor: dispatcher-reachable is warn-not-running, fail-only-when-wedged +**File(s):** `packages/cli/src/commands/doctor.ts` (`checkDispatcher`) +**Date:** 2026-05-29 + +**Decision:** `/health` reachable → pass; live pidfile but unreachable `/health` → fail; no/dead pidfile → warn. +**Why:** Operators routinely run `mm doctor` before `mm start`; a not-running dispatcher is normal, not an error (warn keeps exit 0). A *wedged* daemon (pid alive, port dead) is a real failure worth a non-zero exit. Distinguishing the two needs the pidfile, not just the probe. + +## doctor: state-issue check is a self-test against the canonical fixture +**File(s):** `packages/cli/src/checks/state-issue.ts` +**Date:** 2026-05-29 + +**Decision:** Re-validate the parser by parse→render(byte-identical)→validate of `packages/state-issue`'s canonical fixture, using the fixture's own adapter set (`claude`, `codex`) — not the operator's configured adapters. +**Why:** The check verifies the *machinery* conforms to `schemas/state-issue.v1.md`, independent of whether a given operator configured codex. Using operator adapters would make a conforming fixture fail validate (rule 5) on a single-adapter install. Paths resolve from the module's location (like the module-index/skills checks) so it inspects middle's own source tree. + +## doctor: retention/db check degrades, never spuriously fails +**File(s):** `packages/cli/src/commands/doctor.ts` (`checkDatabase`, `summarizeRetention`) +**Date:** 2026-05-29 + +**Decision:** No db file → warn; db below retention schema (< v6) → warn; unreadable db → fail; a *failed* last retention run → warn (surfaced as `FAILED`). +**Why:** Doctor must be safe to run any time. Only genuine corruption (can't open) is a hard fail; the rest are degraded-but-functional states. `existsSync` guards before `openDb` (which has `create:true`) so doctor never creates the db as a side effect. + +## scripts pass paths via env, not `bun -e` argv +**File(s):** `scripts/backup.sh` (`snapshot_db`) +**Date:** 2026-05-29 + +**Decision:** The backup DB-snapshot one-liner reads source/target from `MIDDLE_SNAP_SRC`/`MIDDLE_SNAP_DST` env vars, not `process.argv`. +**Why:** `bun -e '' a b` does NOT expose `a`/`b` as `process.argv[2+]` — `new Database(undefined)` silently opens an *in-memory* db, so an argv-based snapshot writes an empty file with no error. Env vars are the reliable channel. The snapshot uses `VACUUM INTO` on a read-write handle (readonly → SQLITE_CANTOPEN), which is a consistent single-file snapshot even against a live WAL — so backup works without stopping the dispatcher. Restore, which overwrites the db, refuses while the dispatcher is up. diff --git a/planning/issues/64/plan.md b/planning/issues/64/plan.md new file mode 100644 index 00000000..bf2c497f --- /dev/null +++ b/planning/issues/64/plan.md @@ -0,0 +1,37 @@ +# Issue #64: Operator polish (Epic) + +**Link:** https://github.com/thejustinwalsh/middle/issues/64 +**Branch:** middle-issue-64 + +## Goal +Ship Phase 11 operator polish so a new user can clone middle, `bun install`, `mm start`, `mm init `, and reach a working dispatch within 5 minutes: a real `mm doctor` health check, retention crons that bound operational state, backup/reset-db scripts, and the README + `docs/` set. + +## Approach +- The Epic's four open sub-issues are the phases. One branch, one PR, gates between phases. +- **Implementation order follows the dependency edge, not the issue numbers.** `mm doctor` (#65) must report "recent retention-run status", which only exists once the `retention_runs` table does. So **#66 (retention) lands first**, then #65 (doctor), then #67 (scripts), then #68 (docs — written last so they describe shipped behavior, not intentions). +- New checks and the retention pass are built as **pure functions with injected seams** (db handle, config, `fetch`, fixture body), matching the existing `recommender-cron` / doctor style, so each is unit-testable without a live dispatcher. +- Retention touches **only** middle's SQLite. GitHub is the system of record — never backed up, never pruned by middle. + +## Phases (one per sub-issue) +1. **#66 — Retention crons.** Migration `006`: add `workflows.archived_at` + a `retention_runs` table. `retention.ts` (`runRetentionPass`) + `retention-cron.ts` (`startRetentionCron`), mirroring `recommender-cron.ts`. Wire into `main.ts` startup + shutdown. Daily cron: delete `events` older than 14d; archive completed `workflows` older than 30d (drop their events, preserve row + `meta_json`/config snapshot + final state); record each run in `retention_runs`. Tests cover both cutoffs. +2. **#65 — `mm doctor` full health check.** Extend `doctor.ts` with: config-files-parse check (`loadConfig`), dispatcher-reachable check (`GET /health`), state-issue parser re-validation (parse → render byte-identical round-trip + `validate()` of the canonical fixture, with schema-version assertion against `schemas/state-issue.v1.md`), and SQLite row counts + most-recent retention-run status. Any check failing → non-zero exit. New pure helpers get unit tests. +3. **#67 — Backup + reset-db scripts.** `scripts/backup.sh` (SQLite DB via `.backup` + config files → timestamped restorable archive) and `scripts/reset-db.sh` (nuke `~/.middle/db.sqlite3` + WAL/SHM, never touch GitHub). Both: clear output, confirm-before-destroy, documented. Restore yields a working dispatcher. +4. **#68 — README + docs/.** Refresh README quickstart against shipped behavior; author `docs/architecture.md`, `docs/adapters.md`, `docs/bootstrap.md`, `docs/skill-enforcement.md`, `docs/operator.md` (`docs/dogfooding.md` already exists). Verify the 5-minute quickstart end to end. Follow the `documenting-the-repo` skill (Diátaxis, repo voice). + +## Files likely to change +- `packages/dispatcher/src/db/migrations/006_retention.sql` — new: `archived_at` column + `retention_runs` table +- `packages/dispatcher/src/retention.ts`, `retention-cron.ts` — new: pass + cron +- `packages/dispatcher/src/main.ts` — register/stop the retention cron +- `packages/dispatcher/test/retention*.test.ts` — cutoff tests +- `packages/cli/src/commands/doctor.ts` — new checks (config, dispatcher, state-issue, db/retention) +- `packages/cli/test/doctor.test.ts` — unit tests for new pure helpers +- `scripts/backup.sh`, `scripts/reset-db.sh` — new +- `README.md`, `docs/architecture.md|adapters.md|bootstrap.md|skill-enforcement.md|operator.md` — docs + +## Out of scope +- The skill-sync pre-commit hook (delivered with the Phase 3 skills task). +- Backing up any GitHub data — GitHub is the system of record. +- CodexAdapter behavior (Phase 10, #60 — closed); docs describe it as roadmap where not shipped. + +## Open questions +- None blocking. Retention defaults (14d events / 30d workflows) and daily cadence are fixed by the spec. diff --git a/scripts/backup.sh b/scripts/backup.sh new file mode 100755 index 00000000..4113cf5e --- /dev/null +++ b/scripts/backup.sh @@ -0,0 +1,152 @@ +#!/usr/bin/env bash +# scripts/backup.sh — back up and restore middle's operational state. +# +# Captures middle's SQLite database and config into a single restorable +# .tar.gz, and restores from one. This is middle's OWN state only — its +# dispatch bookkeeping. It does NOT back up GitHub: issues, sub-issues, and PRs +# are the system of record and live on GitHub, not here. +# +# The database snapshot uses SQLite's `VACUUM INTO`, which produces a single +# consistent file even while the dispatcher is running against the live WAL — +# so you can back up without stopping `mm start`. (Restore, by contrast, +# requires the dispatcher stopped — see below.) +# +# Usage: +# scripts/backup.sh [--home DIR] [--out DIR] # create a backup archive +# scripts/backup.sh --restore ARCHIVE [--home DIR] [--yes] +# +# Options: +# --home DIR middle home dir (default: $MIDDLE_HOME, else ~/.middle) +# --db PATH database to back up (default: the configured db_path, else +# /db.sqlite3) +# --out DIR where to write the archive (default: current directory) +# --restore ARCH restore from archive ARCH instead of backing up +# --yes skip the restore confirmation prompt +# -h, --help show this help +# +# Restore overwrites the db (and config) in --home. It refuses to run while the +# dispatcher is up — stop it first with `mm stop`. After restoring, start the +# dispatcher (`mm start`); it reopens the restored db and migrates if needed. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +HOME_DIR="${MIDDLE_HOME:-$HOME/.middle}" +HOME_EXPLICIT=0 +DB_OVERRIDE="" +OUT_DIR="." +RESTORE_ARCHIVE="" +ASSUME_YES=0 + +usage() { sed -n '2,29p' "$0" | sed 's/^# \{0,1\}//'; exit "${1:-0}"; } + +while [ $# -gt 0 ]; do + case "$1" in + --home) HOME_DIR="${2:?--home needs a directory}"; HOME_EXPLICIT=1; shift 2 ;; + --db) DB_OVERRIDE="${2:?--db needs a path}"; shift 2 ;; + --out) OUT_DIR="${2:?--out needs a directory}"; shift 2 ;; + --restore) RESTORE_ARCHIVE="${2:?--restore needs an archive path}"; shift 2 ;; + --yes|-y) ASSUME_YES=1; shift ;; + -h|--help) usage 0 ;; + *) echo "backup.sh: unknown argument: $1" >&2; usage 1 ;; + esac +done + +# Resolve the db path so a relocated `db_path` isn't silently missed. Precedence: +# 1. --db PATH (explicit) +# 2. --home DIR → /db.sqlite3 (the operator/test pinned a home) +# 3. the configured global.dbPath (honors MIDDLE_CONFIG + tilde), via the same +# loader mm doctor uses; bun runs from the middle checkout so @middle/core resolves +# 4. /db.sqlite3 +resolve_db_path() { + if [ -n "$DB_OVERRIDE" ]; then printf '%s' "$DB_OVERRIDE"; return; fi + if [ "$HOME_EXPLICIT" -eq 1 ]; then printf '%s' "$HOME_DIR/db.sqlite3"; return; fi + local p + p="$(cd "$SCRIPT_DIR/.." && bun -e 'import{loadConfig}from"@middle/core";try{const d=loadConfig({globalPath:process.env.MIDDLE_CONFIG}).global.dbPath;if(d)process.stdout.write(d)}catch{}' 2>/dev/null)" || p="" + if [ -n "$p" ]; then printf '%s' "$p"; else printf '%s' "$HOME_DIR/db.sqlite3"; fi +} + +DB_PATH="$(resolve_db_path)" +CONFIG_PATH="${MIDDLE_CONFIG:-$HOME_DIR/config.toml}" +PID_FILE="$HOME_DIR/dispatcher.pid" + +# Is the recorded dispatcher pid a live process? +dispatcher_running() { + [ -f "$PID_FILE" ] || return 1 + local pid + pid="$(tr -d '[:space:]' < "$PID_FILE")" + case "$pid" in ''|*[!0-9]*) return 1 ;; esac + kill -0 "$pid" 2>/dev/null +} + +# Snapshot a live SQLite db to a clean single file via VACUUM INTO (consistent +# even with the WAL active — it reads a snapshot of the source without modifying +# it). bun is middle's runtime, so it is always present. The connection is +# read-write because VACUUM INTO needs a writable handle even though it only +# reads the source; concurrent with a running dispatcher this is safe under WAL. +snapshot_db() { + # Paths go through the environment, not argv: `bun -e` does not expose trailing + # positional args as process.argv, so argv-passing silently opens an in-memory db. + MIDDLE_SNAP_SRC="$1" MIDDLE_SNAP_DST="$2" bun -e ' + import { Database } from "bun:sqlite"; + const db = new Database(process.env.MIDDLE_SNAP_SRC); + db.exec(`VACUUM INTO ${JSON.stringify(process.env.MIDDLE_SNAP_DST)}`); + db.close(); + ' +} + +if [ -n "$RESTORE_ARCHIVE" ]; then + # ---- restore ---- + [ -f "$RESTORE_ARCHIVE" ] || { echo "backup.sh: archive not found: $RESTORE_ARCHIVE" >&2; exit 1; } + if dispatcher_running; then + echo "backup.sh: dispatcher is running — stop it first with \`mm stop\`, then restore." >&2 + exit 1 + fi + echo "About to restore middle state into: $HOME_DIR" + echo " from archive: $RESTORE_ARCHIVE" + echo " this overwrites $DB_PATH (and config.toml if present in the archive)." + if [ "$ASSUME_YES" -ne 1 ]; then + printf "Proceed? [y/N] " + read -r reply + case "$reply" in y|Y|yes|YES) ;; *) echo "Aborted."; exit 1 ;; esac + fi + mkdir -p "$HOME_DIR" + tmp="$(mktemp -d)" + trap 'rm -rf "$tmp"' EXIT + tar -xzf "$RESTORE_ARCHIVE" -C "$tmp" + [ -f "$tmp/db.sqlite3" ] || { echo "backup.sh: archive has no db.sqlite3 — not a middle backup?" >&2; exit 1; } + # Drop stale WAL/SHM so the restored db is the single source of truth. + # Create each destination's parent dir first: a relocated --db / MIDDLE_CONFIG + # can point under a directory that doesn't exist yet, and bare `cp` would abort. + mkdir -p "$(dirname "$DB_PATH")" + rm -f "$DB_PATH-wal" "$DB_PATH-shm" + cp "$tmp/db.sqlite3" "$DB_PATH" + if [ -f "$tmp/config.toml" ]; then + mkdir -p "$(dirname "$CONFIG_PATH")" + cp "$tmp/config.toml" "$CONFIG_PATH" + echo " restored: config.toml" + fi + echo " restored: db.sqlite3" + echo "Done. Start the dispatcher with \`mm start\` (it will migrate the db if needed)." + exit 0 +fi + +# ---- backup ---- +[ -f "$DB_PATH" ] || { echo "backup.sh: no database at $DB_PATH — nothing to back up." >&2; exit 1; } +mkdir -p "$OUT_DIR" +stamp="$(date +%Y%m%d-%H%M%S)" +archive="$OUT_DIR/middle-backup-$stamp.tar.gz" +tmp="$(mktemp -d)" +trap 'rm -rf "$tmp"' EXIT + +echo "Backing up middle state from: $HOME_DIR" +snapshot_db "$DB_PATH" "$tmp/db.sqlite3" +echo " captured: db.sqlite3 (consistent snapshot)" +if [ -f "$CONFIG_PATH" ]; then + cp "$CONFIG_PATH" "$tmp/config.toml" + echo " captured: config.toml" +else + echo " (no config.toml — using built-in defaults; nothing to capture)" +fi +tar -czf "$archive" -C "$tmp" . +echo "Wrote: $archive" +echo "Restore with: scripts/backup.sh --restore \"$archive\"" diff --git a/scripts/reset-db.sh b/scripts/reset-db.sh new file mode 100755 index 00000000..e794264b --- /dev/null +++ b/scripts/reset-db.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +# scripts/reset-db.sh — nuke middle's SQLite database. +# +# Deletes middle's operational database (db.sqlite3 + its -wal/-shm sidecars). +# The dispatcher recreates and migrates a fresh, empty db on the next +# `mm start`. This wipes ONLY middle's local bookkeeping — it does NOT touch +# GitHub. Issues, sub-issues, and PRs are the system of record and live on +# GitHub; a reset loses in-flight workflow rows and the event log, not work. +# +# Usage: +# scripts/reset-db.sh [--home DIR] [--db PATH] [--yes] +# +# Options: +# --home DIR middle home dir (default: $MIDDLE_HOME, else ~/.middle) +# --db PATH database to reset (default: the configured db_path, else +# /db.sqlite3) +# --yes, -y skip the confirmation prompt +# -h, --help show this help +# +# Safety: refuses to run while the dispatcher is up (stop it with `mm stop`), +# lists exactly what it will delete, and confirms before deleting unless --yes. +# Back up first with: scripts/backup.sh +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +HOME_DIR="${MIDDLE_HOME:-$HOME/.middle}" +HOME_EXPLICIT=0 +DB_OVERRIDE="" +ASSUME_YES=0 + +usage() { sed -n '2,22p' "$0" | sed 's/^# \{0,1\}//'; exit "${1:-0}"; } + +while [ $# -gt 0 ]; do + case "$1" in + --home) HOME_DIR="${2:?--home needs a directory}"; HOME_EXPLICIT=1; shift 2 ;; + --db) DB_OVERRIDE="${2:?--db needs a path}"; shift 2 ;; + --yes|-y) ASSUME_YES=1; shift ;; + -h|--help) usage 0 ;; + *) echo "reset-db.sh: unknown argument: $1" >&2; usage 1 ;; + esac +done + +# Resolve the db path (same precedence as backup.sh) so a relocated `db_path` +# isn't silently skipped: --db, else an explicit --home, else the configured +# global.dbPath (via the loader mm doctor uses), else /db.sqlite3. +resolve_db_path() { + if [ -n "$DB_OVERRIDE" ]; then printf '%s' "$DB_OVERRIDE"; return; fi + if [ "$HOME_EXPLICIT" -eq 1 ]; then printf '%s' "$HOME_DIR/db.sqlite3"; return; fi + local p + p="$(cd "$SCRIPT_DIR/.." && bun -e 'import{loadConfig}from"@middle/core";try{const d=loadConfig({globalPath:process.env.MIDDLE_CONFIG}).global.dbPath;if(d)process.stdout.write(d)}catch{}' 2>/dev/null)" || p="" + if [ -n "$p" ]; then printf '%s' "$p"; else printf '%s' "$HOME_DIR/db.sqlite3"; fi +} + +DB_PATH="$(resolve_db_path)" +PID_FILE="$HOME_DIR/dispatcher.pid" + +dispatcher_running() { + [ -f "$PID_FILE" ] || return 1 + local pid + pid="$(tr -d '[:space:]' < "$PID_FILE")" + case "$pid" in ''|*[!0-9]*) return 1 ;; esac + kill -0 "$pid" 2>/dev/null +} + +if dispatcher_running; then + echo "reset-db.sh: dispatcher is running — stop it first with \`mm stop\`, then reset." >&2 + exit 1 +fi + +# Collect the files that actually exist, so the report is honest. +targets=() +for f in "$DB_PATH" "$DB_PATH-wal" "$DB_PATH-shm"; do + [ -f "$f" ] && targets+=("$f") +done + +if [ "${#targets[@]}" -eq 0 ]; then + echo "reset-db.sh: no database at $DB_PATH — nothing to reset." + exit 0 +fi + +echo "This will permanently delete middle's local database (GitHub is NOT touched):" +for f in "${targets[@]}"; do + size="$(du -h "$f" 2>/dev/null | cut -f1)" + echo " - $f (${size:-?})" +done +echo "The dispatcher will recreate an empty, migrated db on the next \`mm start\`." + +if [ "$ASSUME_YES" -ne 1 ]; then + printf "Proceed? [y/N] " + read -r reply + case "$reply" in y|Y|yes|YES) ;; *) echo "Aborted."; exit 1 ;; esac +fi + +rm -f "${targets[@]}" +echo "Done — deleted ${#targets[@]} file(s). GitHub was not touched."