diff --git a/.claude/orchestrator.json b/.claude/orchestrator.json index 5f36871fe..23d5be373 100644 --- a/.claude/orchestrator.json +++ b/.claude/orchestrator.json @@ -3,11 +3,24 @@ "workers": { "claude": { "command": "claude", - "args": ["--permission-mode", "bypassPermissions"], + "args": [ + "--permission-mode", + "bypassPermissions" + ], "models": { - "default": { "model": "opus" }, - "cheap": { "model": "sonnet" }, - "deep": { "model": "opus", "args": ["--effort", "max"] } + "default": { + "model": "opus" + }, + "cheap": { + "model": "sonnet" + }, + "deep": { + "model": "opus", + "args": [ + "--effort", + "max" + ] + } }, "interactive": true, "automationBudget": { @@ -19,27 +32,40 @@ "cheap": 50000, "deep": 250000 } - }, - "notes": "The prompt is the ticket body (D2). Model routing: the default is opus, tier:cheap selects sonnet, and tier:deep selects opus with --effort max so hard work gets a distinct invocation without changing the no-label default. The legacy worker:sonnet label is rejected loudly with remediation to use tier:cheap. The mode MUST be bypassPermissions, never acceptEdits: acceptEdits auto-approves file writes only, so every shell command still prompts, and a worker with nobody at the keyboard is stuck. Measured on the ORB-75 Phase 7 run, where git switch, dotnet build/test/format, gh and orca were all denied and the worker delivered files with zero gates run, zero commit and zero PR. There is deliberately no -p here either: headless mode is invisible to Orca, so the worktree card shows no Agents row and clicking it reveals a bare shell. The worker runs as a TUI, which means orca terminal wait must use --for tui-idle, never --for exit. One more thing a TUI worker needs answered for it: a brand-new worktree is a folder Claude Code has never seen, so its first screen is the workspace-trust gate (Quick safety check: Is this a project you created or one you trust? with 1. Yes, I trust this folder). Measured on the 2026-07-24 ORB-75 launch, where orca terminal wait --for tui-idle returned satisfied: false with blockedReason codex-trust-workspace and the worker sat on that screen with nobody at the keyboard. Same failure class as the permission mode above: the fix is to send 1 and Enter and wait again, which tools/launch-worker.mjs now does on every launch. Two more things measured on that same run, both about talking to a live worker. First, never orca terminal send to a worker that is not tui-idle: a send issued mid-turn is not delivered as a user turn at all. In the worker's own session transcript it appears only as four type: queue-operation records, never as a type: user entry, and the running turn was cut short on the mid-flow sentence Now the auth-side signup tests., leaving 14 modified and 7 untracked files with zero commits, zero gates and no PR. Everything a worker needs belongs in the prompt FILE at launch; mid-run information is appended to that file and pointed at, which is what tools/nudge-worker.mjs enforces by refusing to send while the terminal is busy. Second, tui-idle means stopped, not done: the same run read satisfied: true as completion when the worker had abandoned the task with nothing committed. Idle is a trigger to run tools/worker-status.mjs, which derives the verdict from commits, push, PR and Linear state, never a report of success." + } }, "codex": { "command": "codex", - "args": ["-c", "windows.sandbox=\"unelevated\"", "--dangerously-bypass-approvals-and-sandbox"], + "args": [ + "exec", + "-c", + "windows.sandbox=\"unelevated\"", + "--dangerously-bypass-approvals-and-sandbox" + ], "models": { "default": { "model": "gpt-5.6-terra", - "args": ["-c", "model_reasoning_effort=\"medium\""] + "args": [ + "-c", + "model_reasoning_effort=\"medium\"" + ] }, "cheap": { "model": "gpt-5.6-luna", - "args": ["-c", "model_reasoning_effort=\"low\""] + "args": [ + "-c", + "model_reasoning_effort=\"low\"" + ] }, "deep": { "model": "gpt-5.6-sol", - "args": ["-c", "model_reasoning_effort=\"high\""] + "args": [ + "-c", + "model_reasoning_effort=\"high\"" + ] } }, - "interactive": true, + "interactive": false, "automationBudget": { "tier": "routine", "tokenBudget": 1000000, @@ -49,11 +75,11 @@ "cheap": 50000, "deep": 250000 } - }, - "notes": "Model routing: the default is gpt-5.6-terra at medium reasoning, tier:cheap selects gpt-5.6-luna at low reasoning, and tier:deep selects gpt-5.6-sol at high reasoning. The default is based on the 23 pull requests in the 2026-07-28 unattended run: every pull request ran on Sol at high, with a mean of 2.0 CHANGES_REQUESTED rounds, a median of 1, and worst results of 13, 7 and 5. Sol at high therefore averaged exactly the two rounds that had condemned Terra at medium, so the falsifier fired and the decision was reversed by ADR. The legacy worker:sonnet label is rejected loudly with remediation to use tier:cheap. Requires a paid ChatGPT plan plus codex login; a headless session can start that itself with codex login --device-auth, which prints a URL and a one-time code for the account owner to enter. CODEX_HOME DECIDES WHETHER A WORKER IS LOGGED IN, and it is the first thing to check before believing any auth verdict: Orca redirects codex's home for the terminals it spawns, so on this machine the real credential is C:\\Users\\thoma\\AppData\\Roaming\\orca\\codex-runtime-home\\home\\auth.json, not the default ~/.codex/auth.json. Measured 2026-07-27: a shell that had lost CODEX_HOME reported 'Not logged in' and codex doctor reported 'no Codex credentials were found', while the very same CLI in an Orca terminal in the same worktree reported 'Logged in using ChatGPT' minutes earlier. For an unattended worker those two states are indistinguishable from having no plan at all, and the worker dies on a login screen nobody is there to answer. Two consequences. Never diagnose codex auth from an ad hoc shell without printing CODEX_HOME first. And a codex worker that never reaches tui-idle should be suspected of sitting on the login splash: that splash ANIMATES, so the repaint check in tools/launch-worker.mjs now turns it into a loud exit 1 rather than a prompt pointer typed into a sign-in screen. Reads AGENTS.md natively. The invocation is bare codex with no subcommand, which is what codex --help means by 'If no subcommand is specified, options will be forwarded to the interactive CLI'. codex exec is the NON-interactive subcommand and lands in the same unsupervisable place as claude -p, so tools/launch-worker.mjs still refuses it (per-engine headless tokens: exec and its alias e for codex, -p and --print for claude, because codex's -p is --profile, a legitimate interactive flag). Approval and sandbox policy is --dangerously-bypass-approvals-and-sandbox, not the equivalent pair -a never --sandbox danger-full-access, and never --full-auto: --full-auto is -a on-request --sandbox workspace-write, and on-request lets the MODEL decide when to ask a human who is not at the keyboard. The single flag beats the pair because the pair has a half-state: an edit that drops -a never while keeping the sandbox flag silently restores approval prompts to a worker nobody is watching, which is the exact failure this entry exists to prevent. The containment story is the disposable Orca worktree, the same one that justifies claude's bypassPermissions. Everything below was measured on 2026-07-27 against codex-cli 0.145.0 on Windows 11. -c windows.sandbox=\"unelevated\" is load-bearing: Codex's default Windows sandbox is elevated, its setup needs Administrator rights, and the first-run TUI paints 'Set up default sandbox (requires Administrator permissions)' then sits on 'Setting up sandbox... Input disabled until setup completes' forever in a PTY that has no desktop to raise UAC on. Measured still spinning after 2.5 minutes, with orca terminal wait reporting satisfied: true throughout, a false idle. The unelevated fallback (restricted tokens and ACLs, learn.chatgpt.com/docs/windows/windows-sandbox) needs no elevation, and the worker never executes inside it anyway because the bypass flag runs commands unsandboxed. cwd needs no -C: orca terminal create --worktree already starts the TUI in the worktree. Supervision, all measured: a real TUI opens, so the Orca card carries an Agents row and orca terminal wait --for tui-idle has a state to observe. The first screen in a brand-new worktree is codex's own trust gate, 'Do you trust the contents of this directory?', which normally reports blockedReason codex-interactive-prompt. Orca 1.4.156 was also measured retaining codex-trust-workspace on an idle codex terminal that never saw a trust gate. WHY: PR #629 owner adjudication makes that live capture authoritative over the older one-reason-per-engine mapping. It is a preselected list whose own instruction reads 'Press enter to continue', so the answer is Enter ALONE; the 1 then Enter that answers Claude Code's gate left the codex process exited (-1). launch-worker.mjs therefore keeps the trust answer per engine. Never blind-Enter an unrecognised codex screen: a stray Enter meant for one prompt was measured confirming the NEXT one, which is how the Administrator sandbox setup above got selected. Everything the claude entry says about talking to a live worker holds here unchanged: never send to a terminal that is not tui-idle, and tui-idle means stopped, not done, so it is the trigger to run tools/worker-status.mjs, never a report of success." + } } }, "maxParallelWorktrees": 4, + "maxSlicesPerWorker": 3, "attemptsBeforeRewrite": 2, "linear": { "team": "ORB", diff --git a/.claude/skills/orchestrate/SKILL.md b/.claude/skills/orchestrate/SKILL.md index 556d8547a..f21d52f2b 100644 --- a/.claude/skills/orchestrate/SKILL.md +++ b/.claude/skills/orchestrate/SKILL.md @@ -302,26 +302,16 @@ what is unmet. For a `visible-effect` ticket, also inspect the issue evidence an attached critique paired with the final screenshots before treating the contract as met. That list plus this critique check is what you nudge with. Nothing else counts as "done". -**Never `terminal send` to a worker that is not idle.** Measured on the same run: a send -issued mid-turn never became a user turn at all. It appears in the worker's session -transcript only as `queue-operation` records, and the running turn was cut short on a -mid-flow sentence. Everything a worker needs belongs in its prompt FILE at launch. When new -information arrives mid-run, append it to that file and point the worker back at it: - -``` -node tools/nudge-worker.mjs --terminal --prompt-file < update.md -node tools/nudge-worker.mjs --terminal --text "" -``` - -Either form waits for tui-idle first and REFUSES with exit 1 (sending nothing) while the -worker is busy, so a mid-turn send is not reachable through the sanctioned path. +**Headless workers cannot receive a mid-run user turn.** `codex exec` has no terminal +injection channel. When information arrives mid-run, wait for the worker process to exit, +derive the artifact verdict, update the prompt, and relaunch. Do not promise a nudge that +cannot be delivered. **What the fleet is doing right now** is `/watch` (`tools/worker-watch.mjs`): per worktree, the -ticket, the branch, the Linear state, BUSY or IDLE by repaint delta, the last meaningful output -lines, and the contract verdict above. Liveness and delivery answer different questions, and -`IDLE + NOT MET` is the pair that costs a run: a worker that stopped on a question with nobody -at the keyboard. Read it instead of hand-running `orca terminal read`, which returns a busy -worker's tail as thousands of characters of concatenated `Working` fragments. +ticket, the branch, the Linear state, BUSY or IDLE from the launcher-written worker PID, and the +contract verdict above. Liveness and delivery answer different questions, and `IDLE + NOT MET` is +the pair that costs a run: a worker process that exited without delivering. Read it instead of +hand-running `orca terminal read`, which for a headless worker shows no live turn at all. After the PR opens, the worker owns its automated review cycle. The orchestrator does not read review bodies, author review-round files, or relay findings back to the worker. It waits for one @@ -704,9 +694,10 @@ deletion, the orchestrate-skill fix) each ended their turn on "the monitor will with no live background child. Two of the three prompts already carried a warning against exactly that, so the subagent-side half alone does not hold. Both halves are the rule: -- **In a prompt whose task includes waiting on CI or a review:** poll in the FOREGROUND, - sleep 60 to 120s per loop, inside your own turn. End the turn only on the goal state or a - genuinely unfixable blocker, and say which one. +- **In a prompt whose task includes waiting on CI or a review:** use one FOREGROUND blocking + `node tools/pr-watch.mjs --repo --pr ` invocation without `--once`. + State `yield_time_ms` explicitly at or above the whole expected wait. End the turn only on + the goal state or a genuinely unfixable blocker, and say which one. - **On any completion notification whose result reads "waiting", "standing by", or "monitor armed":** read the real PR/CI state yourself and send the agent back to work with it. Standing by is not progress. diff --git a/.claude/skills/watch/SKILL.md b/.claude/skills/watch/SKILL.md index a752e339a..bf4a7c36d 100644 --- a/.claude/skills/watch/SKILL.md +++ b/.claude/skills/watch/SKILL.md @@ -3,11 +3,10 @@ name: watch description: >- Answer "what is every child session doing right now" in one screen. Reads every Orca worktree in the Orbit repos via tools/worker-watch.mjs and reports, per worker: the Linear ticket, the - branch, the ticket's Linear state, BUSY or IDLE classified by repaint delta, the last - meaningful output lines with TUI repaint noise stripped, and the worker-status.mjs contract - verdict. Read-only: it sends nothing to a worker and moves no ticket. Use when the question is - "is that worker still working", "is anything stuck", "what are my workers doing", or "where - does the fleet stand" during an /orchestrate run. + branch, the ticket's Linear state, BUSY or IDLE from the launcher-written worker PID, and the + worker-status.mjs contract verdict. Read-only: it sends nothing to a worker and moves no + ticket. Use when the question is "is that worker still working", "is anything stuck", "what + are my workers doing", or "where does the fleet stand" during an /orchestrate run. argument-hint: "[ui|api|landing] (default: every repo)" effort: low --- @@ -25,19 +24,20 @@ question. Never answer it by handing over the `node` line. node tools/worker-watch.mjs ``` -With a repo argument, scope it: `--repo ui|api|landing`. Add `--no-contract` when the question -is purely "is it alive" and the wait for a fetch plus a `gh` call per worktree is not worth it; -add `--lines ` when the default eight output lines cut off the thing you are looking for. +With a repo argument, scope it: `--repo ui|api|landing`. Add `--json` for the machine-readable +report, which additionally carries each worker PID and whether it is alive. Those are the only +flags; `--no-contract` and `--lines ` were deleted with the repaint sampler. -Run it from the orbit-ui-mobile repo root. It samples liveness over 3 seconds, so it takes a few -seconds plus roughly a second per worktree for the contract verdict. Exit 0 means the report -printed, and that includes "no Orca worktrees" - an empty fleet is a result, not a failure. +Run it from the orbit-ui-mobile repo root. Liveness is a single PID probe, so the only wait is +roughly a second per worktree for the contract verdict. Exit 0 means the report printed, and +that includes "no Orca worktrees" - an empty fleet is a result, not a failure. ## 2. Read the two verdicts, which answer different questions -**BUSY or IDLE is LIVENESS**, measured as repaint delta across two `orca terminal list` samples: -a running turn repaints its spinner continuously, an idle TUI emits nothing at all. It says -whether a turn is running. It says nothing about whether the work is any good or even started. +**BUSY or IDLE is LIVENESS**, measured as the launcher-written worker PID: `launch-worker.mjs` +appends the headless worker's PID to `orbit-worker-pids.jsonl` in that worktree's git directory, +and BUSY means that process is still running. It says whether the worker is alive. It says +nothing about whether the work is any good or even started. **CONTRACT MET or NOT MET is DELIVERY**, from `worker-status.mjs`, derived from artifacts (commits above the fetched base, a clean worktree, the branch pushed, a PR open, the issue In @@ -53,8 +53,9 @@ The pairs mean different things, and the diagnosis is in the combination: | IDLE | NOT MET | **stopped early**: it ended a turn on a question, hit a wall, or died. Read its last output lines, then decide | | BUSY | MET | finishing up after the PR, or drifted past its contract. Read the output lines | -`IDLE + NOT MET` is the one that costs a run, because nobody is at that keyboard. The last -output lines in the report are usually enough to tell a question apart from a crash. +`IDLE + NOT MET` is the one that costs a run: the worker process exited without delivering. +Headless workers take no mid-run turn, so the remedy is always to update the prompt file and +relaunch, never to nudge. ## 3. Answer, then stop @@ -63,13 +64,13 @@ Lead with the count and the exceptions, not with a table of everything that is f ``` 4 workers: 3 BUSY, 1 IDLE ORB-88 IDLE NOT MET: commits, pushed, pr-open - last output: "Which of these two approaches do you want?" - -> stopped on a question. Its unmet list is the nudge. + worker PID 24180 exited + -> died or finished early. Its unmet list is the relaunch prompt. ORB-90 BUSY NOT MET (working, 12 commits) ... ``` -Then stop. This skill decides nothing: what to send a stalled worker is `/orchestrate`'s -judgement, and the sanctioned way to send it is `tools/nudge-worker.mjs`, which refuses to -deliver into a busy TUI. Never send to a worker from here, never move a Linear ticket from here, -and never treat IDLE as done. +Then stop. This skill decides nothing: what to do about a stalled worker is `/orchestrate`'s +judgement. A headless worker has no live turn channel at all, so `tools/nudge-worker.mjs` +refuses every invocation; the remedy is an updated prompt file and a relaunch. Never move a +Linear ticket from here, and never treat IDLE as done. diff --git a/tools/README.md b/tools/README.md index d6879c0ca..a521e506b 100644 --- a/tools/README.md +++ b/tools/README.md @@ -32,7 +32,7 @@ Read `CONVENTIONS.md` before adding one. Use the `/make-tool` skill to scaffold | `redesign-coverage.mjs` | Asserts the redesign denominator twice: every surface in `surfaces.json` is claimed by exactly one #539 redesign ticket (D35), and every `.tsx` under `apps/*/app` + `apps/*/components` maps to exactly one ticket by directory rule (D38); exits 1 on any unclaimed surface or orphaned file. Survives until #539 completes (D39). | `npm run redesign:coverage` (`--json`) | | `arch-map.mjs` | Generates `architecture.json` + `architecture.html` (routes, parity pairs, endpoints, i18n ownership); the drift CI job (`arch-map.yml`) regenerates and fails on drift. | `node tools/arch-map.mjs` | | `ai-quota.mjs` | Reads the current account-level Claude and Codex quota windows from Orca's labeled Usage control and the Codex app-server JSON-RPC API, returning both engines in one object while preserving a healthy side when the other is unavailable. Its usage percentages are context only: they belong to no invocation, never feed the fuse, and are never subtracted to infer invocation cost. When `launch-worker.mjs` copies one into a ledger record, the launcher adds the observation timestamp there. Backs `/quota`. | `node tools/ai-quota.mjs --json` (`--help`) | -| `automation-budget.mjs` | Maintains the append-only per-invocation automation ledger using authoritative input and output token counts. Provider-estimated cost is recorded only when the provider supplies it. Missing token measurements stay absent and make the fuse fail closed rather than becoming zero. The engine-local fuse atomically checks and appends a pending reservation under one ledger lock before worktree creation, so concurrent launchers cannot both pass the same remaining budget. Verified pre-delivery rollback appends a cancellation tombstone; delivered work keeps the pending entry until the worker appends its provider-authoritative measurement with the exact command and ledger path it received. Routine work projected to cross 1,000,000 tokens blocks, while explicitly reserved deep work proceeds with a warning. | `node tools/automation-budget.mjs check|reserve|record|cancel|report --help` | +| `automation-budget.mjs` | Maintains the append-only per-invocation automation ledger using authoritative input and output token counts. Provider-estimated cost is recorded only when the provider supplies it. Measured spend is UNCACHED input plus output: a record's `cachedInputTokens` are subtracted, because a cache read is not fresh spend. Missing token measurements stay absent and make the fuse fail closed rather than becoming zero, and a record reporting exactly one of the two token figures keeps failing closed for the whole window. The engine-local fuse atomically checks and appends a pending reservation under one ledger lock before worktree creation, so concurrent launchers cannot both pass the same remaining budget. Verified pre-delivery rollback appends a cancellation tombstone; delivered work keeps the pending entry until the worker appends its provider-authoritative measurement with the exact command and ledger path it received. A reservation is a lease with two arms: `claim` attaches the spawned worker's PID, so the reservation expires the moment that process is gone and in any case at a 16 hour recycled-PID backstop that clears the longest measured session, while a reservation that never recorded a PID expires after 2 hours. A killed launcher and every legacy row written before reservations carried `pending` therefore release themselves instead of poisoning the fuse for the whole seven-day window. Any tier projected to cross the configured budget blocks; no tier bypasses the fuse. | `node tools/automation-budget.mjs check|reserve|record|cancel|report --help` | | `check-dashes.mjs` | The cross-repo dash ban: em dashes banned everywhere, en dashes only in numeric ranges. Backs the Dash Ban CI job, lefthook, and the shrink-only `dash-baseline.json`. | `--files ...` \| `--check-baseline` \| `--write-baseline` \| `--text ""` | | `check-copy.mjs` | The copy register: whole-file, values-only scan of locale copy for AI cliches, placeholder content, typed uppercase, and hardcoded brand colors. Backs the Copy Register CI job. | `node tools/check-copy.mjs --check` \| `--write-baseline` | | `check-context-budget.mjs` | Enforces the shrink-only byte budget for repo-visible always-loaded context and the structural allowlists for sibling `@` imports and unconditional rule files. Reports resolvable sibling context without enforcing it. Backs the Context Budget CI job. | `node tools/check-context-budget.mjs --check` \| `--write-baseline` \| `--json` | @@ -49,14 +49,13 @@ Read `CONVENTIONS.md` before adding one. Use the `/make-tool` skill to scaffold | `new-ticket.mjs` | Thin wrapper over `orca linear create` that validates the issue it just created, using the identifier orca REPORTED rather than one typed by hand. Use it instead of calling `orca linear create` directly whenever the result must be a valid ticket. | `node tools/new-ticket.mjs --help` | | `wave-plan.mjs` | Builds a merge-gated wave table for a Linear project, label, all non-done team issues, or an explicit `--issues` list. Reports same-wave file collisions and unknown affected-file declarations without changing table order. Backs `/orchestrate` (which then launches workers) and `/next` (which stops at the answer). Relation reads run in a bounded pool. | `node tools/wave-plan.mjs --help` | | `compose-prompt.mjs` | Composes the worker prompt from a Linear issue's verbatim body plus every chronological comment, preserving comment Markdown. Write outside an Orbit repository so the prompt cannot be committed. | `node tools/compose-prompt.mjs --issue ORB-N --output ` | -| `launch-worker.mjs` | Launches one ticket's Orca worktree + TUI worker end to end (`/orchestrate` step 2), and is the single place the four measured launch gotchas are handled: `worktree create` needs `--name`, a fresh checkout blocks on the workspace-trust prompt, Orca's `/` branch is not the contract branch, and a multi-line prompt through `terminal send` arrives mangled. Reads the engine, model routing, repo paths and concurrency cap through `tools/lib/orchestrator-config.mjs`; before creating anything, it serialises the target repo's live Orca inventory and worktree creation, excludes main and archived worktrees, and refuses at the cap with the current count and occupying paths. It also refuses any engine that does not declare `interactive: true` (a headless worker cannot be supervised) or whose command or args carry a headless token; prints handle + worktree path + branch as JSON. Exit 0 means the worker ACCEPTED the prompt as a user turn, read back off the TUI after the send, because orca accepting a `terminal send` is not delivery: measured on ORB-88, the composer swallowed the pointer and the launch reported success on a worker sitting idle with no work. Any non-zero exit rolls the worktree and its new branches back out, so a relaunch starts clean. | `node tools/launch-worker.mjs --issue ORB-N --prompt-file ` (`--dry-run`, `--help`) | -| `teardown-worktree.mjs` | Removes one completed ticket's Orca worktree, terminals, and local branch immediately after the Linear issue is verified Done. Refuses unless the worktree is clean, the forge-reported pull request merge commit is in the target branch, the local branch tip is contained in the pull request head, the live issue is Done, and no terminal is repainting. Verifies removal from the filesystem and `git worktree list`, rather than trusting Orca's removal response. | `node tools/teardown-worktree.mjs --issue ORB-N` (or a worktree selector; `--help`) | -| `nudge-worker.mjs` | Delivers a message to a running TUI worker only when it is tui-idle, and exits 1 sending NOTHING while it is mid-turn (a send while busy is queued and cuts the running turn short). Appends the update to the worker's prompt file from stdin and sends a one-line re-read pointer; rejects multi-line `--text`. | `node tools/nudge-worker.mjs --terminal --prompt-file < update.md` (`--text`, `--help`) | +| `launch-worker.mjs` | Launches one ticket's Orca worktree and worker end to end (`/orchestrate` step 2). The engine's `interactive` declaration must agree with its invocation in BOTH directions: a headless token with `interactive: true`, or none with `interactive: false`, is refused. Codex ships `interactive: false`, so the worker is spawned as a real process, its PID appended to `orbit-worker-pids.jsonl` in the worktree's git directory, and its budget reservation claimed with that PID. Node refuses to spawn a `.cmd` without a shell, and a shell would re-parse the prompt, so an npm shim is resolved to the script it execs and Node is spawned on that; a shim of any other shape fails closed. `--existing-worktree` launches an additional slice into an existing worktree under `maxSlicesPerWorker`, and both launch modes hold the repo launch lock across the read, check, spawn and append. Interactive engines still get the trust-screen answer and the read-back pointer. | `node tools/launch-worker.mjs --issue ORB-N --prompt-file ` (`--repo`, `--base-branch`, `--branch-prefix`, `--max-parallel-worktrees`, `--comment`, `--workspace-status`, `--existing-worktree`, `--dry-run`, `--help`) | +| `teardown-worktree.mjs` | Removes one completed ticket's Orca worktree, terminals, and local branch immediately after the Linear issue is verified Done. Refuses unless the worktree is clean, the forge-reported pull request merge commit is in the target branch, the local tip is contained in that pull request's head, the issue is Done, and the launcher-written worker PID has exited. Removal is verified against the filesystem and Git rather than trusted from orca's response, and the worker PID marker is pruned once removal is proven. | `node tools/teardown-worktree.mjs (--issue ORB-N \| --worktree ) [--base ]` | +| `nudge-worker.mjs` | Refuses every invocation, by design. A headless worker has no live user-turn channel to send into, so there is nothing to deliver a mid-run update to: exit 1 with no arguments, exit 2 for an attempted injection, and orca is never called on either path. The remedy it names is to wait for the worker process to exit, update the prompt file, and relaunch. | `node tools/nudge-worker.mjs --help` | | `worker-status.mjs` | The artifact-backed worker completion gate and one-time pre-merge verifier. Requires commits above fresh `origin/`, a clean worktree, local HEAD equal to the remote PR head, an open PR with an approving review on that exact head, complete review activity inventories, zero unresolved threads, auditable acknowledgements for standalone automated activity, Linear In Review with the PR attached, and both screenshot and critique evidence for `visible-effect`. `--verify-review` also proves every resolved automated thread, including an informational one, has reconciliation evidence after its latest finding-bearing nested activity, names a later PR commit that changed the reviewed path, and rejects human threads resolved by the worker. | `node tools/worker-status.mjs --worktree --issue ORB-N` (`--base`, `--verify-review`, `--json`, `--help`) | -| `worker-watch.mjs` | The LIVENESS half of babysitting, alongside `worker-status.mjs`'s delivery verdict: for every Orca worktree in the Orbit repos, the ticket, branch, Linear state, BUSY or IDLE classified by repaint delta across two `terminal list` samples, the last meaningful output lines with TUI repaint noise stripped, and the contract verdict. Reports only; deciding what to send a stalled worker stays the orchestrator's judgement. Backs `/watch`. | `node tools/worker-watch.mjs` (`--repo`, `--lines`, `--no-contract`, `--json`, `--help`) | +| `worker-watch.mjs` | The LIVENESS half of babysitting, alongside `worker-status.mjs`'s delivery verdict: for every Orca worktree in the Orbit repos, the ticket, branch, Linear state, BUSY or IDLE from the launcher-written worker PID, and the contract verdict with its unmet list, since `IDLE + NOT MET` is the pair that costs a run. A verdict that cannot be read is reported as unavailable rather than folded into MET or NOT MET, and an empty fleet says so. Reports only; deciding what to do about a stalled worker stays the orchestrator's judgement. Backs `/watch`. | `node tools/worker-watch.mjs` (`--repo`, `--json`, `--help`) | | `pr-watch.mjs` | Low-level transition poller used by each worker's own review loop. Polls one or more PRs until a current-head state the caller has not already acted on and exits naming the transition (`gone`, `checks-failed`, `changes-requested`, `review-comment`, `approved`, `ready-to-merge`, `head-changed`, `review-decision`, `merge-clean`, `timeout`). `UNKNOWN` retains the last stable merge state, and changes among other merge states do not fire. The first poll establishes the transition baseline, while fresh unacted verdicts and unhandled readiness still fire immediately. `review-comment` is a submitted `COMMENTED` review, not an inline thread or PR conversation comment; `worker-status.mjs` remains the completion and pre-merge gate. | `node tools/pr-watch.mjs --repo --pr [--acted =:]` (`--once`, `--help`) | | `lib/orchestrator-config.mjs` | Not a tool, the shared `.claude/orchestrator.json` reader and engine-neutral worker tier resolver. It maps `tier:cheap` and `tier:deep` through the selected engine, rejects legacy, unknown, conflicting, missing or unchanged mappings, and keeps config failures consistent across harness tools. | imported, never invoked | -| `lib/tui-repaint.mjs` | Not a tool, the one shared module: is this TUI mid-turn? Two `lastOutputAt` samples a window apart, since `--for tui-idle` reports satisfied on a working codex and the terminal text keeps stale output. Imported by `launch-worker.mjs`, `nudge-worker.mjs` and `worker-watch.mjs`; exercised through them. | imported, never invoked | | `orca-web-port.mjs` | Assigns each linked Orca worktree a deterministic web port in the 3100-4099 window, detects a collision before persisting it, reports the assigned port, and starts Next on that port. The root checkout reports the unchanged default 3000. | `node tools/orca-web-port.mjs --setup` on worktree creation; `node tools/orca-web-port.mjs` to report; `npm run web` to start Next | ## Cross-repository harness lockstep diff --git a/tools/__fixtures__/legacy-reservation.jsonl b/tools/__fixtures__/legacy-reservation.jsonl new file mode 100644 index 000000000..e16bbdc9a --- /dev/null +++ b/tools/__fixtures__/legacy-reservation.jsonl @@ -0,0 +1 @@ +{"identity":"ORB-163:2026-07-30T22:02:19.076Z:af1bfc6a-4d7d-4def-8c8a-9f5da7060dff","engine":"codex","tier":"routine","startedAt":"2026-07-30T22:02:19.076Z","endedAt":"2026-07-30T22:02:32.179Z","accountContext":{"scope":"account","attributed":false,"usedPercent":11,"observedAt":"2026-07-30T22:02:32.179Z"}} diff --git a/tools/automation-budget.mjs b/tools/automation-budget.mjs index e6477dd1f..be4c40b31 100644 --- a/tools/automation-budget.mjs +++ b/tools/automation-budget.mjs @@ -4,29 +4,74 @@ import { homedir } from "node:os" import { dirname, resolve } from "node:path" const WINDOW_MILLISECONDS = 7 * 24 * 60 * 60 * 1000 +/** + * A reservation is a LEASE, not a permanent claim. `reserve` appends a row carrying no + * measurement and `record` or `cancel` closes it, so anything that kills the launcher in + * between leaves that row open forever, and every ledger row written before reservations + * carried `pending` is open by construction. An expired lease holds no budget and never + * fails the fuse closed. + * + * Expiry reads the clock AND the worker process, and liveness can only ever expire a + * reservation earlier, never hold one open past the backstop. Both directions of a clock-only + * answer are wrong: expiring a live worker's reservation stops counting real projected spend + * and can authorise a launch past the budget, while never expiring a dead one poisons the fuse + * for the whole seven-day window. A recorded `workerPid` settles the first: `process.kill(pid, + * 0)` sends no signal and throws ESRCH when the process is gone, EPERM when it exists but is + * not ours, so EPERM is alive. Both errnos confirmed by running it. The clock still settles the + * second, because the operating system recycles pids and a liveness-only answer would let one + * recycled pid hold the fuse forever. + * + * The two populations need OPPOSITE lease lengths, so one global TTL is wrong for one of them. + * A row carrying a PID is paying for a process that demonstrably started, so its clock arm is + * only the recycled-pid terminator and must clear the longest real session. A row carrying NO + * PID was never claimed, which means the worker it was paying for either never started or died + * before it could be recorded; that row is stranded by definition and wants a short lease. The + * whole truth table, with no case left unterminated: + * + * PID alive, inside the backstop holds its reserved tokens the real 14.9 hour session + * PID alive, past the backstop expires a recycled pid, never immortal + * PID gone, any age expires at once the killed worker, fast path + * no PID, inside the lease holds a launcher still mid-setup + * no PID, past the lease expires every legacy row + * + * Both numbers are derived, neither is a round guess. The backstop is 16 hours because 275 codex + * rollouts measured on this machine, first to last event per session, give p50 8.1 min, p90 + * 3.8 h, p95 6.8 h, p99 13.4 h and max 14.9 h, which 16 hours clears with margin. The unclaimed + * lease is 2 hours because the gap it covers is reserve to claim, which is bounded by worktree + * creation and dependency install rather than by the worker's runtime, so two hours is already + * orders of magnitude of margin. Applying the 16 hour figure to unclaimed rows was measured + * wrong: it re-poisoned the fuse against the real production ledger for up to 16 hours. + */ +const CLAIMED_RESERVATION_BACKSTOP_MILLISECONDS = 16 * 60 * 60 * 1000 +const UNCLAIMED_RESERVATION_LEASE_MILLISECONDS = 2 * 60 * 60 * 1000 const ENGINES = new Set(["claude", "codex"]) const TIERS = new Set(["routine", "reserved"]) const DEFAULT_LEDGER_PATH = resolve(homedir(), ".orbit", "automation-budget.jsonl") const USAGE = `usage: automation-budget.mjs check --engine --identity --tier --reset-at --warning-tokens --budget-tokens --invocation-tokens [--ledger ] [--json] automation-budget.mjs reserve --engine --identity --tier --started-at --ended-at --reset-at --warning-tokens --budget-tokens --invocation-tokens [--account-used-percent --account-observed-at ] [--ledger ] [--json] - automation-budget.mjs record --identity --engine --tier --started-at --ended-at [--input-tokens ] [--output-tokens ] [--provider-estimated-cost ] [--account-used-percent --account-observed-at ] [--ledger ] [--json] + automation-budget.mjs record --identity --engine --tier --started-at --ended-at [--input-tokens --cached-input-tokens ] [--output-tokens ] [--provider-estimated-cost ] [--account-used-percent --account-observed-at ] [--ledger ] [--json] + automation-budget.mjs claim --identity --engine --tier --started-at --ended-at --invocation-tokens --worker-pid [--ledger ] [--json] automation-budget.mjs cancel --identity --engine --tier --started-at --ended-at [--ledger ] [--json] automation-budget.mjs report --engine --reset-at [--ledger ] [--json] check evaluate an invocation against the current engine's token budget reserve atomically evaluate and append a pending invocation before launch mutation record append one invocation observation to the ledger + claim re-append an open reservation carrying the worker PID now running it cancel append a tombstone for a pending invocation proven not to have started - report print one engine's current seven-day token totals and missing identities + report print one engine's current seven-day token totals, missing identities, and expired reservations --identity stable identity for the invocation --engine quota pool charged by the invocation; engines are never combined - --tier routine automation or explicitly reserved deep work + --tier routine automation; legacy reserved ledger rows remain readable --started-at invocation start as ISO-8601 with a timezone, or Unix seconds --ended-at invocation end as ISO-8601 with a timezone, or Unix seconds --input-tokens measured provider input tokens; omitted while measurement is unavailable --output-tokens measured provider output tokens; omitted while measurement is unavailable + --cached-input-tokens + provider cache-read input tokens, retained with the raw provider measurement + and subtracted from it, because a cache read is not fresh spend --provider-estimated-cost optional provider-estimated monetary cost; reporting context only --account-used-percent @@ -40,23 +85,33 @@ const USAGE = `usage: non-negative token warning level below the engine budget --invocation-tokens non-negative token reservation for the proposed invocation + --worker-pid process id of the worker this reservation is paying for; while it is alive the + reservation never expires, and once it is gone the reservation expires at once --ledger JSONL ledger path; defaults to ORBIT_AUTOMATION_BUDGET_LEDGER or ${DEFAULT_LEDGER_PATH} --json emit the command result as JSON; without it check and record are quiet on success --help, -h print this usage and exit 0 -The fuse blocks routine automation when measured input plus output tokens and the proposed -reservation would exceed the token budget. Explicitly reserved deep work proceeds beyond the -routine budget with RESERVED status and a warning carrying the budget figures. The fuse fails -closed when the latest in-window record for any identity lacks either token measurement. +The fuse blocks automation when measured spend, every live reservation, and the proposed +reservation would exceed the token budget. Measured spend is UNCACHED input plus output: a +record's cachedInputTokens are subtracted from its inputTokens, because a cache read is not +fresh spend. The fuse fails closed when the latest in-window record for any identity lacks +either token measurement. Duplicate identities are append-only; the latest in-window record is authoritative. A cancelled -pending invocation contributes no tokens. Account usage percentage and estimated cost are context -only and never affect token totals. Records are attributed to the seven-day window containing -their end timestamp. Mutations share an adjacent lock file and fail closed on lock contention. +pending invocation contributes no tokens. A reservation is a lease. One that recorded a worker PID +expires the moment that process is gone, and in any case once its end timestamp is more than +${CLAIMED_RESERVATION_BACKSTOP_MILLISECONDS / 3_600_000} hours old, which terminates a recycled PID. One that never recorded a PID was never +confirmed spawned, so it expires once its end timestamp is more than ${UNCLAIMED_RESERVATION_LEASE_MILLISECONDS / 3_600_000} hours old. An expired +reservation holds no budget and no longer fails the fuse closed. No lease applies to a record +reporting exactly one of the two token figures: that is a half-measured real invocation, not a +reservation, and it fails the fuse closed for the whole window. Account usage percentage +and estimated cost are context only and never affect token totals. Records are attributed to the +seven-day window containing their end timestamp. Mutations share an adjacent lock file and fail +closed on lock contention. exit codes: 0 success or permitted invocation 2 invalid command-line input - 3 ledger read, validation, lock, append, or incomplete-measurement failure + 3 ledger read, validation, lock, append, claim, or incomplete-measurement failure 4 routine invocation blocked because its token reservation would exceed the budget` let releaseActiveLock = null @@ -77,7 +132,7 @@ const parseArguments = (argumentsList) => { process.exit(0) } const command = argumentsList[0] - if (!["check", "reserve", "record", "cancel", "report"].includes(command)) fail(`expected check, reserve, record, cancel, or report\n\n${USAGE}`, 2) + if (!["check", "reserve", "record", "claim", "cancel", "report"].includes(command)) fail(`expected check, reserve, record, claim, cancel, or report\n\n${USAGE}`, 2) const values = new Map() const switches = new Set() for (let index = 1; index < argumentsList.length; index++) { @@ -94,6 +149,7 @@ const parseArguments = (argumentsList) => { "--started-at", "--ended-at", "--input-tokens", + "--cached-input-tokens", "--output-tokens", "--provider-estimated-cost", "--account-used-percent", @@ -102,6 +158,7 @@ const parseArguments = (argumentsList) => { "--warning-tokens", "--budget-tokens", "--invocation-tokens", + "--worker-pid", "--ledger", ].includes(argument)) { fail(`unknown argument ${argument}\n\n${USAGE}`, 2) @@ -341,9 +398,23 @@ const validateRecord = (record, lineNumber) => { validated.cancelled = true return validated } + if (hasOwn(record, "pending")) { + if (record.pending !== true || !hasOwn(record, "reservedTokens")) fail(`${prefix} pending record must carry reservedTokens`, 3) + validated.pending = true + validated.reservedTokens = parseTokenCount(record.reservedTokens, `${prefix} reservedTokens`, 3) + if (hasOwn(record, "workerPid")) { + validated.workerPid = parseTokenCount(record.workerPid, `${prefix} workerPid`, 3, true) + } + } else if (hasOwn(record, "workerPid")) { + fail(`${prefix} workerPid is only valid on a pending reservation`, 3) + } if (hasOwn(record, "inputTokens")) { validated.inputTokens = parseTokenCount(record.inputTokens, `${prefix} inputTokens`, 3) } + if (hasOwn(record, "cachedInputTokens")) { + validated.cachedInputTokens = parseTokenCount(record.cachedInputTokens, `${prefix} cachedInputTokens`, 3) + if (!hasOwn(record, "inputTokens") || validated.cachedInputTokens > validated.inputTokens) fail(`${prefix} cachedInputTokens must not exceed inputTokens`, 3) + } if (hasOwn(record, "outputTokens")) { validated.outputTokens = parseTokenCount(record.outputTokens, `${prefix} outputTokens`, 3) } @@ -396,6 +467,8 @@ const readLedger = (path) => { const summarize = (records, engine, resetAt) => { const resetMilliseconds = resetAt.getTime() const windowStart = new Date(resetMilliseconds - WINDOW_MILLISECONDS) + const claimedFloor = Date.now() - CLAIMED_RESERVATION_BACKSTOP_MILLISECONDS + const unclaimedFloor = Date.now() - UNCLAIMED_RESERVATION_LEASE_MILLISECONDS const latestByIdentity = new Map() for (const record of records) { const endedMilliseconds = Date.parse(record.endedAt) @@ -407,18 +480,50 @@ const summarize = (records, engine, resetAt) => { let routineTokens = 0 let reservedTokens = 0 const missingIdentities = [] + const expiredIdentities = [] + let pendingTokens = 0 for (const record of latestByIdentity.values()) { if (record.cancelled === true) continue - if (!hasOwn(record, "inputTokens") || !hasOwn(record, "outputTokens")) { - missingIdentities.push(record.identity) + const hasInput = hasOwn(record, "inputTokens") + const hasOutput = hasOwn(record, "outputTokens") + const open = record.pending === true || !hasInput || !hasOutput + if (open) { + /** + * A reservation reports NEITHER token figure, because nothing was ever measured. A record + * reporting exactly one of them is a real invocation that reported half its usage, and no + * lease applies to it: it keeps failing the fuse closed for the whole window, which is what + * this tool's contract promises for an absent measurement. Without this line the two are + * structurally identical, and a half-measured invocation would quietly stop counting. + */ + if (record.pending !== true && (hasInput || hasOutput)) { + missingIdentities.push(record.identity) + continue + } + /** + * A claimed row expires on EITHER its dead process or the recycled-pid backstop, never on + * liveness alone, because the operating system recycles pids and a liveness-only answer + * would let one recycled pid hold the fuse for the whole window. An unclaimed row has no + * process to ask, so its own much shorter lease is the only terminator. `endedAt` is + * always a validated ISO timestamp here because `validateRecord` reparses it and exits 3 + * on anything else, so neither comparison can silently be NaN. + */ + const endedMilliseconds = Date.parse(record.endedAt) + const expired = hasOwn(record, "workerPid") + ? !processIsAlive(record.workerPid) || endedMilliseconds < claimedFloor + : endedMilliseconds < unclaimedFloor + if (expired) expiredIdentities.push(record.identity) + else if (record.pending === true) pendingTokens += record.reservedTokens + else missingIdentities.push(record.identity) continue } - inputTokens += record.inputTokens + const uncachedInputTokens = record.inputTokens - (record.cachedInputTokens ?? 0) + inputTokens += uncachedInputTokens outputTokens += record.outputTokens - if (record.tier === "routine") routineTokens += record.inputTokens + record.outputTokens - else reservedTokens += record.inputTokens + record.outputTokens + if (record.tier === "routine") routineTokens += uncachedInputTokens + record.outputTokens + else reservedTokens += uncachedInputTokens + record.outputTokens } missingIdentities.sort() + expiredIdentities.sort() return { engine, inputTokens, @@ -426,7 +531,9 @@ const summarize = (records, engine, resetAt) => { totalTokens: inputTokens + outputTokens, routineTokens, reservedTokens, + pendingTokens, missingIdentities, + expiredIdentities, windowStart: windowStart.toISOString(), resetsAt: resetAt.toISOString(), } @@ -452,38 +559,30 @@ const parseBudgetRequest = (values, allowedFlags) => { const evaluateBudget = (request, records, json) => { const { engine, identity, tier, resetAt, warningTokens, budgetTokens, invocationTokens } = request const summary = summarize(records, engine, resetAt) - const projectedTokens = summary.totalTokens + invocationTokens - if (tier !== "reserved" && summary.missingIdentities.length > 0) { + const projectedTokens = summary.totalTokens + summary.pendingTokens + invocationTokens + const status = projectedTokens > budgetTokens ? "BLOCK" : projectedTokens >= warningTokens ? "WARN" : "PROCEED" + if (status === "BLOCK") { + const result = { status, identity, tier, warningTokens, budgetTokens, invocationTokens, projectedTokens, ...summary } + emitJson(result, json) + fail(`invocation "${identity}" blocked: budget ${budgetTokens} tokens, observed spend ${summary.totalTokens} tokens, pending ${summary.pendingTokens} tokens, reservation ${invocationTokens} tokens, projected spend ${projectedTokens} tokens; resets at ${summary.resetsAt}`, 4) + } + if (summary.missingIdentities.length > 0) { emitJson({ status: "INCOMPLETE", identity, tier, warningTokens, budgetTokens, invocationTokens, ...summary }, json) fail(`cannot check invocation "${identity}": latest in-window records lack input or output tokens for identities ${summary.missingIdentities.join(", ")}`, 3) } - const status = tier === "reserved" - ? "RESERVED" - : projectedTokens > budgetTokens - ? "BLOCK" - : projectedTokens >= warningTokens - ? "WARN" - : "PROCEED" const result = { status, identity, tier, warningTokens, budgetTokens, invocationTokens, projectedTokens, ...summary } - if (status === "BLOCK") { - emitJson(result, json) - fail(`invocation "${identity}" blocked: budget ${budgetTokens} tokens, observed spend ${summary.totalTokens} tokens, reservation ${invocationTokens} tokens, projected spend ${projectedTokens} tokens; resets at ${summary.resetsAt}`, 4) - } return result } const emitBudgetResult = (result, json) => { emitJson(result, json) - const { status, identity, warningTokens, budgetTokens, invocationTokens, projectedTokens, totalTokens, missingIdentities } = result + const { status, identity, warningTokens, budgetTokens, invocationTokens, projectedTokens, totalTokens, expiredIdentities } = result + if (expiredIdentities.length > 0) { + console.error(`automation-budget: reservation lease expired for identities ${expiredIdentities.join(", ")}; they hold no budget and no longer fail the fuse closed`) + } if (status === "WARN") { console.error(`automation-budget: warning: invocation "${identity}" projects ${projectedTokens} tokens; warning ${warningTokens} tokens, budget ${budgetTokens} tokens, observed spend ${totalTokens} tokens`) } - if (status === "RESERVED" && (projectedTokens >= warningTokens || missingIdentities.length > 0)) { - const missingContext = missingIdentities.length > 0 - ? `, missing measurements for identities ${missingIdentities.join(", ")}` - : "" - console.error(`automation-budget: warning: reserved invocation "${identity}" proceeds with ${projectedTokens} projected tokens; warning ${warningTokens} tokens, budget ${budgetTokens} tokens, observed spend ${totalTokens} tokens, reservation ${invocationTokens} tokens${missingContext}`) - } } const budgetFlags = new Set([ @@ -529,6 +628,8 @@ const runReserve = (values, json) => { tier: request.tier, startedAt: startedAt.toISOString(), endedAt: endedAt.toISOString(), + pending: true, + reservedTokens: request.invocationTokens, } const hasAccountPercent = values.has("--account-used-percent") const hasAccountTimestamp = values.has("--account-observed-at") @@ -561,6 +662,7 @@ const runRecord = (values, json) => { "--started-at", "--ended-at", "--input-tokens", + "--cached-input-tokens", "--output-tokens", "--provider-estimated-cost", "--account-used-percent", @@ -580,6 +682,10 @@ const runRecord = (values, json) => { if (values.has("--input-tokens")) { record.inputTokens = parseTokenCount(values.get("--input-tokens"), "--input-tokens") } + if (values.has("--cached-input-tokens")) { + record.cachedInputTokens = parseTokenCount(values.get("--cached-input-tokens"), "--cached-input-tokens") + if (!hasOwn(record, "inputTokens") || record.cachedInputTokens > record.inputTokens) fail(`--cached-input-tokens requires --input-tokens and cannot exceed it`, 2) + } if (values.has("--output-tokens")) { record.outputTokens = parseTokenCount(values.get("--output-tokens"), "--output-tokens") } @@ -607,6 +713,48 @@ const runRecord = (values, json) => { emitJson({ status: "RECORDED", record }, json) } +/** + * The reservation is appended BEFORE the worktree exists, so the worker process it is paying + * for does not exist yet and its PID cannot be on that row. `claim` appends the same pending + * reservation again once the process is running, this time carrying its PID, and the + * latest-in-window rule makes that the authoritative one. It re-evaluates nothing on purpose: + * the budget was gated at `reserve`, and blocking here would refuse a worker already working. + */ +const runClaim = (values, json) => { + rejectUnexpected(values, new Set([ + "--identity", + "--engine", + "--tier", + "--started-at", + "--ended-at", + "--invocation-tokens", + "--worker-pid", + "--ledger", + ])) + const startedAt = parseTimestamp(requireValue(values, "--started-at"), "--started-at") + const endedAt = parseTimestamp(requireValue(values, "--ended-at"), "--ended-at") + if (startedAt.getTime() > endedAt.getTime()) fail(`--started-at must not be after --ended-at`, 2) + const claimed = { + identity: parseIdentity(requireValue(values, "--identity")), + engine: parseEngine(requireValue(values, "--engine")), + tier: parseTier(requireValue(values, "--tier")), + startedAt: startedAt.toISOString(), + endedAt: endedAt.toISOString(), + pending: true, + reservedTokens: parseTokenCount(requireValue(values, "--invocation-tokens"), "--invocation-tokens"), + workerPid: parseTokenCount(requireValue(values, "--worker-pid"), "--worker-pid", 2, true), + } + const path = ledgerPath(values) + withLedgerLock(path, () => { + const existingLedger = readLedger(path) + const latest = [...existingLedger.records].reverse().find((record) => record.identity === claimed.identity) + if (!latest) fail(`cannot claim invocation "${claimed.identity}": no ledger record exists`, 3) + if (latest.pending !== true) fail(`cannot claim invocation "${claimed.identity}": its latest record is not an open reservation`, 3) + appendRecord(path, existingLedger, claimed) + }) + emitJson({ status: "CLAIMED", record: claimed }, json) +} + const runCancel = (values, json) => { rejectUnexpected(values, new Set([ "--identity", @@ -656,7 +804,8 @@ const runReport = (values, json) => { if (json) console.log(JSON.stringify(result)) else { const missing = result.missingIdentities.length > 0 ? result.missingIdentities.join(", ") : "none" - console.log(`${result.engine}: ${result.totalTokens} tokens (${result.inputTokens} input, ${result.outputTokens} output; ${result.routineTokens} routine, ${result.reservedTokens} reserved); missing identities: ${missing}; resets at ${result.resetsAt}`) + const expired = result.expiredIdentities.length > 0 ? result.expiredIdentities.join(", ") : "none" + console.log(`${result.engine}: ${result.totalTokens} tokens (${result.inputTokens} input, ${result.outputTokens} output; ${result.routineTokens} routine, ${result.reservedTokens} reserved, ${result.pendingTokens} pending); missing identities: ${missing}; expired reservations: ${expired}; resets at ${result.resetsAt}`) } } @@ -664,5 +813,6 @@ const options = parseArguments(process.argv.slice(2)) if (options.command === "check") runCheck(options.values, options.json) else if (options.command === "reserve") runReserve(options.values, options.json) else if (options.command === "record") runRecord(options.values, options.json) +else if (options.command === "claim") runClaim(options.values, options.json) else if (options.command === "cancel") runCancel(options.values, options.json) else runReport(options.values, options.json) diff --git a/tools/launch-worker.mjs b/tools/launch-worker.mjs index 0194df717..ce7ab6ab0 100644 --- a/tools/launch-worker.mjs +++ b/tools/launch-worker.mjs @@ -17,7 +17,7 @@ * reviews, or moves a Linear issue. */ -import { execFileSync, spawnSync } from "node:child_process" +import { execFileSync, spawn, spawnSync } from "node:child_process" import { randomUUID } from "node:crypto" import { appendFileSync, @@ -30,11 +30,11 @@ import { writeFileSync, } from "node:fs" import { homedir } from "node:os" -import { basename, join, resolve } from "node:path" +import { basename, delimiter, dirname, extname, join, resolve } from "node:path" import { fileURLToPath } from "node:url" import { readOrchestratorConfig, resolveWorkerInvocation } from "./lib/orchestrator-config.mjs" -import { SETTLE_MS, isRepainting, pause } from "./lib/tui-repaint.mjs" +const pause = (milliseconds) => Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, milliseconds) const USAGE = `usage: launch-worker.mjs --issue ORB-N --prompt-file [options] @@ -53,6 +53,7 @@ const USAGE = `usage: launch-worker.mjs --issue ORB-N --prompt-file [opti override the configured concurrency cap for this invocation --comment "" worktree card comment (default: " launched: worker running") --workspace-status Orca board status id (default: in-progress) + --existing-worktree launch an additional headless worker in this existing Orca worktree --dry-run resolve everything and print the plan; run no mutating orca or git command --help, -h print this usage and exit 0 @@ -85,6 +86,8 @@ const ORCA = process.env.ORCA_BIN || "C:\\Users\\thoma\\AppData\\Local\\Programs /** How long one tui-idle wait may block, and how many waits a launch gets before it fails. */ const WAIT_TIMEOUT_MS = 60000 const MAX_WAIT_ATTEMPTS = 6 +// Conservative: two observations establish that 178 fails and 42 succeeds, not the boundary. +const MAX_INTERACTIVE_PROMPT_PATH_LENGTH = 120 /** * What each worker CLI does that this script has to know, keyed by the binary it runs. @@ -99,6 +102,35 @@ const MAX_WAIT_ATTEMPTS = 6 * reason each CLI prints on the screen itself: Claude Code takes the digit, codex paints * a preselected list saying "Press enter to continue" and takes Enter alone. Both * measured; sending codex the digit left its process exited (-1). + * + * The measured facts behind each engine's REQUIRED run-permitting policy, which used to live in + * .claude/orchestrator.json's per-worker `notes` and belong next to the guard that enforces them: + * + * claude: the mode must be `bypassPermissions`, never `acceptEdits`. `acceptEdits` auto-approves + * file writes only, so every shell command still prompts and a worker with nobody at the keyboard + * is stuck. Measured on the 2026-07-24 ORB-75 run, where git switch, dotnet build/test/format, gh + * and orca were all denied and the worker delivered files with zero gates run, zero commit, no PR. + * + * codex: the policy is the single `--dangerously-bypass-approvals-and-sandbox`, not the equivalent + * pair `-a never --sandbox danger-full-access`, and never `--full-auto`. `--full-auto` is + * `-a on-request --sandbox workspace-write`, and `on-request` lets the MODEL decide when to ask a + * human who is not there. The single flag beats the pair because the pair has a half-state: an + * edit dropping `-a never` while keeping the sandbox flag silently restores approval prompts. The + * containment story is the disposable Orca worktree, the same one that justifies bypassPermissions. + * + * codex, measured 2026-07-27 against codex-cli 0.145.0 on Windows 11: `-c windows.sandbox="unelevated"` + * is load-bearing. The default Windows sandbox is elevated, its setup needs Administrator rights, + * and the first run paints "Setting up sandbox... Input disabled until setup completes" forever in + * a PTY with no desktop to raise UAC on. The unelevated fallback needs no elevation, and the worker + * never executes inside it anyway because the bypass flag runs commands unsandboxed. + * + * codex auth: CODEX_HOME DECIDES WHETHER A WORKER IS LOGGED IN, and it is the first thing to check + * before believing any auth verdict. Orca redirects codex's home for the terminals it spawns, so on + * this machine the real credential is + * C:\\Users\\thoma\\AppData\\Roaming\\orca\\codex-runtime-home\\home\\auth.json, not ~/.codex/auth.json. + * Measured 2026-07-27: a shell that had lost CODEX_HOME reported "Not logged in" while the same CLI + * in an Orca terminal in the same worktree reported "Logged in using ChatGPT" minutes earlier. Never + * diagnose codex auth from an ad hoc shell without printing CODEX_HOME first. */ const ENGINE_PROFILES = { claude: { @@ -124,11 +156,8 @@ const ENGINE_PROFILES = { const TRUST_BLOCKED_REASON = /trust/i const flatten = (text) => text.replace(/\s+/g, "").toLowerCase() -/** The prompt pointer below is a `terminal send`, and a send to a busy worker is the ORB-75 - * failure this whole script exists to avoid, so a satisfied tui-idle wait alone is not enough - * to send on. Why the repaint delta is the signal that works for both engines, and why the - * terminal text is not: tools/lib/tui-repaint.mjs. */ -const busy = (handle) => isRepainting(orca, handle) +// Orca WORKTREES remain required for isolation, concurrency accounting, and cleanup. Only +// Orca TERMINALS are optional now that headless workers are ordinary child processes. /** * How many times the pointer may be sent before the launch is a failure, and how long the TUI @@ -148,6 +177,13 @@ const TERMINAL_CREATE_BACKOFF_MS = 1000 * repainting after all of them is never re-sent to: there is no safe moment, and a queued send * would cut its running turn short. */ const MAX_POINTER_SETTLES = 3 +const SETTLE_MS = 1000 +const terminalIsRepainting = (terminal) => { + const first = orca(["terminal", "show", "--terminal", terminal]).terminal?.lastOutputAt + pause(SETTLE_MS) + const second = orca(["terminal", "show", "--terminal", terminal]).terminal?.lastOutputAt + return Number.isFinite(first) && Number.isFinite(second) && second > first +} /** * The standing worker contract, owned HERE rather than by whoever composed the prompt file. @@ -165,6 +201,7 @@ const MAX_POINTER_SETTLES = 3 * CLASS: a .gitignore entry only ever covers the one artifact somebody already got burned by. */ const WORKER_CONTRACT_MARKER = "## Standing worker contract (injected by tools/launch-worker.mjs)" +const SLICE_CONTRACT_MARKER = "## Slice worker contract (injected by tools/launch-worker.mjs)" const WORKER_CONTRACT = ` --- @@ -185,8 +222,8 @@ conflict with anything above, these win. is acceptable; unmentioned is not. The pull request must be ready for review, never a draft. Never silently drop a criterion. 3. **Own the automated review cycle.** After the PR is open, attached, and In Review, poll its - review transitions with \`node tools/pr-watch.mjs --repo --pr - --once\` only as a low-level wake-up. After every call and before waiting or reporting + review transitions with a foreground blocking \`node tools/pr-watch.mjs --repo --pr \`. + Every wait must state \`yield_time_ms\` explicitly, at or above the whole expected wait. After every call and before waiting or reporting completion, run \`node tools/worker-status.mjs --worktree --issue ORB-N --json\`. That full-surface completion poll inventories review submissions, review threads and their nested comments, and PR conversation comments, and fails closed on an incomplete inventory. @@ -205,11 +242,11 @@ conflict with anything above, these win. finding and your reasoning. 5. **Your job ends on one report.** Report completion once the PR is approved with zero unresolved threads, or send the escalation from clause 4. An earlier instruction to stop - after opening or attaching the PR does not replace this endpoint. Never watch another - ticket, worktree, or PR. -6. **Never arm a background monitor, watcher or wait loop that outlives this contract.** + after opening or attaching the PR does not replace this endpoint. If your work order tells you + both to watch something and to stop, STOP wins. Never watch another ticket, worktree, or PR. +6. **Never arm a detached background monitor, watcher or wait loop that outlives this contract.** A foreground blocking wait is permitted. 7. **Never merge any PR, never push to \`main\`, never use \`--no-verify\`, never edit a gate - baseline.** + baseline, never run \`gh pr merge --admin\`, never directly call \`PUT /repos/{owner}/{repo}/pulls/{number}/merge\`, and never directly call the GraphQL \`mergePullRequest\` mutation. If a merge genuinely needs an admin override, STOP and ask Thomas to merge it himself; never perform the override.** 8. **Stage explicitly.** Commit only the paths you edited yourself. \`git add -A\`, \`git add .\` and \`git commit -a\` are forbidden. A worktree is a shared filesystem that sibling workers, dev servers and tooling all write into, so a blanket stage turns any of their runtime @@ -226,6 +263,19 @@ conflict with anything above, these win. more than one independent finding is dispatched one subagent per finding, not fixed inline. ` +const SLICE_CONTRACT = ` + +--- + +${SLICE_CONTRACT_MARKER} + +This process is one slice in a coordinator-owned worktree. Make only the requested change and run +the relevant checks. Do not commit, push, open or edit a pull request, merge, change the Linear +issue, or modify files outside the requested slice. Report the changed paths and raw check output +to the coordinator when finished. Never ask a question: decide from the work order and repository +rules, and record any blocked sub-step in the report. +` + /** * Everything created after `orca worktree create` succeeds has to come back out on any later * failure, or a failed launch leaves a full checkout, its terminals and an `npm install` @@ -517,6 +567,39 @@ const reserveAutomationBudget = ( return { identity, engineName, tier, startedAt, ledgerPath } } +/** + * The reservation is appended before the worktree exists, so it cannot carry the PID of a worker + * that does not exist yet. Attaching it here is what lets `summarize` expire a reservation the + * instant its process is gone instead of waiting out the whole lease. A failure to attach is + * reported, never fatal: the worker is already running, and the reservation simply falls back to + * the timestamp backstop. + */ +const claimBudgetReservation = ({ identity, engineName, tier, startedAt, ledgerPath }, projectedTokens, workerPid) => { + const result = spawnSync(process.execPath, [ + budgetToolPath, + "claim", + "--identity", + identity, + "--engine", + engineName, + "--tier", + tier, + "--started-at", + startedAt, + "--ended-at", + new Date().toISOString(), + "--invocation-tokens", + String(projectedTokens), + "--worker-pid", + String(workerPid), + "--ledger", + ledgerPath, + ], { encoding: "utf8", maxBuffer: 32 * 1024 * 1024 }) + if (!result.error && result.status === 0) return true + console.error(`automation-budget claim failed, the reservation keeps its timestamp lease: ${(result.stderr || result.stdout || result.error?.message || "unknown error").trim()}`) + return false +} + cancelBudgetReservation = ({ identity, engineName, tier, startedAt, ledgerPath }) => { const result = spawnSync(process.execPath, [ budgetToolPath, @@ -623,6 +706,7 @@ const acquireConcurrencyReservation = (repoPath) => { const issue = argOf("--issue") const promptFileArg = argOf("--prompt-file") +const existingWorktreeArg = argOf("--existing-worktree") const repoOverride = argOf("--repo") const baseBranch = argOf("--base-branch") ?? "main" const branchPrefix = argOf("--branch-prefix") ?? "feature" @@ -687,12 +771,15 @@ if ( ) { fail(2, `.claude/orchestrator.json worker "${engineName}" must declare positive integer automationBudget.invocationTokens for every declared model tier: ${invocationTokenTiers.join(", ")}`) } -if (engine.interactive !== true) { +if (typeof engine.interactive !== "boolean") { fail( 2, - `.claude/orchestrator.json worker "${engineName}" does not declare interactive: true. Everything below this line assumes a supervisable TUI: the trust-prompt answer, the tui-idle poll, nudge-worker's busy refusal, worker-status' idle-then-check. A headless engine has none of that, so it launches unwatched and lands zero commits, zero gates and no PR. Declare the engine interactive only when its invocation really opens a TUI.`, + `.claude/orchestrator.json worker "${engineName}" must explicitly declare interactive as true or false; silence must not select a launch mode.`, ) } +if (engine.interactive && promptFile.length > MAX_INTERACTIVE_PROMPT_PATH_LENGTH) { + fail(2, `prompt file path is ${promptFile.length} characters; interactive terminal delivery can swallow long paths. Use a shorter path or a headless worker.`) +} if (!config.repos || typeof config.repos !== "object") { fail(2, ".claude/orchestrator.json carries no repos map; add one keyed by the repo:* label ids (ui, api, landing)") } @@ -737,11 +824,91 @@ try { fail(2, error.message) } const engineArgs = resolvedInvocation.args -const budgetTier = resolvedInvocation.tier === "deep" ? "reserved" : automationBudget.tier +const budgetTier = automationBudget.tier const projectedTokens = automationBudget.invocationTokens[resolvedInvocation.tier] const invocationStartedAt = new Date().toISOString() const invocationIdentity = `${issue}:${invocationStartedAt}:${randomUUID()}` const command = [engine.command, ...engineArgs].join(" ") +/** + * The one instruction that closes this launch's reservation. It must reach the HEADLESS pointer as + * well as the interactive one: headless is the default engine shape, and a headless worker that is + * never told to record leaves a pending row nothing ever closes, which is the stranded reservation + * measured in the production ledger on 2026-07-30. + */ +const measurementCommand = `node "${budgetToolPath}" record --identity "${invocationIdentity}" --engine ${engineName} --tier ${budgetTier} --started-at "${invocationStartedAt}" --ended-at --input-tokens --cached-input-tokens --output-tokens --ledger "${automationLedgerPath}"` +const measurementInstruction = `Before finishing, replace this launch's pending ledger record with an append carrying provider-authoritative token totals: ${measurementCommand}. --input-tokens is the provider's RAW input count and --cached-input-tokens is its cache-read share of that same count; the fuse charges the difference, so omit --cached-input-tokens only when the provider reports no cache read at all. Add --provider-estimated-cost only when the provider supplies its own estimate. If the token measurement is unavailable, leave the pending record unchanged so the next launch fails closed; never record zero or infer tokens from account usedPercent.` +const workerPointer = (worktreePath, branch) => `Read ${promptFile} and execute it in full. That file is your complete work order for ${issue}. You are on branch ${branch} in ${worktreePath}. Do not summarise the file back to me, start the work now. ${measurementInstruction}` +/** + * Resolve a bare command the way the platform's launcher does, so the result is a real file rather + * than a name Node will refuse. On win32 only PATHEXT candidates count: npm also drops an + * extensionless shell script next to the shim, and Windows cannot execute it. + */ +const resolveOnPath = (command) => { + if (command.includes("/") || command.includes("\\")) { + return existsSync(command) ? resolve(command) : null + } + const extensions = process.platform === "win32" + ? (process.env.PATHEXT ?? ".COM;.EXE;.BAT;.CMD").split(";").filter(Boolean) + : [""] + for (const directory of (process.env.PATH ?? "").split(delimiter).filter(Boolean)) { + for (const extension of extensions) { + const candidate = join(directory, `${command}${extension}`) + if (existsSync(candidate) && statSync(candidate).isFile()) return candidate + } + } + return null +} + +/** + * Node has refused to spawn a `.cmd` or `.bat` without `shell: true` since the CVE-2024-27980 fix, + * and `spawn("codex.cmd", ...)` throws EINVAL before codex ever starts. `shell: true` avoids the + * errno but hands the worker pointer to cmd.exe to re-parse, and that pointer is a positional + * prompt carrying spaces and quotes, which is the ORB-88 mangled-prompt class. So resolve the npm + * shim to the script it execs and spawn Node on that: the argv array survives with no shell in the + * path. Verified against the installed codex.cmd, whose last line is + * `"%_prog%" "%dp0%\\node_modules\\@openai\\codex\\bin\\codex.js" %*`. A shim that does not match + * that shape fails closed here rather than falling through to a spawn known to throw. + */ +const NPM_SHIM_SCRIPT = /"%dp0%\\+([^"]+\.js)"/i +const headlessInvocation = () => { + const resolved = resolveOnPath(engine.command) + if (!resolved) { + fail(3, `could not resolve the ${engineName} worker executable "${engine.command}" on PATH; a headless launch has no shell to resolve it later`) + } + if (!/\.(?:cmd|bat)$/i.test(resolved)) return { executable: resolved, scriptArgs: [] } + let shim + try { + shim = readFileSync(resolved, "utf8") + } catch (error) { + fail(3, `could not read the ${engineName} shim ${resolved}: ${error.message}`) + } + const match = shim.match(NPM_SHIM_SCRIPT) + if (!match) { + fail(3, `${resolved} is a ${extname(resolved)} shim that tools/launch-worker.mjs cannot run headlessly: Node refuses to spawn it without a shell, and no "%dp0%...js" script line was found to spawn directly. Point .claude/orchestrator.json at the executable or the script itself.`) + } + const script = resolve(dirname(resolved), match[1]) + if (!existsSync(script)) { + fail(3, `${resolved} names the script ${script}, which does not exist`) + } + return { executable: process.execPath, scriptArgs: [script] } +} + +const startHeadlessWorker = (worktreePath, branch) => { + const { executable, scriptArgs } = headlessInvocation() + const child = spawn(executable, [...scriptArgs, ...engineArgs, workerPointer(worktreePath, branch)], { + cwd: worktreePath, + detached: true, + stdio: "ignore", + windowsHide: true, + env: { ...process.env, ORBIT_LAUNCH_WORKER: "1" }, + }) + if (!child.pid) fail(3, `could not start headless ${engineName} worker`) + child.unref() + const gitDirectory = resolve(worktreePath, git(["-C", worktreePath, "rev-parse", "--git-dir"])) + appendFileSync(join(gitDirectory, "orbit-worker-pids.jsonl"), `${JSON.stringify({ issue, worktreePath, pid: child.pid, startedAt: new Date().toISOString() })}\n`) + if (budgetReservation) claimBudgetReservation(budgetReservation, projectedTokens, child.pid) + return child.pid +} /** * Second level, for an entry that declares interactive: true while carrying a headless @@ -759,9 +926,12 @@ if (!profile) { fail(2, `worker "${engineName}" runs "${binary}", which tools/launch-worker.mjs has no engine profile for. Add one to ENGINE_PROFILES naming that CLI's headless tokens (the subcommand or flag that runs it with no TUI), its first-run trust screen and the keystroke that answers it. Known: ${Object.keys(ENGINE_PROFILES).join(", ")}`) } const headless = invocationTokens.slice(1).find((token) => profile.headlessTokens.includes(token)) -if (headless) { +if (engine.interactive === true && headless) { fail(2, `worker "${engineName}" declares interactive: true but its invocation "${command}" carries "${headless}", which is a headless invocation of ${binary}. Fix the command or args, or the declaration, in .claude/orchestrator.json`) } +if (engine.interactive === false && !headless) { + fail(2, `worker "${engineName}" declares interactive: false but its invocation "${command}" has no known headless token for ${binary}`) +} const runPermissionIndex = invocationTokens.findIndex((token, index) => token === profile.runPermissionTokens[0] && profile.runPermissionTokens.every((expected, offset) => invocationTokens[index + offset] === expected)) if (runPermissionIndex === -1) { const modeIndex = profile.permissionModeToken ? invocationTokens.indexOf(profile.permissionModeToken) : -1 @@ -769,6 +939,13 @@ if (runPermissionIndex === -1) { fail(2, `worker "${engineName}" invocation "${command}" does not carry ${binary}'s required run-permitting policy "${profile.runPermissionTokens.join(" ")}"${mode}. A worker without that policy can stop for approval with nobody at the keyboard. Fix the command or args in .claude/orchestrator.json`) } +/** + * Held by BOTH launch modes, for different caps that race the same way. A new worktree races + * `maxParallelWorktrees` against `orca worktree list`; an additional slice races + * `maxSlicesPerWorker` against `orbit-worker-pids.jsonl`, which it reads, counts, checks, and + * only then appends to after spawning. Two slice launches into one worktree is the mode's whole + * purpose, so unlocked they both read before either appends and both pass a cap of one. + */ if (!dryRun) acquireConcurrencyReservation(repoPath) const listedWorktrees = orca(["worktree", "list", "--repo", `path:${repoPath}`]).worktrees if (!Array.isArray(listedWorktrees)) { @@ -783,7 +960,11 @@ const occupyingWorktrees = listedWorktrees.filter( && worktree.git?.isMainWorktree !== true && worktree.isArchived !== true, ) -if (occupyingWorktrees.length >= maxParallelWorktrees) { +const maxSlicesPerWorker = config.maxSlicesPerWorker +if (!Number.isSafeInteger(maxSlicesPerWorker) || maxSlicesPerWorker < 1) { + fail(2, ".claude/orchestrator.json must declare maxSlicesPerWorker as a positive integer") +} +if (!existingWorktreeArg && occupyingWorktrees.length >= maxParallelWorktrees) { const paths = occupyingWorktrees.map((worktree) => worktree.path ?? worktree.git?.path ?? worktree.id) fail( 1, @@ -793,7 +974,8 @@ if (occupyingWorktrees.length >= maxParallelWorktrees) { /** Reported in the plan so a dry run shows whether this launch would inject the contract, and a * relaunch against an already-injected file is visibly a no-op rather than a silent second copy. */ -const workerContract = readFileSync(promptFile, "utf8").includes(WORKER_CONTRACT_MARKER) ? "already present" : "appended" +const contractMarker = existingWorktreeArg ? SLICE_CONTRACT_MARKER : WORKER_CONTRACT_MARKER +const workerContract = readFileSync(promptFile, "utf8").includes(contractMarker) ? "already present" : "appended" const plan = { issue, @@ -839,12 +1021,36 @@ budgetReservation = reserveAutomationBudget( * for the relaunch. A dry run resolves this decision but writes nothing. */ if (workerContract === "appended") { try { - appendFileSync(promptFile, WORKER_CONTRACT, "utf8") + appendFileSync(promptFile, existingWorktreeArg ? SLICE_CONTRACT : WORKER_CONTRACT, "utf8") } catch (error) { fail(3, `could not append the worker contract to ${promptFile}: ${error.message}`) } } +if (existingWorktreeArg) { + const worktreePath = resolve(existingWorktreeArg) + if (!existsSync(worktreePath)) fail(2, `existing worktree not found: ${worktreePath}`) + if (isInside(promptFile, worktreePath)) fail(2, `prompt file lives inside the existing worktree (${worktreePath})`) + const actualRoot = git(["-C", worktreePath, "rev-parse", "--show-toplevel"]) + if (normalize(actualRoot) !== normalize(worktreePath)) fail(2, `--existing-worktree must name a Git worktree root: ${worktreePath}`) + const existing = listedWorktrees.find((worktree) => normalize(worktree.path) === normalize(worktreePath)) + if (!existing || existing.isMainWorktree || existing.isArchived || existing.linkedLinearIssue !== issue) { + fail(2, `--existing-worktree must be an active Orca worktree linked to ${issue}`) + } + const marker = join(resolve(worktreePath, git(["-C", worktreePath, "rev-parse", "--git-dir"])), "orbit-worker-pids.jsonl") + const activeSlices = existsSync(marker) + ? readFileSync(marker, "utf8").trim().split(/\r?\n/).filter(Boolean).flatMap((line) => { try { const row = JSON.parse(line); return row.issue === issue && Number.isInteger(row.pid) && (() => { try { process.kill(row.pid, 0); return true } catch (error) { return error.code !== "ESRCH" } })() ? [row] : [] } catch { return [] } }) + : [] + if (activeSlices.length >= maxSlicesPerWorker) fail(1, `maxSlicesPerWorker cap ${maxSlicesPerWorker} reached for ${issue}`) + const branch = git(["-C", worktreePath, "rev-parse", "--abbrev-ref", "HEAD"]) + const workerPid = startHeadlessWorker(worktreePath, branch) + /** Only once the new PID is in the marker file, so the next launcher counts this slice. */ + releaseConcurrencyReservation() + rollback = null + console.log(JSON.stringify({ ...plan, launchMode: "existing-worktree", worktreePath, worktreeSelector: `path:${worktreePath}`, branch, workerPid }, null, 2)) + process.exit(0) +} + console.error(`creating worktree ${worktreeName} in ${repoKey} from ${baseBranch}`) const created = orca([ "worktree", "create", @@ -876,6 +1082,14 @@ rollback.contractBranch = branch const actualBranch = git(["-C", worktreePath, "rev-parse", "--abbrev-ref", "HEAD"]) if (actualBranch !== branch) fail(3, `expected the worktree on ${branch}, found ${actualBranch}`) +if (engine.interactive === false) { + const workerPid = startHeadlessWorker(worktreePath, branch) + rollback = null + orca(["worktree", "set", "--worktree", worktreeSelector, "--comment", comment, "--workspace-status", workspaceStatus]) + console.log(JSON.stringify({ ...plan, launchMode: "new-worktree", worktreePath, worktreeSelector, workerPid }, null, 2)) + process.exit(0) +} + console.error(`starting the ${engineName} TUI: ${command}`) const terminal = createTerminal(worktreeSelector, command) @@ -891,7 +1105,7 @@ while (waitAttempts < MAX_WAIT_ATTEMPTS && !idle) { * text from a gate that is long gone and type into the worker's live composer. */ if (wait.satisfied) { - if (!busy(terminal)) { + if (!terminalIsRepainting(terminal)) { idle = true break } @@ -916,8 +1130,7 @@ if (!idle) { fail(1, `${terminal} never reached tui-idle after ${waitAttempts} waits; the worker is not running. Inspect it with: orca terminal read --terminal ${terminal}`) } -const measurementCommand = `node "${budgetToolPath}" record --identity "${invocationIdentity}" --engine ${engineName} --tier ${budgetTier} --started-at "${invocationStartedAt}" --ended-at --input-tokens --output-tokens --ledger "${automationLedgerPath}"` -const pointer = `Read ${promptFile} and execute it in full. That file is your complete work order for ${issue}: the ticket body verbatim, then the finishing contract. You are on branch ${branch} in ${worktreePath}. Do not summarise the file back to me, start the work now. Before finishing, replace this launch's pending ledger record with an append carrying provider-authoritative token totals: ${measurementCommand}. Add --provider-estimated-cost only when the provider supplies its own estimate. If the token measurement is unavailable, leave the pending record unchanged so the next launch fails closed; never record zero or infer tokens from account usedPercent.` +const pointer = `Read ${promptFile} and execute it in full. That file is your complete work order for ${issue}: the ticket body verbatim, then the finishing contract. You are on branch ${branch} in ${worktreePath}. Do not summarise the file back to me, start the work now. ${measurementInstruction}` /** * What a DELIVERED pointer looks like on screen: a send that became a user turn makes the TUI @@ -955,14 +1168,14 @@ while (pointerSends < MAX_POINTER_SENDS && !pointerDelivered) { * shape that settled once and then fell through to the top of this loop resent into a busy TUI * in precisely the case this branch exists to prevent (PR #616 review round 1). */ - painting = !pointerDelivered && busy(terminal) + painting = !pointerDelivered && terminalIsRepainting(terminal) let settles = 0 while (painting && settles < MAX_POINTER_SETTLES) { settles += 1 console.error(`the pointer is not on screen and the TUI is painting, so settling instead of sending again (${settles} of ${MAX_POINTER_SETTLES})`) pause(SETTLE_MS) pointerDelivered = pointerOnScreen() - painting = !pointerDelivered && busy(terminal) + painting = !pointerDelivered && terminalIsRepainting(terminal) } /** Still painting past the bound: there is no safe moment to re-send, so this launch is over. */ if (painting) break diff --git a/tools/lib/tui-repaint.mjs b/tools/lib/tui-repaint.mjs deleted file mode 100644 index 1fa1fd4c2..000000000 --- a/tools/lib/tui-repaint.mjs +++ /dev/null @@ -1,61 +0,0 @@ -/** - * The one place that answers "is this TUI mid-turn?", for every tool that has to know. - * - * `orca terminal wait --for tui-idle` is NOT a busy signal for every engine. Measured - * 2026-07-27 against a live codex worker mid-turn: the wait returned satisfied: true while - * the TUI was painting `Working (30s - esc to interupt)`. The terminal TEXT cannot correct - * it either, because a read keeps stale output: an IDLE codex composer still carried the - * `Starting MCP servers ... esc to interrupt` line from its own startup, so matching the - * interrupt hint refuses forever. Repaint activity separates them cleanly, and for BOTH - * engines: a running turn repaints its spinner continuously while an idle TUI emits nothing - * at all. Measured lastOutputAt advancing 2.4s to 3.7s per sample window on a busy codex and - * on a busy claude, and frozen at delta 0 on an idle one of each. - * - * PR #614 landed that measurement as two identical copies, one in launch-worker.mjs and one - * in nudge-worker.mjs, and worker-watch.mjs would have been the third. Two copies of an - * invariant drift apart silently; this module is the third-use extraction (CLAUDE.md rule 6). - * It takes the caller's own `orca` runner rather than shelling out itself, because each tool - * already owns how an orca failure ends its process, and a helper that exits is a helper that - * cannot be reused. - * - * Not a tool: it has no CLI and is never invoked directly, so it carries no `--help` and no - * `test-tools.mjs` coverage row of its own. It is exercised through the three tools that - * import it. - */ - -/** One sample window. Long enough that a spinner frame lands inside it, short enough to poll with. */ -export const REPAINT_SAMPLE_MS = 3000 - -/** A satisfied-but-repainting wait returns instantly, so a retry needs its own pause or the - * caller's allowed attempts burn in seconds while the engine is merely still starting up. */ -export const SETTLE_MS = 10000 - -export const pause = (ms) => Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms) - -/** - * True while `handle` is painting, which for a TUI worker means mid-turn. Two `terminal show` - * samples one window apart; `orca` is the caller's runner, which must return the parsed - * `result` payload. - */ -export const isRepainting = (orca, handle, sampleMs = REPAINT_SAMPLE_MS) => { - const paintedAt = () => orca(["terminal", "show", "--terminal", handle]).terminal?.lastOutputAt ?? 0 - const before = paintedAt() - pause(sampleMs) - return paintedAt() !== before -} - -/** - * The same delta across EVERY live terminal in one pair of calls, for a watcher that would - * otherwise pay two `terminal show` round trips per worker. `terminal list` carries - * lastOutputAt per terminal, so one sample covers the whole fleet. - */ -export const sampleTerminals = (orca) => - new Map((orca(["terminal", "list"]).terminals ?? []).map((terminal) => [terminal.handle, terminal.lastOutputAt ?? 0])) - -/** - * BUSY when the handle painted between the two samples. A handle present only in the second - * sample painted by definition (it did not exist for the first), so it is BUSY rather than - * silently absent from the report. - */ -export const classifyTerminals = (before, after) => - new Map([...after].map(([handle, paintedAt]) => [handle, paintedAt !== (before.get(handle) ?? null) ? "BUSY" : "IDLE"])) diff --git a/tools/nudge-worker.mjs b/tools/nudge-worker.mjs index 6f479ceff..8b823b12c 100644 --- a/tools/nudge-worker.mjs +++ b/tools/nudge-worker.mjs @@ -1,299 +1,23 @@ #!/usr/bin/env node -/** - * Deliver a message to a running TUI worker WITHOUT cutting its turn short. - * Measured 2026-07-24 on ORB-75: an `orca terminal send` issued while the worker - * was mid-turn never arrived as a user turn at all. It appears in the worker's - * session transcript only as four `type: "queue-operation"` records, the running - * turn ended on a mid-flow sentence, and the worktree was left with 14 modified - * and 7 untracked files, zero commits, zero gates and no PR. So the sanctioned - * path requires a stopped repaint signal and no live trust prompt before sending, - * and the way to hand a worker new information is to APPEND it to the prompt file - * it already has and send a one-line pointer telling it to re-read that file. - */ +/** Headless codex exec has no live user-turn channel, so every invocation fails closed. */ +const USAGE = `usage: nudge-worker.mjs -import { execFileSync, spawnSync } from "node:child_process" -import { appendFileSync, existsSync, readFileSync } from "node:fs" -import { resolve } from "node:path" + Mid-run worker injection is unavailable for headless workers. There is no live user-turn + channel to send into, so this tool sends nothing, calls orca not at all, and takes no flags: + no --terminal, no --text, no --prompt-file, no --wait-attempts, no --engine, no --dry-run. + Relaunch after exit with the updated prompt file instead. -import { readOrchestratorConfig } from "./lib/orchestrator-config.mjs" -import { SETTLE_MS, isRepainting, pause } from "./lib/tui-repaint.mjs" + --help, -h print this usage and exit 0 -const USAGE = `usage: nudge-worker.mjs --terminal (--text "" | --prompt-file < update.md) - - --terminal the worker's terminal handle, as printed by launch-worker.mjs (required) - --prompt-file the worker's prompt file. The update arrives on STDIN and is appended - to that file first; the sent text is then a one-line pointer telling the - worker to re-read it. This is the only safe way to add work mid-run. - MUST live outside every Orbit repo, exactly as at launch - --text "" send this exact one-liner instead. Newlines are rejected: a multi-line - payload through a TUI submits early and arrives quoting-damaged - --engine readiness profile override: claude or codex. Otherwise uses the - orchestrator worker. Claude has no verified readiness profile and - always refuses stale-block recovery. Missing, auto or unknown values - also fail closed - --wait-attempts how many 60s tui-idle waits to allow before refusing (default: 3) - --dry-run resolve and print what would be sent; append nothing, send nothing - --help, -h print this usage and exit 0 - -Prints one JSON object on stdout: terminal, sent, promptFile, appendedBytes, waitAttempts, -resolved engine and engine source. - -exit codes: 0 delivered, 1 the worker was busy so NOTHING was sent, 2 usage error, - 3 an orca command failed` +exit codes: 0 usage printed, 1 nothing to nudge, 2 an injection was attempted and refused` if (process.argv.includes("--help") || process.argv.includes("-h")) { console.log(USAGE) process.exit(0) } - -const ORCA = process.env.ORCA_BIN || "C:\\Users\\thoma\\AppData\\Local\\Programs\\orca\\resources\\bin\\orca" - -/** One wait is a full minute; three of them is a worker that is genuinely working, not one that is stuck. */ -const WAIT_TIMEOUT_MS = 60000 -const STALE_BLOCKED_REASON = "codex-trust-workspace" -// WHY: ORB-129 measured Codex marker/status/no-working structure against three live terminals; readiness stays absent for unmeasured engines. https://github.com/thomasluizon/orbit-ui-mobile/pull/629 -const ENGINE_PROFILES = { - claude: { - // WHY: No Claude worker ran during ORB-129, so readiness stays disabled pending captured composer screens with and without a live working indicator. https://github.com/thomasluizon/orbit-ui-mobile/pull/629 - trustOnScreen: /isthisaprojectyoucreatedoroneyoutrust|doyoutrustthefiles|trustthisfolder/, - }, - codex: { - trustOnScreen: /doyoutrustthecontentsofthisdirectory/, - composerMarker: "›", - statusOnScreen: /(?:^| )[a-z0-9][a-z0-9._/-]* (?:low|medium|high|xhigh|max|ultra) · (?:~[\\/]|[a-z]:[\\/]|\/)[^\s·]+(?: · main \[default\])?(?: ─ worked for \d+m \d+s ─+)?\s*$/, - workingOnScreen: /esctointer\w*/, - }, -} -const flatten = (text) => text.replace(/\s+/g, "").toLowerCase() - -const fail = (code, message) => { - console.error(message) - process.exit(code) -} - -const argOf = (flag) => { - const index = process.argv.indexOf(flag) - return index === -1 ? null : process.argv[index + 1] -} - -/** orca prints its `ok: false` payload on STDOUT and leaves stderr empty, so a failed call whose - * reason is only read off stderr reports "Command failed" and nothing else. Read stdout first. */ -const orcaFailureReason = (error) => { - const payload = error.stdout?.toString() ?? "" - try { - const parsed = JSON.parse(payload) - if (parsed.error?.message) return parsed.error.message - } catch { - if (payload.trim()) return payload.trim().slice(0, 400) - } - return error.stderr?.toString().trim() || error.message -} - -const orca = (args) => { - let raw - try { - raw = execFileSync(ORCA, [...args, "--json"], { encoding: "utf8", maxBuffer: 32 * 1024 * 1024 }) - } catch (error) { - fail(3, `orca ${args.join(" ")} failed: ${orcaFailureReason(error)}`) - } - let parsed - try { - parsed = JSON.parse(raw) - } catch { - fail(3, `orca ${args.join(" ")} returned unparseable output: ${raw.slice(0, 400)}`) - } - if (parsed.ok === false) fail(3, `orca ${args.join(" ")} failed: ${parsed.error?.message ?? "unknown orca error"}`) - return parsed.result ?? parsed -} - -/** - * A busy worker is the normal case here, and orca reports it as an exit-1 timeout payload - * rather than a clean "not yet". Treating that exit code as a tool failure would turn the - * guard into a crash, so read the payload instead. - */ -const waitForIdle = (handle) => { - const result = spawnSync( - ORCA, - ["terminal", "wait", "--terminal", handle, "--for", "tui-idle", "--timeout-ms", String(WAIT_TIMEOUT_MS), "--json"], - { encoding: "utf8", maxBuffer: 32 * 1024 * 1024 }, - ) - if (result.error) fail(3, `orca terminal wait failed: ${result.error.message}`) - let parsed - try { - parsed = JSON.parse(result.stdout) - } catch { - fail(3, `orca terminal wait returned unparseable output: ${(result.stdout || result.stderr || "").slice(0, 400)}`) - } - if (parsed.ok === false) { - if (parsed.error?.code === "timeout") return { satisfied: false, status: "timeout" } - fail(3, `orca terminal wait failed: ${parsed.error?.message ?? "unknown orca error"}`) - } - return parsed.result?.wait ?? {} -} - -/** Why a satisfied tui-idle wait is not enough to send on, and why the delta is measured - * instead of read off the screen: tools/lib/tui-repaint.mjs. This tool's entire reason to - * exist failed open without it. */ -const busy = (handle) => isRepainting(orca, handle) - -const screenSignals = (handle, resolvedEngine) => { - const tail = (orca(["terminal", "read", "--terminal", handle, "--limit", "60"]).terminal?.tail ?? []).join("\n") - const screen = tail.replace(/\s+/g, " ").toLowerCase() - const profile = resolvedEngine ? ENGINE_PROFILES[resolvedEngine] : null - const hasVerifiedReadiness = Boolean(profile?.composerMarker && profile?.statusOnScreen && profile?.workingOnScreen) - const composerIndex = hasVerifiedReadiness ? screen.lastIndexOf(profile.composerMarker) : -1 - const currentScreen = composerIndex === -1 ? null : screen.slice(composerIndex) - const trustScreen = flatten(currentScreen ?? screen) - const trustEngine = Object.entries(ENGINE_PROFILES).find(([, candidate]) => candidate.trustOnScreen.test(trustScreen))?.[0] ?? null - const statusStructureOnScreen = Boolean(currentScreen && profile.statusOnScreen.test(currentScreen)) - const workingOnScreen = Boolean(currentScreen && profile.workingOnScreen.test(flatten(currentScreen))) - const readyEngine = currentScreen && statusStructureOnScreen && !workingOnScreen ? resolvedEngine : null - return { trustEngine, readyEngine, hasVerifiedReadiness, currentScreenLocated: currentScreen !== null, statusStructureOnScreen, workingOnScreen } +if (process.argv.length > 2) { + console.error(`nudge-worker: mid-run injection is unavailable for headless workers; nothing was sent. Wait for process exit, then relaunch with the updated prompt file.\n\n${USAGE}`) + process.exit(2) } - -const terminal = argOf("--terminal") -const promptFileArg = argOf("--prompt-file") -const textArg = argOf("--text") -const engineOverridePresent = process.argv.includes("--engine") -const engineOverride = engineOverridePresent ? argOf("--engine") : null -const waitAttemptsAllowed = Number(argOf("--wait-attempts") ?? 3) -const dryRun = process.argv.includes("--dry-run") - -if (!terminal) fail(2, `${USAGE}\n\n--terminal is required`) -if (!promptFileArg && !textArg) fail(2, `${USAGE}\n\n--prompt-file or --text is required`) -if (promptFileArg && textArg) fail(2, "--prompt-file and --text are alternatives; pass one") -if (!Number.isInteger(waitAttemptsAllowed) || waitAttemptsAllowed < 1) fail(2, "--wait-attempts must be a positive integer") -if (textArg && /[\r\n]/.test(textArg)) fail(2, "--text must be a single line; append the long form to the prompt file instead") - -let orchestrator -try { - orchestrator = { config: readOrchestratorConfig(), error: null } -} catch (error) { - orchestrator = { config: null, error: error.message } -} -const engineSource = engineOverridePresent ? "--engine" : ".claude/orchestrator.json worker" -const engineValue = engineOverridePresent ? engineOverride : orchestrator.config?.worker -const normalizedEngine = typeof engineValue === "string" ? engineValue.trim().toLowerCase() : "" -const resolvedEngine = Object.hasOwn(ENGINE_PROFILES, normalizedEngine) ? normalizedEngine : null -const displayedEngine = normalizedEngine || "" -const readinessRefusalReason = () => { - if (!resolvedEngine) return `engine "${displayedEngine}" from ${engineSource} does not resolve to a known readiness profile` - const profile = ENGINE_PROFILES[resolvedEngine] - if (!profile.composerMarker || !profile.statusOnScreen || !profile.workingOnScreen) { - return `the ${resolvedEngine} readiness profile is unverified; enabling it requires a captured Claude Code composer screen with and without a live working indicator; see https://github.com/thomasluizon/orbit-ui-mobile/pull/629` - } - return `no known ready composer is on screen for the ${resolvedEngine} profile` -} - -const staleBlockVerdict = (handle, blockedReason) => { - if (busy(handle)) { - return { - verdict: "blocked", - message: `orca reports ${blockedReason} and the TUI is repainting, so both signals say the worker is mid-turn`, - } - } - const { trustEngine, readyEngine, hasVerifiedReadiness, currentScreenLocated, statusStructureOnScreen, workingOnScreen } = screenSignals(handle, resolvedEngine) - if (!hasVerifiedReadiness) { - return { - verdict: "blocked", - message: `orca reports ${blockedReason}, the TUI is not repainting, but ${readinessRefusalReason()}, so the worker remains blocked`, - } - } - if (!currentScreenLocated) { - const trustReason = trustEngine - ? `the ${trustEngine} trust prompt is still on screen in retained tail` - : "no known trust prompt is on screen" - return { - verdict: "blocked", - message: `orca reports ${blockedReason}, the TUI is not repainting, but the current screen region could not be located because no ${resolvedEngine} composer marker is on screen, ${trustReason}, and no known ready composer is on screen, so the worker remains blocked`, - } - } - if (trustEngine) { - return { - verdict: "blocked", - message: `orca reports ${blockedReason}, the TUI is not repainting, and the ${trustEngine} trust prompt is still on screen, so the worker remains blocked`, - } - } - if (!readyEngine) { - const missingSignal = !statusStructureOnScreen - ? "the live model, effort, separator and working-directory status structure is absent" - : "the live working indicator follows the composer marker" - return { - verdict: "blocked", - message: `orca reports ${blockedReason}, the TUI is not repainting, and no known trust prompt is on screen, but no known ready composer is on screen for the ${resolvedEngine} profile because ${missingSignal}, so the worker remains blocked`, - } - } - return { - verdict: "idle", - message: `orca reports ${blockedReason}, but the TUI is not repainting, no known trust prompt is on screen, and the ${readyEngine} ready composer is on screen with its status structure and no live working indicator, so the retained blocked reason is stale and the current screen and repaint signals win`, - } -} - -let promptFile = null -let update = "" -if (promptFileArg) { - promptFile = resolve(promptFileArg) - if (!existsSync(promptFile)) fail(2, `prompt file not found: ${promptFile}`) - if (!orchestrator.config) fail(2, orchestrator.error) - const repos = orchestrator.config.repos - const normalize = (path) => path.replaceAll("\\", "/").replace(/\/+$/, "").toLowerCase() - for (const [key, path] of Object.entries(repos ?? {})) { - if (normalize(promptFile) === normalize(path) || normalize(promptFile).startsWith(`${normalize(path)}/`)) { - fail(2, `prompt file lives inside the ${key} repo (${path}); the worker would commit the appended update. Point at the scratchpad file launch-worker.mjs was given`) - } - } - if (process.stdin.isTTY) fail(2, "--prompt-file expects the update on stdin") - update = readFileSync(0, "utf8").trim() - if (!update) fail(2, "stdin was empty; nothing to append") -} - -const text = textArg ?? `New information was appended to ${promptFile}. Re-read that file in full and continue the work order from where you are.` - -let waitAttempts = 0 -let idle = false -while (waitAttempts < waitAttemptsAllowed && !idle) { - waitAttempts += 1 - if (dryRun) { - idle = true - break - } - const wait = waitForIdle(terminal) - if (wait.status === "exited") fail(1, `${terminal} has exited; there is no worker to nudge`) - if (wait.satisfied) { - if (!busy(terminal)) { - idle = true - break - } - console.error(`attempt ${waitAttempts}: orca reports tui-idle but the TUI is still repainting, so the repaint signal wins and the worker is mid-turn`) - if (waitAttempts < waitAttemptsAllowed) pause(SETTLE_MS) - continue - } - if (wait.blockedReason === STALE_BLOCKED_REASON) { - const { verdict, message } = staleBlockVerdict(terminal, wait.blockedReason) - console.error(`attempt ${waitAttempts}: ${message}`) - if (verdict === "idle") { - idle = true - break - } - if (waitAttempts < waitAttemptsAllowed) pause(SETTLE_MS) - continue - } - console.error(`attempt ${waitAttempts}: worker is busy (${wait.blockedReason ?? wait.status ?? "not idle"})`) -} -if (!idle) { - fail( - 1, - `${terminal} is still mid-turn after ${waitAttempts} waits; NOTHING was sent. A send while busy is queued and can cut the running turn short (ORB-75, 2026-07-24). Wait for it to go idle and run this again.`, - ) -} - -let appendedBytes = 0 -if (promptFile && update) { - const block = `\n\n## Update appended ${new Date().toISOString()}\n\n${update}\n` - appendedBytes = Buffer.byteLength(block, "utf8") - if (!dryRun) appendFileSync(promptFile, block, "utf8") -} - -if (!dryRun) orca(["terminal", "send", "--terminal", terminal, "--text", text, "--enter"]) - -console.log(JSON.stringify({ terminal, sent: text, promptFile, appendedBytes, waitAttempts, engine: resolvedEngine, engineSource, dryRun }, null, 2)) +console.error("nudge-worker: mid-run injection is unavailable for headless workers; wait for process exit, then relaunch with the updated prompt file") +process.exit(1) diff --git a/tools/teardown-worktree.mjs b/tools/teardown-worktree.mjs index 570f8fb06..92f47be34 100644 --- a/tools/teardown-worktree.mjs +++ b/tools/teardown-worktree.mjs @@ -6,10 +6,9 @@ */ import { execFileSync, spawnSync } from "node:child_process" -import { existsSync } from "node:fs" -import { resolve } from "node:path" +import { existsSync, readFileSync, unlinkSync } from "node:fs" +import { join, resolve } from "node:path" -import { isRepainting } from "./lib/tui-repaint.mjs" const USAGE = `usage: teardown-worktree.mjs (--issue ORB-N | --worktree ) [--base ] @@ -113,8 +112,11 @@ const issue = worktree.linkedLinearIssue const branch = (worktree.branch ?? git(path, ["rev-parse", "--abbrev-ref", "HEAD"])).replace(/^refs\/heads\//, "") const base = requestedBase ?? worktree.baseRef ?? "main" const dirty = git(path, ["status", "--short"]).split("\n").filter(Boolean) -const terminals = (orca(["terminal", "list"]).terminals ?? []).filter((terminal) => normalize(terminal.worktreePath) === normalize(path)) -const busy = terminals.filter((terminal) => isRepainting(orca, terminal.handle)) +const workerMarker = join(resolve(path, git(path, ["rev-parse", "--git-dir"])), "orbit-worker-pids.jsonl") +const workerPids = existsSync(workerMarker) + ? readFileSync(workerMarker, "utf8").trim().split(/\r?\n/).filter(Boolean).flatMap((line) => { try { const row = JSON.parse(line); return Number.isInteger(row.pid) ? [row.pid] : [] } catch { return [] } }) + : [] +const workerAlive = workerPids.filter((pid) => { try { process.kill(pid, 0); return true } catch (error) { return error.code !== "ESRCH" } }) const detail = orca(["linear", "issue", issue]) const linearIssue = detail.issue ?? detail const state = linearIssue.state?.name ?? linearIssue.state @@ -156,7 +158,7 @@ const checks = [ { name: "worktree-clean", ok: dirty.length === 0, detail: dirty.length ? `uncommitted paths: ${dirty.join(", ")}` : "no uncommitted work" }, ...pullRequestChecks, { name: "linear-done", ok: state === "Done", detail: `issue is ${state ?? "unknown"}, expected Done` }, - { name: "terminals-idle", ok: busy.length === 0, detail: busy.length ? `worker is still working: ${busy.map((terminal) => terminal.handle).join(", ")}` : `${terminals.length} terminal(s) idle` }, + { name: "worker-pid-exited", ok: workerAlive.length === 0, detail: workerAlive.length ? `worker PID is still running: ${workerAlive.join(", ")}` : "the worker PID has exited" }, ] const unmet = checks.filter((check) => !check.ok) if (unmet.length > 0) { @@ -191,6 +193,9 @@ if (branchExists) { } const branchRemaining = gitCommon(["show-ref", "--verify", "--quiet", `refs/heads/${branch}`], { allowFailure: true }) !== null if (branchRemaining) fail(1, `removed worktree but local branch ${branch} still exists`) +if (existsSync(workerMarker)) { + try { unlinkSync(workerMarker) } catch (error) { fail(3, `could not prune worker PID marker ${workerMarker}: ${error.message}`) } +} console.log(`REMOVED worktree ${path}`) console.log(`REMOVED terminals for ${path}`) console.log(`REMOVED local branch ${branch}`) diff --git a/tools/test-tools.mjs b/tools/test-tools.mjs index 025efd8be..0deb8ff62 100644 --- a/tools/test-tools.mjs +++ b/tools/test-tools.mjs @@ -554,7 +554,7 @@ const DEFAULT_AUTOMATION_BUDGET = { }, } -const orchestratorConfig = (repoPath, worker, engineName, maxParallelWorktrees = 8) => +const orchestratorConfig = (repoPath, worker, engineName, maxParallelWorktrees = 8, maxSlicesPerWorker = 3) => JSON.stringify({ worker: engineName, workers: { @@ -564,6 +564,7 @@ const orchestratorConfig = (repoPath, worker, engineName, maxParallelWorktrees = }, }, maxParallelWorktrees, + maxSlicesPerWorker, attemptsBeforeRewrite: 2, linear: { team: "ORB", states: { working: "In Progress", review: "In Review", done: "Done" } }, repos: { ui: repoPath }, @@ -601,7 +602,7 @@ const INTERACTIVE_CODEX = { * The engine name is what the top-level `worker` key selects, which is the only way to * exercise a non-default engine: the tool has no engine-override flag by design. */ -const stageLaunchWorker = (label, worker, engineName = "claude", maxParallelWorktrees = 8) => { +const stageLaunchWorker = (label, worker, engineName = "claude", maxParallelWorktrees = 8, maxSlicesPerWorker = 3) => { const base = join(root, "launch", label) const repoPath = join(base, "repos", "ui") mkdirSync(repoPath, { recursive: true }) @@ -615,7 +616,7 @@ const stageLaunchWorker = (label, worker, engineName = "claude", maxParallelWork mkdirSync(join(base, ".claude"), { recursive: true }) writeFileSync( join(base, ".claude", "orchestrator.json"), - orchestratorConfig(repoPath, worker, engineName, maxParallelWorktrees), + orchestratorConfig(repoPath, worker, engineName, maxParallelWorktrees, maxSlicesPerWorker), ) cpSync(join(TOOLS_DIR, "launch-worker.mjs"), join(base, "tools", "launch-worker.mjs")) cpSync(join(TOOLS_DIR, "automation-budget.mjs"), join(base, "tools", "automation-budget.mjs")) @@ -631,7 +632,6 @@ console.log(JSON.stringify(result)) process.exit(result.claude?.status === "OK" || result.codex?.status === "OK" ? 0 : 1) `, ) - /** The copy imports tools/lib/tui-repaint.mjs by relative path, so the staged tree carries it too. */ cpSync(join(TOOLS_DIR, "lib"), join(base, "tools", "lib"), { recursive: true }) return { path: join(base, "tools", "launch-worker.mjs"), repoPath, base } } @@ -736,29 +736,6 @@ const preflightEnv = (plan) => ({ DOTNET_BIN: process.execPath, }) -const stageNudgeWorker = (label, worker, instrumentPause = false) => { - const base = join(root, "nudge", label) - mkdirSync(join(base, "tools"), { recursive: true }) - mkdirSync(join(base, ".claude"), { recursive: true }) - writeFileSync( - join(base, ".claude", "orchestrator.json"), - JSON.stringify({ worker, maxParallelWorktrees: 8, repos: {} }), - ) - cpSync(join(TOOLS_DIR, "nudge-worker.mjs"), join(base, "tools", "nudge-worker.mjs")) - cpSync(join(TOOLS_DIR, "lib"), join(base, "tools", "lib"), { recursive: true }) - if (instrumentPause) { - cpSync(join(TOOLS_DIR, "lib", "tui-repaint.mjs"), join(base, "tools", "lib", "tui-repaint-real.mjs")) - writeFileSync( - join(base, "tools", "lib", "tui-repaint.mjs"), - `import { appendFileSync } from "node:fs" -export { SETTLE_MS, isRepainting } from "./tui-repaint-real.mjs" -export const pause = (ms) => appendFileSync(process.env.ORBIT_PAUSE_LOG, String(ms) + "\\n") -`, - ) - } - return { path: join(base, "tools", "nudge-worker.mjs"), base } -} - const launchWorktreeStub = (path, isMainWorktree = false) => ({ id: path, path, @@ -802,7 +779,8 @@ const REQUIRED_CONTRACT_CLAUSES = { "leaving human threads unresolved": /Never resolve a thread opened by a human account/, "refusing completion with unresolved threads": /approval with an[\s\S]*unresolved[\s\S]*thread is not done/, "watching only its own ticket": /Never watch another[\s\S]*ticket, worktree, or PR/, - "arming a monitor that outlives the contract": /Never arm a background monitor/, + "arming a detached monitor that outlives the contract": /Never arm a detached background monitor/, + "permitting an affordable foreground blocking wait": /foreground blocking wait is permitted/, "merging or pushing to main": /Never merge any PR, never push to/, "blanket staging that sweeps in a sibling's artifacts": /Stage explicitly[\s\S]*git add -A/, "pushing a commit it has not read back": /Verify before pushing[\s\S]*git show --stat HEAD/, @@ -1010,7 +988,7 @@ const pointerDeliveryCases = () => { "launch-worker.mjs: the worker receives its launcher-owned authoritative completion-record command", firstPointer.includes(`node "${join(first.staged.base, "tools", "automation-budget.mjs")}" record`) && !firstPointer.includes("node tools/automation-budget.mjs record") && - /automation-budget\.mjs" record[\s\S]*--identity "ORB-75:[^"]+"[\s\S]*--input-tokens [\s\S]*--ledger "[^"]+"[\s\S]*never record zero or infer tokens from account usedPercent/.test(firstPointer) && + /automation-budget\.mjs" record[\s\S]*--identity "ORB-75:[^"]+"[\s\S]*--input-tokens --cached-input-tokens [\s\S]*--ledger "[^"]+"[\s\S]*the fuse charges the difference[\s\S]*never record zero or infer tokens from account usedPercent/.test(firstPointer) && firstPointer.includes(`--ledger "${firstPlan?.automationBudget?.ledgerPath}"`), firstPointer, ) @@ -1542,6 +1520,11 @@ const launchWorkerCases = async () => { const insidePrompt = join(good.repoPath, "prompt.md") writeFileSync(insidePrompt, "the ticket body verbatim\n") check("launch-worker.mjs", "refuses a prompt file inside a repo", ["--issue", "ORB-75", "--prompt-file", insidePrompt, "--dry-run"], { status: 2, stderr: /would be committed/ }, { path: good.path, env: orcaEnv(linearIssueStub(["repo:ui"])) }) + const longPromptDirectory = join(root, "prompt-path-guard", "x".repeat(130)) + mkdirSync(longPromptDirectory, { recursive: true }) + const longPrompt = join(longPromptDirectory, "prompt.md") + writeFileSync(longPrompt, "the ticket body verbatim\n") + check("launch-worker.mjs", "interactive delivery refuses a conservatively over-long prompt path", ["--issue", "ORB-75", "--prompt-file", longPrompt, "--dry-run"], { status: 2, stderr: /interactive terminal delivery can swallow long paths/ }, { path: good.path, env: orcaEnv(linearIssueStub(["repo:ui"])) }) const noModels = stageLaunchWorker("no-models", { command: "claude", args: ["--permission-mode", "bypassPermissions"], interactive: true }) check("launch-worker.mjs", "refuses an engine with no models map", ["--issue", "ORB-75", "--prompt-file", promptFile, "--dry-run"], { status: 2, stderr: /claude[\s\S]*models/ }, { path: noModels.path, env: orcaEnv(linearIssueStub(["repo:ui"])) }) @@ -1577,10 +1560,10 @@ const launchWorkerCases = async () => { ) const notInteractive = stageLaunchWorker("not-interactive", { ...INTERACTIVE_WORKER, interactive: false }) - check("launch-worker.mjs", "refuses an engine declaring interactive: false", ["--issue", "ORB-75", "--prompt-file", promptFile, "--dry-run"], { status: 2, stderr: /does not declare interactive: true/ }, { path: notInteractive.path, env: orcaEnv(linearIssueStub(["repo:ui"])) }) + check("launch-worker.mjs", "interactive false without a headless token is refused", ["--issue", "ORB-75", "--prompt-file", promptFile, "--dry-run"], { status: 2, stderr: /interactive: false[\s\S]*no known headless token/ }, { path: notInteractive.path, env: orcaEnv(linearIssueStub(["repo:ui"])) }) const omitted = stageLaunchWorker("omits-interactive", { command: "claude", args: [], models: CLAUDE_MODELS }) - check("launch-worker.mjs", "refuses an engine that omits interactive entirely", ["--issue", "ORB-75", "--prompt-file", promptFile, "--dry-run"], { status: 2, stderr: /does not declare interactive: true/ }, { path: omitted.path, env: orcaEnv(linearIssueStub(["repo:ui"])) }) + check("launch-worker.mjs", "refuses an engine that omits interactive entirely", ["--issue", "ORB-75", "--prompt-file", promptFile, "--dry-run"], { status: 2, stderr: /must explicitly declare interactive/ }, { path: omitted.path, env: orcaEnv(linearIssueStub(["repo:ui"])) }) const missingBudgetTier = stageLaunchWorker("missing-budget-tier", { ...INTERACTIVE_WORKER, automationBudget: {} }) check( @@ -1677,9 +1660,9 @@ const launchWorkerCases = async () => { ) const codexDeep = check( "launch-worker.mjs", - "tier:deep selects Sol at high effort and the reserved budget on Codex", + "tier:deep selects Sol at high effort with the routine budget on Codex", ["--issue", "ORB-75", "--prompt-file", promptFile, "--dry-run"], - { status: 0, stdout: /codex[\s\S]*model_reasoning_effort[\s\S]*high[\s\S]*--model gpt-5\.6-sol[\s\S]*"automationBudget":\s*\{[\s\S]*"tier":\s*"reserved"[\s\S]*"tokenBudget":\s*1000000[\s\S]*"warningTokens":\s*800000[\s\S]*"projectedTokens":\s*250000/ }, + { status: 0, stdout: /codex[\s\S]*model_reasoning_effort[\s\S]*high[\s\S]*--model gpt-5\.6-sol[\s\S]*"automationBudget":\s*\{[\s\S]*"tier":\s*"routine"[\s\S]*"tokenBudget":\s*1000000[\s\S]*"warningTokens":\s*800000[\s\S]*"projectedTokens":\s*250000/ }, { path: codex.path, env: orcaEnv(linearIssueStub(["repo:ui", "tier:deep"])) }, ) const codexDefaultCommand = codexPlan.status === 0 ? JSON.parse(codexPlan.stdout).command : "" @@ -1709,6 +1692,10 @@ const launchWorkerCases = async () => { const codexExecAlias = stageLaunchWorker("codex-exec-alias", { ...INTERACTIVE_CODEX, args: ["e"] }, "codex") check("launch-worker.mjs", "refuses codex e, the documented alias for exec", ["--issue", "ORB-75", "--prompt-file", promptFile, "--dry-run"], { status: 2, stderr: /headless invocation of codex/ }, { path: codexExecAlias.path, env: orcaEnv(linearIssueStub(["repo:ui"])) }) + const headlessCodex = stageLaunchWorker("headless-codex", { ...INTERACTIVE_CODEX, args: ["exec", "--dangerously-bypass-approvals-and-sandbox"], interactive: false }, "codex") + check("launch-worker.mjs", "accepts codex exec when interactive false agrees with its headless token", ["--issue", "ORB-75", "--prompt-file", promptFile, "--dry-run"], { status: 0, stdout: /codex exec/ }, { path: headlessCodex.path, env: orcaEnv(linearIssueStub(["repo:ui"])) }) + check("launch-worker.mjs", "rejects a headless declaration without a headless token", ["--issue", "ORB-75", "--prompt-file", promptFile, "--dry-run"], { status: 2, stderr: /has no known headless token/ }, { path: stageLaunchWorker("headless-without-token", { ...INTERACTIVE_CODEX, interactive: false }, "codex").path, env: orcaEnv(linearIssueStub(["repo:ui"])) }) + const unknownEngine = stageLaunchWorker("unknown-engine", { command: "aider", args: [], models: CLAUDE_MODELS, interactive: true }, "aider") check("launch-worker.mjs", "refuses an engine with no quota reader rather than waving it through", ["--issue", "ORB-75", "--prompt-file", promptFile, "--dry-run"], { status: 2, stderr: /has no quota reader/ }, { path: unknownEngine.path, env: orcaEnv(linearIssueStub(["repo:ui"])) }) @@ -1731,7 +1718,7 @@ const launchWorkerCases = async () => { const appendFailure = stageLaunchWorker("contract-append-failure", INTERACTIVE_WORKER) const appendFailureSource = readFileSync(appendFailure.path, "utf8") - const appendCall = 'appendFileSync(promptFile, WORKER_CONTRACT, "utf8")' + const appendCall = 'appendFileSync(promptFile, existingWorktreeArg ? SLICE_CONTRACT : WORKER_CONTRACT, "utf8")' if (!appendFailureSource.includes(appendCall)) { throw new Error("launch-worker fixture could not locate the worker-contract append") } @@ -1809,6 +1796,185 @@ const launchWorkerCases = async () => { `exit ${pendingResult.status}\n ${pendingResult.stderr}\n ${pendingCalls}`, ) + /** + * The REAL spawn, never --dry-run. A dry run returns before startHeadlessWorker, which is how a + * headless launcher that could not start anything reached CI twice: Node has refused to spawn a + * .cmd without a shell since CVE-2024-27980, so `spawn("codex.cmd", ...)` throws EINVAL before + * the engine exists. The win32 fixture is an npm shim of the same shape as the installed + * codex.cmd (`"%dp0%\\...js" %*`, read off disk, not invented), and the child reports its own + * argv back so a shell re-parse of the prompt would be visible rather than silent. + */ + const headlessEngineDirectory = join(root, "launch", "headless-bin") + mkdirSync(headlessEngineDirectory, { recursive: true }) + const headlessArgvLog = join(root, "launch", "headless-argv.json") + const headlessScript = join(headlessEngineDirectory, "worker-shim.js") + writeFileSync( + headlessScript, + `const { writeFileSync } = require("node:fs")\nwriteFileSync(process.env.ORBIT_HEADLESS_ARGV_LOG, JSON.stringify(process.argv.slice(2)))\nconst holdMilliseconds = Number(process.env.ORBIT_HEADLESS_HOLD_MS || 0)\nif (holdMilliseconds > 0) Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, holdMilliseconds)\n`, + ) + if (process.platform === "win32") { + writeFileSync( + join(headlessEngineDirectory, "codex.cmd"), + `@ECHO off\r\nGOTO start\r\n:find_dp0\r\nSET dp0=%~dp0\r\nEXIT /b\r\n:start\r\nSETLOCAL\r\nCALL :find_dp0\r\nSET "_prog=node"\r\nendLocal & goto #_undefined_# 2>NUL || title %COMSPEC% & "%_prog%" "%dp0%\\worker-shim.js" %*\r\n`, + ) + } else { + const posixShim = join(headlessEngineDirectory, "codex") + writeFileSync(posixShim, `#!/usr/bin/env node\nrequire(${JSON.stringify(headlessScript)})\n`) + chmodSync(posixShim, 0o755) + } + const headlessWorker = { + command: "codex", + args: ["exec", "-c", 'windows.sandbox="unelevated"', "--dangerously-bypass-approvals-and-sandbox"], + models: CODEX_MODELS, + interactive: false, + automationBudget: DEFAULT_AUTOMATION_BUDGET, + } + const headlessStage = stageLaunchWorker("headless-spawn", headlessWorker, "codex") + const headlessCheckout = stageCheckout(headlessStage.base) + if (!headlessCheckout) { + T("launch-worker.mjs: a headless launch starts a real worker process", false, "could not stage the headless launch checkout") + } else { + const headlessPrompt = stage("headless-launch-prompt.md", "the ticket body verbatim\n") + const headlessResult = run("launch-worker.mjs", ["--issue", "ORB-75", "--prompt-file", headlessPrompt], { + path: headlessStage.path, + env: { + ...orcaEnv([ + ...linearIssueStub(["repo:ui"]), + { match: "worktree create", stdout: JSON.stringify({ ok: true, result: { worktree: { path: headlessCheckout, branch: "refs/heads/thomasluizon/orb-75" } } }) }, + { match: "worktree set", stdout: JSON.stringify({ ok: true, result: {} }) }, + { match: "worktree rm", stdout: JSON.stringify({ ok: true, result: {} }) }, + ]), + PATH: `${headlessEngineDirectory}${delimiter}${process.env.PATH}`, + ORBIT_AUTOMATION_BUDGET_LEDGER: join(headlessStage.base, "automation-budget.jsonl"), + ORBIT_HEADLESS_ARGV_LOG: headlessArgvLog, + }, + }) + let headlessPlan = null + try { + headlessPlan = JSON.parse(headlessResult.stdout) + } catch { + headlessPlan = null + } + const headlessDeadline = Date.now() + 15_000 + while (!existsSync(headlessArgvLog) && Date.now() < headlessDeadline) { + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 50) + } + const childArgv = existsSync(headlessArgvLog) ? JSON.parse(readFileSync(headlessArgvLog, "utf8")) : null + const markerPath = join( + resolve(headlessCheckout, spawnSync("git", ["-C", headlessCheckout, "rev-parse", "--git-dir"], { encoding: "utf8" }).stdout.trim()), + "orbit-worker-pids.jsonl", + ) + const markerRows = existsSync(markerPath) + ? readFileSync(markerPath, "utf8").trim().split(/\r?\n/).filter(Boolean).map((line) => JSON.parse(line)) + : [] + T( + "launch-worker.mjs: a headless launch starts a real worker process and records its PID", + headlessResult.status === 0 && + headlessPlan?.launchMode === "new-worktree" && + Number.isInteger(headlessPlan?.workerPid) && + markerRows.length === 1 && + markerRows[0].pid === headlessPlan.workerPid && + markerRows[0].worktreePath === headlessCheckout, + `exit ${headlessResult.status}\n stdout: ${headlessResult.stdout.slice(0, 400)}\n stderr: ${headlessResult.stderr.slice(0, 600)}\n marker: ${JSON.stringify(markerRows)}`, + ) + const headlessLedger = join(headlessStage.base, "automation-budget.jsonl") + const headlessRows = existsSync(headlessLedger) + ? readFileSync(headlessLedger, "utf8").trim().split(/\r?\n/).filter(Boolean).map((line) => JSON.parse(line)) + : [] + T( + "launch-worker.mjs: the launcher claims its reservation with the PID it just spawned", + headlessRows.length === 2 && + headlessRows[0]?.pending === true && + !Object.hasOwn(headlessRows[0] ?? {}, "workerPid") && + headlessRows[1]?.pending === true && + headlessRows[1]?.workerPid === headlessPlan?.workerPid, + `plan pid ${headlessPlan?.workerPid}\n ledger: ${JSON.stringify(headlessRows)}`, + ) + const expectedEngineArgs = [...headlessWorker.args, ...CODEX_MODELS.default.args, "--model", CODEX_MODELS.default.model] + T( + "launch-worker.mjs: the headless worker receives its engine args and the whole pointer as one argument", + Array.isArray(childArgv) && + childArgv.length === expectedEngineArgs.length + 1 && + expectedEngineArgs.every((argument, index) => childArgv[index] === argument) && + childArgv.at(-1).includes(headlessPrompt) && + childArgv.at(-1).includes('automation-budget.mjs" record') && + childArgv.at(-1).includes("--cached-input-tokens"), + `expected ${JSON.stringify(expectedEngineArgs)} plus one pointer + child argv: ${JSON.stringify(childArgv)}`, + ) + } + + /** + * The slice cap races the same way the worktree cap does, and the whole point of + * --existing-worktree is two slices in one worktree. The cap is read from + * orbit-worker-pids.jsonl, checked, and only appended to after the spawn, so unlocked both + * launchers read before either appends and both pass a cap of one. Two real concurrent + * processes are the only thing that can prove the lock; a serial pair passes either way. + */ + const sliceStage = stageLaunchWorker("slice-cap", headlessWorker, "codex", 8, 1) + const sliceWorktree = stageCheckout(sliceStage.base) + if (!sliceWorktree) { + T("launch-worker.mjs: concurrent slice launches cannot both pass one slice cap", false, "could not stage the slice checkout") + } else { + const slicePrompt = stage("slice-cap-prompt.md", "the ticket body verbatim\n") + const sliceRunner = stage( + "slice-cap-runner.mjs", + `import { spawn } from "node:child_process" +const [tool, promptFile, worktreePath] = process.argv.slice(2) +const run = () => new Promise((resolve) => { + const child = spawn(process.execPath, [tool, "--issue", "ORB-75", "--prompt-file", promptFile, "--existing-worktree", worktreePath], { + stdio: ["ignore", "pipe", "pipe"], + }) + let stderr = "" + child.stderr.setEncoding("utf8") + child.stderr.on("data", (chunk) => { stderr += chunk }) + child.stdout.resume() + child.on("exit", (status) => resolve({ status, stderr })) +}) +process.stdout.write(JSON.stringify(await Promise.all([run(), run()]))) +`, + ) + const sliceEnv = { + ...orcaEnv([ + ...linearIssueStub(["repo:ui"], [{ ...launchWorktreeStub(sliceWorktree), isArchived: false, linkedLinearIssue: "ORB-75" }]), + { match: "worktree set", stdout: JSON.stringify({ ok: true, result: {} }) }, + ]), + PATH: `${headlessEngineDirectory}${delimiter}${process.env.PATH}`, + ORBIT_AUTOMATION_BUDGET_LEDGER: join(sliceStage.base, "automation-budget.jsonl"), + ORBIT_HEADLESS_ARGV_LOG: join(root, "launch", "slice-argv.json"), + ORBIT_HEADLESS_HOLD_MS: "6000", + } + const sliceResult = spawnSync(process.execPath, [sliceRunner, sliceStage.path, slicePrompt, sliceWorktree], { + encoding: "utf8", + cwd: REPO_ROOT, + env: { ...process.env, ...sliceEnv }, + timeout: 120_000, + }) + let sliceOutcomes = [] + try { + sliceOutcomes = JSON.parse(sliceResult.stdout) + } catch { + sliceOutcomes = [] + } + const sliceStatuses = sliceOutcomes.map((outcome) => outcome.status).sort((first, second) => first - second) + const sliceMarker = join( + resolve(sliceWorktree, spawnSync("git", ["-C", sliceWorktree, "rev-parse", "--git-dir"], { encoding: "utf8" }).stdout.trim()), + "orbit-worker-pids.jsonl", + ) + const sliceRows = existsSync(sliceMarker) + ? readFileSync(sliceMarker, "utf8").trim().split(/\r?\n/).filter(Boolean).map((line) => JSON.parse(line)) + : [] + T( + "launch-worker.mjs: concurrent slice launches cannot both pass one slice cap", + sliceStatuses.length === 2 && + sliceStatuses[0] === 0 && + sliceStatuses[1] === 1 && + sliceOutcomes.some((outcome) => /maxSlicesPerWorker cap 1 reached for ORB-75/.test(outcome.stderr)) && + sliceRows.length === 1, + `runner exit ${sliceResult.status}\n statuses ${JSON.stringify(sliceStatuses)}\n marker ${JSON.stringify(sliceRows)}\n stderr ${sliceOutcomes.map((outcome) => (outcome.stderr ?? "").trim().split("\n").slice(-2).join(" | ")).join("\n ")}`, + ) + } + const concurrentWorker = { ...INTERACTIVE_WORKER, automationBudget: { @@ -1908,8 +2074,8 @@ process.stdout.write(JSON.stringify(await Promise.all([first, second]))) "launch-worker.mjs: concurrent launchers share one atomic pre-worktree reservation", concurrentResult.status === 0 && concurrentOutcomes[0]?.status === 0 && - concurrentOutcomes[1]?.status === 3 && - /lack input or output tokens/.test(concurrentOutcomes[1]?.stderr ?? "") && + concurrentOutcomes[1]?.status === 4 && + /blocked:/.test(concurrentOutcomes[1]?.stderr ?? "") && createdWorktree(firstCalls) && !createdWorktree(secondCalls) && concurrentRecords.length === 1 && @@ -1938,11 +2104,10 @@ process.stdout.write(JSON.stringify(await Promise.all([first, second]))) ? readFileSync(reservedLog, "utf8").trim().split(/\r?\n/).filter(Boolean).map((line) => JSON.parse(line)) : [] T( - "launch-worker.mjs: tier:deep uses its reserved tier and 250000-token projection", - reservedResult.status === 3 && - /reserved invocation[\s\S]*proceeds with 1200000 projected tokens/.test(reservedResult.stderr) && - /worktree create[\s\S]*failed: stop after reserved budget/.test(reservedResult.stderr) && - reservedCalls.some((argumentsList) => argumentsList[0].split(/[\\/]/).pop() === "worktree" && argumentsList[1] === "create"), + "launch-worker.mjs: tier:deep is blocked by the routine token budget before worktree creation", + reservedResult.status === 4 && + /blocked:[\s\S]*projected spend 1200000 tokens/.test(reservedResult.stderr) && + !reservedCalls.some((argumentsList) => argumentsList[0].split(/[\\/]/).pop() === "worktree" && argumentsList[1] === "create"), `exit ${reservedResult.status}\n ${reservedResult.stderr}\n ${reservedCalls}`, ) @@ -2377,211 +2542,6 @@ const preflightCases = () => { ) } -const TIMEOUT_PAYLOAD = JSON.stringify({ ok: false, error: { code: "timeout", message: "condition not met in time" } }) -const BUSY_STUB = [{ match: "terminal wait", stdout: TIMEOUT_PAYLOAD, exit: 1 }] -const BROKEN_STUB = [{ match: "terminal wait", stdout: JSON.stringify({ ok: false, error: { code: "no-such-terminal", message: "unknown handle" } }), exit: 1 }] -const STALE_BLOCKED_WAIT = JSON.stringify({ ok: true, result: { wait: { satisfied: false, status: "running", blockedReason: "codex-trust-workspace" } } }) -const DOCUMENTED_CODEX_BLOCKED_WAIT = JSON.stringify({ ok: true, result: { wait: { satisfied: false, status: "running", blockedReason: "codex-interactive-prompt" } } }) -const CODEX_READY_PLACEHOLDER_CASES = [ - ["explain-codebase", "› Explain this codebase"], - ["review-changes", "› Run /review on my current changes"], - ["write-tests", "› Write tests for @filename"], - ["list-skills", "› Use /skills to list available skills"], -] -/** - * WHY: Captured 2026-07-28 from three live Codex composers. Placeholder text rotates, while - * every ready region carries model, effort, separator and working-directory structure. - * https://github.com/thomasluizon/orbit-ui-mobile/pull/629 - * - * › Run /review on my current changes gpt-5.6-sol high · ~\orca\workspaces\orbit-ui-mobile\orb-106-... · Main [default] - * › Improve documentation in @filename gpt-5.6-sol high · ~\orca\workspaces\orbit-ui-mobile\orb-113-... - * › Explain this codebase gpt-5.6-sol high · ~\orca\workspaces\orbit-ui-mobile\orb-122-... · Main [default] - */ -const CODEX_STATUS_STRUCTURE = "gpt-5.6-sol high · ~\\orca\\workspaces\\orbit-ui-mobile\\orb-129-nudge-worker-is-unreachable-when-orca · Main [default]" -const MEASURED_CODEX_READY_TAIL = [ - "Working (52s · esc to interrupt)", - "a · Main [default]", - "", - "─ Worked for 11m 02s ─────────────────────────────────────────────────────────── › Explain this codebase gpt-5.6-sol high · ~\\orca\\workspaces\\orbit-ui-mobile\\orb-129-nudge-worker-is-unreachable-when-orca · Main [default]", -] -const MEASURED_CODEX_WORKING_TAIL = [ - ...MEASURED_CODEX_READY_TAIL, - "(7s • esc to interrupt)", -] -const LIVE_CODEX_SAMPLE_CASES = [ - ["term-0c6e56a7-idle", "recognizes the first live idle composer shape", [ - "a · Main [default]", - "› Improve documentation in @filename", - "gpt-5.6-sol high · ~\\orca\\workspaces\\orbit-ui-mobile\\orb-129-nudge-worker-is-unreachable-when-orca", - "─ Worked for 10m 03s ───────────────────────────────────────────────────────────", - ], true], - ["term-65aa37cd-busy", "refuses the live busy composer shape", [ - "a · Main [default]", - "› Improve documentation in @filename", - CODEX_STATUS_STRUCTURE, - "(7s • esc to interrupt)", - ], false], - ["term-652dd931-idle", "recognizes the second live idle composer shape", [ - "› Use /skills to list available skills", - CODEX_STATUS_STRUCTURE, - ], true], -] -/** A settled TUI emits nothing, so lastOutputAt is the SAME on both samples. */ -const IDLE_STUB = [ - { match: "terminal wait", stdout: JSON.stringify({ ok: true, result: { wait: { satisfied: true } } }), exit: 0 }, - { match: "terminal show", stdout: JSON.stringify({ ok: true, result: { terminal: { lastOutputAt: 1785168487585 } } }), exit: 0 }, - { match: "terminal send", stdout: JSON.stringify({ ok: true, result: {} }), exit: 0 }, -] -const staleBlockedIdleStub = (tail) => [ - { match: "terminal wait", stdout: STALE_BLOCKED_WAIT, exit: 0 }, - { match: "terminal show", stdout: JSON.stringify({ ok: true, result: { terminal: { lastOutputAt: 1785168487585 } } }), exit: 0 }, - { match: "terminal read", stdout: JSON.stringify({ ok: true, result: { terminal: { tail } } }), exit: 0 }, - { match: "terminal send", stdout: JSON.stringify({ ok: true, result: {} }), exit: 0 }, -] -const WORKING_COMPOSER_IDLE_STUB = staleBlockedIdleStub([ - "› Explain this codebase", - CODEX_STATUS_STRUCTURE, - "Working (52s · esc to interrupt)", -]) -const MISSPELLED_WORKING_COMPOSER_IDLE_STUB = staleBlockedIdleStub([ - "› Explain this codebase", - CODEX_STATUS_STRUCTURE, - "Working (52s · esc to interupt)", -]) -const LIVE_BLOCKED_IDLE_STUB = [ - { match: "terminal wait", stdout: STALE_BLOCKED_WAIT, exit: 0 }, - { match: "terminal show", stdout: JSON.stringify({ ok: true, result: { terminal: { lastOutputAt: 1785168487585 } } }), exit: 0 }, - { match: "terminal read", stdout: JSON.stringify({ ok: true, result: { terminal: { tail: ["Doyoutrustthecontents", "ofthisdirectory?"] } } }), exit: 0 }, - { match: "terminal send", stdout: JSON.stringify({ ok: true, result: {} }), exit: 0 }, -] -const ANSWERED_TRUST_BEFORE_READY_TAIL = [ - "Do you trust the contents of this directory?", - "Trust once and continue", - "› Explain this codebase", - CODEX_STATUS_STRUCTURE, -] -const LIVE_TRUST_AFTER_COMPOSER_TAIL = [ - "› Explain this codebase", - CODEX_STATUS_STRUCTURE, - "Do you trust the contents of this directory?", -] -const RETAINED_COMPOSER_STATIC_SCREEN_TAIL = [ - "› Run /review on my current changes", - "Permission required", - "Allow this command?", - "[y] Yes [n] No", -] -const RETAINED_READY_STATIC_SCREEN_TAIL = [ - "› Run /review on my current changes", - CODEX_STATUS_STRUCTURE, - "Permission required", - "Allow this command?", - "[y] Yes [n] No", -] -const ALTERNATE_MODEL_READY_TAIL = [ - "› Explain this codebase", - "orbit-coder.v2 ultra · C:\\worktrees\\orbit-ui-mobile\\orb-129 · Main [default]", -] -const UNRECOGNIZED_BLOCKED_IDLE_STUB = [ - { match: "terminal wait", stdout: STALE_BLOCKED_WAIT, exit: 0 }, - { match: "terminal show", stdout: JSON.stringify({ ok: true, result: { terminal: { lastOutputAt: 1785168487585 } } }), exit: 0 }, - { match: "terminal read", stdout: JSON.stringify({ ok: true, result: { terminal: { tail: ["Allow this command?", "[y] Yes [n] No"] } } }), exit: 0 }, - { match: "terminal send", stdout: JSON.stringify({ ok: true, result: {} }), exit: 0 }, -] -/** - * The measured codex failure: orca reports tui-idle while the worker is mid-turn. The stub - * says satisfied AND repaints (lastOutputAt is stamped fresh on every call), which is exactly - * what a running turn looks like. A send here is the ORB-75 corruption, so this must refuse. - */ -const FALSE_IDLE_STUB = [ - { match: "terminal wait", stdout: JSON.stringify({ ok: true, result: { wait: { satisfied: true } } }), exit: 0 }, - { match: "terminal show", stdout: '{"ok":true,"result":{"terminal":{"lastOutputAt":__NOW__}}}', exit: 0 }, - { match: "terminal send", stdout: JSON.stringify({ ok: true, result: {} }), exit: 0 }, -] -const BLOCKED_BUSY_STUB = [ - { match: "terminal wait", stdout: STALE_BLOCKED_WAIT, exit: 0 }, - { match: "terminal show", stdout: '{"ok":true,"result":{"terminal":{"lastOutputAt":__NOW__}}}', exit: 0 }, - { match: "terminal send", stdout: JSON.stringify({ ok: true, result: {} }), exit: 0 }, -] -const DOCUMENTED_CODEX_BLOCKED_IDLE_STUB = [ - { match: "terminal wait", stdout: DOCUMENTED_CODEX_BLOCKED_WAIT, exit: 0 }, - { match: "terminal send", stdout: JSON.stringify({ ok: true, result: {} }), exit: 0 }, -] - -const runNudgeSignalCase = (label, name, plan, expect, expectedSends, options = {}) => { - const log = join(root, `nudge-${label}.log`) - check("nudge-worker.mjs", name, ["--terminal", "t1", "--text", "hi", "--wait-attempts", String(options.waitAttempts ?? 1), ...(options.argv ?? [])], expect, { - path: options.path, - env: { ...orcaEnv(plan), ...(options.env ?? {}), ORBIT_ORCA_LOG: log }, - }) - const calls = existsSync(log) ? readFileSync(log, "utf8").trim().split("\n").filter(Boolean).map((line) => JSON.parse(line)) : [] - const sends = calls.filter((argv) => argv[0].split(/[\\/]/).pop() === "terminal" && argv[1] === "send").length - T(`nudge-worker.mjs: ${name} sends ${expectedSends} time(s)`, sends === expectedSends, `sent ${sends} time(s)`) -} - -const nudgeWorkerCases = () => { - check("nudge-worker.mjs", "--help documents the engine override and fail-closed rule", ["--help"], { status: 0, stdout: /--engine [\s\S]*Claude has no verified readiness profile[\s\S]*Missing, auto or unknown values[\s\S]*fail closed/ }) - check("nudge-worker.mjs", "rejects multi-line text", ["--terminal", "t1", "--text", "first line\nsecond line"], { status: 2, stderr: /single line/ }) - check("nudge-worker.mjs", "rejects --text together with --prompt-file", ["--terminal", "t1", "--text", "hi", "--prompt-file", stage("nudge-prompt.md", "body\n")], { status: 2, stderr: /alternatives/ }) - check("nudge-worker.mjs", "rejects a non-positive --wait-attempts", ["--terminal", "t1", "--text", "hi", "--wait-attempts", "0"], { status: 2, stderr: /positive integer/ }) - check("nudge-worker.mjs", "refuses to send while the worker is busy", ["--terminal", "t1", "--text", "hi", "--wait-attempts", "1"], { status: 1, stderr: /NOTHING was sent/ }, { env: orcaEnv(BUSY_STUB) }) - check("nudge-worker.mjs", "an orca failure that is not a timeout is a tool error", ["--terminal", "t1", "--text", "hi", "--wait-attempts", "1"], { status: 3, stderr: /unknown handle/ }, { env: orcaEnv(BROKEN_STUB) }) - runNudgeSignalCase("both-idle", "sends once both signals say the worker is idle", IDLE_STUB, { status: 0, stdout: /"sent": "hi"/ }, 1, { path: stageNudgeWorker("both-idle", "codex").path }) - for (const [label, placeholder] of CODEX_READY_PLACEHOLDER_CASES) { - const readyTail = ["Worked for 13m 01s", "PR opened and issue moved to In Review", placeholder, CODEX_STATUS_STRUCTURE] - runNudgeSignalCase(`stale-block-${label}`, `trusts the codex ready composer structure with ${placeholder}`, staleBlockedIdleStub(readyTail), { status: 0, stdout: /"sent": "hi"/, stderr: /codex-trust-workspace[\s\S]*not repainting[\s\S]*no known trust prompt[\s\S]*codex ready composer is on screen[\s\S]*blocked reason is stale[\s\S]*screen and repaint signals win/ }, 1, { path: stageNudgeWorker(`stale-block-${label}`, "codex").path }) - } - runNudgeSignalCase("retained-composer-static-screen", "refuses a retained composer marker followed by a static permission screen", staleBlockedIdleStub(RETAINED_COMPOSER_STATIC_SCREEN_TAIL), { status: 1, stderr: /no known ready composer is on screen[\s\S]*NOTHING was sent/ }, 0, { path: stageNudgeWorker("retained-composer-static-screen", "codex").path }) - runNudgeSignalCase("retained-ready-static-screen", "refuses a retained composer and status followed by a static permission screen", staleBlockedIdleStub(RETAINED_READY_STATIC_SCREEN_TAIL), { status: 1, stderr: /no known ready composer is on screen[\s\S]*NOTHING was sent/ }, 0, { path: stageNudgeWorker("retained-ready-static-screen", "codex").path }) - runNudgeSignalCase("measured-ready-composer", "trusts the measured idle codex tail despite a historical working indicator", staleBlockedIdleStub(MEASURED_CODEX_READY_TAIL), { status: 0, stdout: /"sent": "hi"/, stderr: /codex ready composer is on screen/ }, 1, { path: stageNudgeWorker("measured-ready-composer", "codex").path }) - runNudgeSignalCase("measured-working-composer", "refuses the measured codex tail with a live working indicator after the composer", staleBlockedIdleStub(MEASURED_CODEX_WORKING_TAIL), { status: 1, stderr: /no known ready composer is on screen[\s\S]*NOTHING was sent/ }, 0, { path: stageNudgeWorker("measured-working-composer", "codex").path }) - runNudgeSignalCase("alternate-model-ready-composer", "recognizes structural status with a different codex model and effort", staleBlockedIdleStub(ALTERNATE_MODEL_READY_TAIL), { status: 0, stdout: /"sent": "hi"/, stderr: /codex ready composer is on screen/ }, 1, { path: stageNudgeWorker("alternate-model-ready-composer", "codex").path }) - for (const [label, name, tail, ready] of LIVE_CODEX_SAMPLE_CASES) { - const expect = ready - ? { status: 0, stdout: /"sent": "hi"/, stderr: /codex ready composer is on screen/ } - : { status: 1, stderr: /no known ready composer is on screen[\s\S]*NOTHING was sent/ } - runNudgeSignalCase(label, name, staleBlockedIdleStub(tail), expect, ready ? 1 : 0, { path: stageNudgeWorker(label, "codex").path }) - } - runNudgeSignalCase("answered-trust-before-ready", "ignores answered trust text before the current codex composer", staleBlockedIdleStub(ANSWERED_TRUST_BEFORE_READY_TAIL), { status: 0, stdout: /"sent": "hi"/, stderr: /codex ready composer is on screen[\s\S]*blocked reason is stale/ }, 1, { path: stageNudgeWorker("answered-trust-before-ready", "codex").path }) - runNudgeSignalCase("live-trust-after-composer", "refuses a live trust prompt after the current codex composer", staleBlockedIdleStub(LIVE_TRUST_AFTER_COMPOSER_TAIL), { status: 1, stderr: /codex trust prompt is still on screen[\s\S]*worker remains blocked[\s\S]*NOTHING was sent/ }, 0, { path: stageNudgeWorker("live-trust-after-composer", "codex").path }) - runNudgeSignalCase("trust-without-composer", "fails closed when a trust prompt has no current composer region", LIVE_BLOCKED_IDLE_STUB, { status: 1, stderr: /current screen region could not be located[\s\S]*no codex composer marker[\s\S]*codex trust prompt is still on screen in retained tail[\s\S]*NOTHING was sent/ }, 0, { path: stageNudgeWorker("trust-without-composer", "codex").path }) - const codexProfile = stageNudgeWorker("codex-profile", "codex") - const incidentalGreaterThanTail = [ - "› Working on the nudge predicate", - "(8s • esc to interrupt)", - "> quoted output painted after the working indicator", - ] - runNudgeSignalCase("codex-incidental-greater-than", "does not let incidental greater-than output select the claude profile for a codex worker", staleBlockedIdleStub(incidentalGreaterThanTail), { status: 1, stderr: /no known ready composer is on screen for the codex profile[\s\S]*NOTHING was sent/ }, 0, { path: codexProfile.path }) - runNudgeSignalCase("explicit-claude-profile", "fails closed for the explicitly selected unverified claude profile", staleBlockedIdleStub(incidentalGreaterThanTail), { status: 1, stderr: /claude readiness profile is unverified[\s\S]*captured Claude Code composer screen with and without a live working indicator[\s\S]*pull\/629[\s\S]*NOTHING was sent/ }, 0, { path: codexProfile.path, argv: ["--engine", "claude"] }) - const autoProfile = stageNudgeWorker("auto-profile", "auto") - runNudgeSignalCase("auto-profile", "fails closed when the orchestrator worker is auto", staleBlockedIdleStub(["› Explain this codebase"]), { status: 1, stderr: /engine "auto" from \.claude\/orchestrator\.json worker does not resolve[\s\S]*NOTHING was sent/ }, 0, { path: autoProfile.path }) - const unknownProfile = stageNudgeWorker("unknown-profile", "future-engine") - runNudgeSignalCase("unknown-profile", "fails closed when the orchestrator worker is unknown", staleBlockedIdleStub(["› Explain this codebase"]), { status: 1, stderr: /engine "future-engine" from \.claude\/orchestrator\.json worker does not resolve[\s\S]*NOTHING was sent/ }, 0, { path: unknownProfile.path }) - runNudgeSignalCase("unknown-engine-override", "fails closed when the engine override is unknown", staleBlockedIdleStub(["› Explain this codebase"]), { status: 1, stderr: /engine "future-engine" from --engine does not resolve[\s\S]*NOTHING was sent/ }, 0, { path: unknownProfile.path, argv: ["--engine", "future-engine"] }) - const missingProfile = stageNudgeWorker("missing-profile", undefined) - runNudgeSignalCase("missing-profile", "fails closed when the orchestrator worker is missing", staleBlockedIdleStub(["› Explain this codebase"]), { status: 1, stderr: /engine "" from \.claude\/orchestrator\.json worker does not resolve[\s\S]*NOTHING was sent/ }, 0, { path: missingProfile.path }) - const claudeProfile = stageNudgeWorker("claude-profile", "claude") - runNudgeSignalCase("configured-claude-profile", "fails closed for the configured unverified claude profile", staleBlockedIdleStub(incidentalGreaterThanTail), { status: 1, stderr: /claude readiness profile is unverified[\s\S]*captured Claude Code composer screen with and without a live working indicator[\s\S]*pull\/629[\s\S]*NOTHING was sent/ }, 0, { path: claudeProfile.path }) - runNudgeSignalCase("engine-override", "--engine overrides a disagreeing orchestrator worker", staleBlockedIdleStub(["› Explain this codebase", CODEX_STATUS_STRUCTURE]), { status: 0, stdout: /"engine": "codex"[\s\S]*"engineSource": "--engine"/ }, 1, { path: claudeProfile.path, argv: ["--engine", "codex"] }) - const pauseProbe = stageNudgeWorker("pause-probe", "codex", true) - const pauseLog = join(pauseProbe.base, "pause.log") - runNudgeSignalCase("trust-prompt-pause", "settles before retrying a trust prompt that remains on screen", LIVE_BLOCKED_IDLE_STUB, { status: 1, stderr: /attempt 1:[\s\S]*trust prompt is still on screen[\s\S]*attempt 2:[\s\S]*trust prompt is still on screen[\s\S]*NOTHING was sent/ }, 0, { - path: pauseProbe.path, - waitAttempts: 2, - env: { ORBIT_PAUSE_LOG: pauseLog }, - }) - const pauses = existsSync(pauseLog) ? readFileSync(pauseLog, "utf8").trim().split("\n") : [] - T("nudge-worker.mjs: trust prompt retry applies one settle pause", pauses.length === 1 && pauses[0] === "10000", `pause log: ${JSON.stringify(pauses)}`) - runNudgeSignalCase("working-composer", "refuses a ready-looking codex composer carrying esc to interrupt", WORKING_COMPOSER_IDLE_STUB, { status: 1, stderr: /no known ready composer is on screen[\s\S]*NOTHING was sent/ }, 0, { path: stageNudgeWorker("working-composer", "codex").path }) - runNudgeSignalCase("misspelled-working-composer", "refuses a ready-looking codex composer carrying esc to interupt", MISSPELLED_WORKING_COMPOSER_IDLE_STUB, { status: 1, stderr: /no known ready composer is on screen[\s\S]*NOTHING was sent/ }, 0, { path: stageNudgeWorker("misspelled-working-composer", "codex").path }) - runNudgeSignalCase("live-block", "refuses a static trust prompt that is still on screen", LIVE_BLOCKED_IDLE_STUB, { status: 1, stderr: /codex-trust-workspace[\s\S]*not repainting[\s\S]*codex trust prompt is still on screen[\s\S]*remains blocked[\s\S]*NOTHING was sent/ }, 0, { path: stageNudgeWorker("live-block", "codex").path }) - runNudgeSignalCase("unrecognized-block", "refuses an unrecognized static screen with no ready composer signal", UNRECOGNIZED_BLOCKED_IDLE_STUB, { status: 1, stderr: /codex-trust-workspace[\s\S]*not repainting[\s\S]*no known trust prompt[\s\S]*no known ready composer is on screen[\s\S]*worker remains blocked[\s\S]*NOTHING was sent/ }, 0, { path: stageNudgeWorker("unrecognized-block", "codex").path }) - runNudgeSignalCase("false-idle", "refuses a tui-idle that is still repainting, which is a worker mid-turn", FALSE_IDLE_STUB, { status: 1, stderr: /tui-idle[\s\S]*still repainting[\s\S]*repaint signal wins[\s\S]*NOTHING was sent/ }, 0, { path: stageNudgeWorker("false-idle", "codex").path }) - runNudgeSignalCase("both-busy", "refuses when both signals say the worker is busy", BLOCKED_BUSY_STUB, { status: 1, stderr: /codex-trust-workspace[\s\S]*TUI is repainting[\s\S]*both signals[\s\S]*NOTHING was sent/ }, 0, { path: stageNudgeWorker("both-busy", "codex").path }) - runNudgeSignalCase("documented-codex-reason", "does not treat codex-interactive-prompt as the measured stale reason", DOCUMENTED_CODEX_BLOCKED_IDLE_STUB, { status: 1, stderr: /worker is busy \(codex-interactive-prompt\)[\s\S]*NOTHING was sent/ }, 0, { path: stageNudgeWorker("documented-codex-reason", "codex").path }) - check("nudge-worker.mjs", "--dry-run calls orca not at all", ["--terminal", "t1", "--text", "hi", "--dry-run"], { status: 0, stdout: /"dryRun": true/ }, { env: orcaEnv([]) }) -} - /** * pr-watch cases. Every one is a state the two hand-rolled ORB-88 loops got wrong, so the * regression they pin is "the watcher went back to sleep with the answer on screen". @@ -2848,101 +2808,6 @@ const prWatchCases = () => { check("pr-watch.mjs", "refuses a repo that is not an owner\\/name slug", ["--repo", "orbit-ui-mobile", "--pr", "615", "--once"], { status: 2, stderr: /owner\/name slug/ }) } -/** - * worker-watch cases. The liveness half is the whole point: a single terminal read cannot tell a - * busy worker from an idle one, and a busy worker's tail is thousands of characters of - * concatenated repaint fragments that hide whatever it last really said. - */ -const workerWatchCases = () => { - const terminalHandle = "term_ca852374-175d-42cd-8407-b579a03cc13a" - const childWorktree = (path) => ({ - path, - repoId: "r-ui", - projectId: "github:thomasluizon/orbit-ui-mobile", - isMainWorktree: false, - isArchived: false, - branch: "refs/heads/feature/orb-75-prove-the-harness-gate", - linkedLinearIssue: "ORB-75", - baseRef: "main", - comment: "ORB-75 launched: worker running", - }) - const linearState = { - match: "linear issue ORB-75", - stdout: JSON.stringify({ ok: true, result: { issue: { identifier: "ORB-75", state: { name: "In Progress" }, labels: [] } } }), - } - const fleet = (path, { lastOutputAt, tail }) => [ - { match: "worktree list", stdout: JSON.stringify({ ok: true, result: { worktrees: [childWorktree(path)] } }) }, - { - match: "terminal list", - stdout: JSON.stringify({ ok: true, result: { terminals: [{ handle: terminalHandle, worktreePath: path, title: "Claude Code", lastOutputAt: 0 }] } }).replace( - '"lastOutputAt":0', - `"lastOutputAt":${lastOutputAt}`, - ), - }, - { match: "terminal read", stdout: JSON.stringify({ ok: true, result: { terminal: { tail } } }) }, - linearState, - ] - - check( - "worker-watch.mjs", - "an empty fleet says so rather than printing nothing", - [], - { status: 0, stdout: /no Orca worktrees/ }, - { env: orcaEnv([{ match: "worktree list", stdout: JSON.stringify({ ok: true, result: { worktrees: [] } }) }]) }, - ) - - /** __NOW__ is stamped per stub call, so the two samples differ: a TUI painting its spinner. */ - const busy = check( - "worker-watch.mjs", - "a repainting terminal is BUSY, and its repaint tail yields no output lines", - ["--no-contract"], - { status: 0, stdout: /BUSY\s+ORB-75/ }, - { - env: orcaEnv( - fleet("C:/wt/orb-75", { - lastOutputAt: "__NOW__", - tail: ["WorkingWorkingWorkingWorking (12s - esc to interupt)WorkingWorking", " ⠋ ⠙ ⠹ ⠸ "], - }), - ), - }, - ) - T("worker-watch.mjs: the repaint tail is stripped to nothing rather than printed raw", /nothing but repaint noise/.test(busy.stdout), busy.stdout.slice(0, 400)) - T("worker-watch.mjs: the ticket's Linear state is reported alongside liveness", /In Progress/.test(busy.stdout), busy.stdout.slice(0, 400)) - T( - "worker-watch.mjs: the rendered terminal handle is complete and directly reusable", - busy.stdout.includes(`${terminalHandle} BUSY`) && !/term_ca852374\s+BUSY/.test(busy.stdout), - busy.stdout.slice(0, 400), - ) - - /** A frozen lastOutputAt is a settled TUI: identical samples, so IDLE. */ - const idle = check( - "worker-watch.mjs", - "two identical samples are IDLE, and real content survives the stripping", - [], - { status: 0, stdout: /IDLE\s+ORB-75/ }, - { - env: orcaEnv( - fleet(root, { - lastOutputAt: "1785168487585", - tail: ["Working (30s - esc to interupt)", "Wrote tools/pr-watch.mjs", "Which of these two approaches do you want?"], - }), - ), - }, - ) - T( - "worker-watch.mjs: the last meaningful lines survive, so a worker stopped on a question is readable", - /Wrote tools\/pr-watch\.mjs/.test(idle.stdout) && /Which of these two approaches/.test(idle.stdout), - idle.stdout.slice(0, 400), - ) - T( - "worker-watch.mjs: an unreadable contract verdict is reported, never silently dropped", - /contract\s+unavailable/.test(idle.stdout), - `worker-status ran against a non-repo path, so the verdict must degrade visibly\n ${idle.stdout.slice(0, 400)}`, - ) - check("worker-watch.mjs", "refuses a repo outside orchestrator.json", ["--repo", "zzz"], { status: 2, stderr: /--repo must be one of/ }) - check("worker-watch.mjs", "refuses a non-positive --lines", ["--lines", "0"], { status: 2, stderr: /positive integer/ }) -} - /** A linked child checkout is the smallest real Git fixture that can prove teardown verification. */ const stageTeardownWorktree = (label, { dirty = false, changed = false, squashMerged = false, fastForwardMerged = false, serverMerged = false, localFollowUp = false, localFollowUpMerged = false, siblingTargetAdvance = false, branchDeleteMode } = {}) => { const primary = join(root, "teardown", label, "primary") @@ -3189,15 +3054,34 @@ const teardownWorktreeRecord = (fixture) => ({ const mergedPullRequest = (fixture, number = 124) => ({ number, mergedAt: "2026-07-28T12:00:00Z", mergeCommit: { oid: fixture.mergeCommit }, headRefOid: fixture.headCommit }) const missingTargetPullRequest = (fixture) => ({ ...mergedPullRequest(fixture), mergeCommit: { oid: fixture.headCommit } }) -const teardownPlan = (fixture, { state = "Done", terminals = [], pullRequest = mergedPullRequest(fixture), pullRequestOutput = JSON.stringify(pullRequest ? [pullRequest] : []), pullRequestExit = 0, removePath, removal = JSON.stringify({ ok: true, result: {} }), removalExit = 0 } = {}) => [ +const teardownPlan = (fixture, { state = "Done", pullRequest = mergedPullRequest(fixture), pullRequestOutput = JSON.stringify(pullRequest ? [pullRequest] : []), pullRequestExit = 0, removePath, removal = JSON.stringify({ ok: true, result: {} }), removalExit = 0 } = {}) => [ { match: "worktree list", stdout: JSON.stringify({ ok: true, result: { worktrees: [teardownWorktreeRecord(fixture)] } }) }, - { match: "terminal list", stdout: JSON.stringify({ ok: true, result: { terminals } }) }, { match: "linear issue ORB-124", stdout: JSON.stringify({ ok: true, result: { issue: { identifier: "ORB-124", state: { name: state } } } }) }, { match: "pr list --head feature/orb-124-teardown --base main --state merged --limit 1 --json number,mergeCommit,headRefOid,mergedAt", stdout: pullRequestOutput, exit: pullRequestExit }, { match: "terminal stop", stdout: JSON.stringify({ ok: true, result: {} }) }, { match: "worktree rm", stdout: removal, exit: removalExit, ...(removePath ? { removePath } : {}) }, ] +/** + * A worker's liveness is its launcher-written PID, not a terminal repaint: headless workers own + * no terminal to repaint. `pid` must be a process this suite can prove alive or dead, so a live + * case uses the harness's own PID and a dead case uses a probe process that has already exited. + */ +const stageWorkerPidMarker = (worktreePath, pid) => { + const gitDirectory = resolve( + worktreePath, + spawnSync("git", ["-C", worktreePath, "rev-parse", "--git-dir"], { encoding: "utf8" }).stdout.trim(), + ) + const marker = join(gitDirectory, "orbit-worker-pids.jsonl") + writeFileSync(marker, `${JSON.stringify({ issue: "ORB-124", worktreePath, pid, startedAt: "2026-07-30T00:00:00.000Z" })}\n`) + return marker +} + +const exitedProbePid = () => { + const probe = spawnSync(process.execPath, ["-e", "process.stdout.write(String(process.pid))"], { encoding: "utf8" }) + return probe.status === 0 ? Number(probe.stdout) : Number.NaN +} + const teardownWorktreeCases = () => { check("teardown-worktree.mjs", "refuses no selector", [], { status: 2, stderr: /provide exactly one selector/ }) check("teardown-worktree.mjs", "refuses both selectors", ["--issue", "ORB-124", "--worktree", "path:C:/other"], { status: 2, stderr: /provide exactly one selector/ }) @@ -3254,21 +3138,19 @@ const teardownWorktreeCases = () => { ) T("teardown-worktree.mjs: verified removal actually deleted the fixture", !existsSync(allGood.child), unavailable.stderr) - const missingTerminalPath = stageTeardownWorktree("missing-terminal-path") + const exitedWorker = stageTeardownWorktree("exited-worker") + const exitedWorkerMarker = stageWorkerPidMarker(exitedWorker.child, exitedProbePid()) check( "teardown-worktree.mjs", - "ignores another fleet terminal without a worktree path", + "a worker PID that has exited is torn down", ["--issue", "ORB-124"], { status: 0, stdout: /REMOVED worktree/ }, - { - env: orcaEnv([ - ...teardownPlan(missingTerminalPath, { - terminals: [{ handle: "term_other_worktree", title: "other worktree" }, { handle: "term_target", worktreePath: missingTerminalPath.child, title: "target worktree" }], - removePath: missingTerminalPath.child, - }), - { match: "terminal show", stdout: JSON.stringify({ ok: true, result: { terminal: { lastOutputAt: 1 } } }) }, - ]), - }, + { env: orcaEnv(teardownPlan(exitedWorker, { removePath: exitedWorker.child })) }, + ) + T( + "teardown-worktree.mjs: teardown prunes the worker PID marker it verified", + !existsSync(exitedWorkerMarker), + `marker still present at ${exitedWorkerMarker}`, ) const dirty = stageTeardownWorktree("dirty", { dirty: true }) @@ -3285,18 +3167,16 @@ const teardownWorktreeCases = () => { check("teardown-worktree.mjs", "an unreadable merge commit refuses with exit 3", ["--issue", "ORB-124"], { status: 3, stderr: /UNMET merge-commit-in-target: could not read pull request #124's merge commit/ }, { env: orcaEnv(teardownPlan(unreadableMergeCommit, { pullRequest: { ...mergedPullRequest(unreadableMergeCommit), mergeCommit: { oid: "0000000000000000000000000000000000000001" } } })) }) const lookupFailure = stageTeardownWorktree("lookup-failure", { dirty: true }) + stageWorkerPidMarker(lookupFailure.child, process.pid) const lookupFailureLog = join(root, "teardown", "lookup-failure.log") check( "teardown-worktree.mjs", "a failed merged-commit lookup reports every independent refusal", ["--issue", "ORB-124"], - { status: 3, stderr: /UNMET worktree-clean: uncommitted paths: (?:\?\? )?dirty\.txt[\s\S]*UNMET pull-request-merged: gh pr list for feature\/orb-124-teardown failed[\s\S]*UNMET linear-done: issue is In Review, expected Done[\s\S]*UNMET terminals-idle: worker is still working/ }, + { status: 3, stderr: /UNMET worktree-clean: uncommitted paths: (?:\?\? )?dirty\.txt[\s\S]*UNMET pull-request-merged: gh pr list for feature\/orb-124-teardown failed[\s\S]*UNMET linear-done: issue is In Review, expected Done[\s\S]*UNMET worker-pid-exited: worker PID is still running/ }, { env: { - ...orcaEnv([ - ...teardownPlan(lookupFailure, { state: "In Review", terminals: [{ handle: "term_busy", worktreePath: lookupFailure.child }], pullRequest: null, pullRequestExit: 1, removePath: lookupFailure.child }), - { match: "terminal show", sequence: [JSON.stringify({ ok: true, result: { terminal: { lastOutputAt: 1 } } }), JSON.stringify({ ok: true, result: { terminal: { lastOutputAt: 2 } } })] }, - ]), + ...orcaEnv(teardownPlan(lookupFailure, { state: "In Review", pullRequest: null, pullRequestExit: 1, removePath: lookupFailure.child })), ORBIT_ORCA_LOG: lookupFailureLog, }, }, @@ -3309,18 +3189,16 @@ const teardownWorktreeCases = () => { check("teardown-worktree.mjs", "a merged-commit lookup with malformed JSON refuses", ["--issue", "ORB-124"], { status: 3, stderr: /gh pr list for feature\/orb-124-teardown returned unparseable output/ }, { env: orcaEnv(teardownPlan(malformedPullRequestPayload, { pullRequestOutput: "not-json" })) }) const notMerged = stageTeardownWorktree("not-merged", { dirty: true }) + stageWorkerPidMarker(notMerged.child, process.pid) const notMergedLog = join(root, "teardown", "not-merged.log") check( "teardown-worktree.mjs", "an unmerged pull request reports every independent refusal", ["--issue", "ORB-124"], - { status: 1, stderr: /UNMET worktree-clean: uncommitted paths: (?:\?\? )?dirty\.txt[\s\S]*UNMET pull-request-merged: pull request for feature\/orb-124-teardown is not a merged pull request with merge and head commits[\s\S]*UNMET linear-done: issue is In Review, expected Done[\s\S]*UNMET terminals-idle: worker is still working/ }, + { status: 1, stderr: /UNMET worktree-clean: uncommitted paths: (?:\?\? )?dirty\.txt[\s\S]*UNMET pull-request-merged: pull request for feature\/orb-124-teardown is not a merged pull request with merge and head commits[\s\S]*UNMET linear-done: issue is In Review, expected Done[\s\S]*UNMET worker-pid-exited: worker PID is still running/ }, { env: { - ...orcaEnv([ - ...teardownPlan(notMerged, { state: "In Review", terminals: [{ handle: "term_busy", worktreePath: notMerged.child }], pullRequest: null, removePath: notMerged.child }), - { match: "terminal show", sequence: [JSON.stringify({ ok: true, result: { terminal: { lastOutputAt: 1 } } }), JSON.stringify({ ok: true, result: { terminal: { lastOutputAt: 2 } } })] }, - ]), + ...orcaEnv(teardownPlan(notMerged, { state: "In Review", pullRequest: null, removePath: notMerged.child })), ORBIT_ORCA_LOG: notMergedLog, }, }, @@ -3341,22 +3219,20 @@ const teardownWorktreeCases = () => { const notDone = stageTeardownWorktree("not-done") check("teardown-worktree.mjs", "a closed-looking but non-Done Linear issue is refused", ["--issue", "ORB-124"], { status: 1, stderr: /linear-done[\s\S]*In Review/ }, { env: orcaEnv(teardownPlan(notDone, { state: "In Review", removePath: notDone.child })) }) - const repainting = stageTeardownWorktree("repainting") - const log = join(root, "teardown", "repainting.log") + const stillRunning = stageTeardownWorktree("still-running") + const stillRunningMarker = stageWorkerPidMarker(stillRunning.child, process.pid) + const stillRunningLog = join(root, "teardown", "still-running.log") check( "teardown-worktree.mjs", - "a repainting terminal is refused because the worker is still working", + "a worker PID that is still running is refused because the worker is still working", ["--issue", "ORB-124"], - { status: 1, stderr: /terminals-idle[\s\S]*worker is still working/ }, - { - env: { - ...orcaEnv([ - ...teardownPlan(repainting, { terminals: [{ handle: "term_busy", worktreePath: repainting.child }] }), - { match: "terminal show", sequence: [JSON.stringify({ ok: true, result: { terminal: { lastOutputAt: 1 } } }), JSON.stringify({ ok: true, result: { terminal: { lastOutputAt: 2 } } })] }, - ]), - ORBIT_ORCA_LOG: log, - }, - }, + { status: 1, stderr: new RegExp(`worker-pid-exited[\\s\\S]*worker PID is still running: ${process.pid}`) }, + { env: { ...orcaEnv(teardownPlan(stillRunning)), ORBIT_ORCA_LOG: stillRunningLog } }, + ) + T( + "teardown-worktree.mjs: a refused teardown leaves the worker PID marker in place", + existsSync(stillRunningMarker) && existsSync(stillRunning.child), + `marker ${existsSync(stillRunningMarker)}, worktree ${existsSync(stillRunning.child)}`, ) const survives = stageTeardownWorktree("survives") @@ -3539,6 +3415,7 @@ const orchestrateFlagCases = () => { .split("\0") .filter((path) => /\.(md|mjs|json|ya?ml|txt)$/i.test(path)) .filter((path) => path.replaceAll("\\", "/") !== `tools/${SELF}`) + .filter((path) => existsSync(join(REPO_ROOT, path))) : [] const oneTicketSingle = [ /\/orchestrate\s+ORB-(?:N|\d+)\s+--single/, @@ -5193,15 +5070,15 @@ const automationBudgetCases = () => { .map((value) => value === "routine" ? "reserved" : value), ) T( - "automation-budget.mjs: a reserved deep invocation may consume the exact remaining token budget", + "automation-budget.mjs: an invocation may consume the exact remaining token budget", exactBudget.status === 0 && /warning[\s\S]*1000 tokens/.test(exactBudget.stderr), `exit ${exactBudget.status}\n stdout: ${exactBudget.stdout}\n stderr: ${exactBudget.stderr}`, ) check( "automation-budget.mjs", - "explicitly reserved deep work proceeds beyond the routine token budget", + "every invocation blocks when it exceeds the token budget", checkArgs("deep-over-budget", ledgerBlock, 250, ["--json"]).map((value) => value === "routine" ? "reserved" : value), - { status: 0, stdout: /"status":"RESERVED"[\s\S]*"projectedTokens":1151/, stderr: /reserved invocation[\s\S]*proceeds with 1151 projected tokens[\s\S]*budget 1000 tokens/ }, + { status: 4, stdout: /"status":"BLOCK"[\s\S]*"projectedTokens":1151/, stderr: /blocked:[\s\S]*projected spend 1151 tokens/ }, ) check( @@ -5302,13 +5179,12 @@ const automationBudgetCases = () => { ) check( "automation-budget.mjs", - "explicitly reserved deep work proceeds with a warning while another measurement is absent", + "an unmeasured record still fails closed for every invocation tier", checkArgs("reserved-after-pending", pendingLedger, 100, ["--json"]) .map((value) => value === "routine" ? "reserved" : value), { - status: 0, - stdout: /"status":"RESERVED"[\s\S]*"missingIdentities":\["pending-invocation"\]/, - stderr: /warning: reserved invocation "reserved-after-pending" proceeds[\s\S]*missing measurements for identities pending-invocation/, + status: 3, + stderr: /lack input or output tokens[\s\S]*pending-invocation/, }, ) const correctedLedger = stage( @@ -5322,6 +5198,427 @@ const automationBudgetCases = () => { { status: 0, stdout: /"projectedTokens":600[\s\S]*"totalTokens":500[\s\S]*"missingIdentities":\[\]/ }, ) + /** + * Cache reads are recorded and never charged. Measured on the ORB-153 launch: raw input was + * 5,681,754 tokens of which 5,399,808 were cache reads, so the raw figure blocks a 1,000,000 + * budget and the uncached figure proceeds. Every fixture here is sized so the two answers + * differ, which is the only way the assertion can fail when the subtraction is dropped. + */ + const cachedLedger = stage( + "budget/cached-input.jsonl", + `${JSON.stringify({ + identity: "cache-heavy", + engine: "claude", + tier: "routine", + startedAt: "2030-01-02T09:00:00.000Z", + endedAt: "2030-01-02T10:00:00.000Z", + inputTokens: 900, + cachedInputTokens: 850, + outputTokens: 20, + })}\n`, + ) + check( + "automation-budget.mjs", + "cache reads are recorded but never counted as spend", + checkArgs("after-cache-heavy", cachedLedger, 100, ["--json"]), + { + status: 0, + stdout: /"status":"PROCEED"[\s\S]*"projectedTokens":170[\s\S]*"inputTokens":50,"outputTokens":20,"totalTokens":70,"routineTokens":70/, + }, + ) + check( + "automation-budget.mjs", + "record keeps the raw provider input alongside its cache-read share", + [ + "record", + "--identity", + "cache-round-trip", + "--engine", + "claude", + "--tier", + "routine", + "--started-at", + "2030-01-02T09:00:00Z", + "--ended-at", + "2030-01-02T10:00:00Z", + "--input-tokens", + "900", + "--cached-input-tokens", + "850", + "--output-tokens", + "20", + "--ledger", + stage("budget/cache-round-trip.jsonl", ""), + "--json", + ], + { status: 0, stdout: /"inputTokens":900,"cachedInputTokens":850,"outputTokens":20/ }, + ) + check( + "automation-budget.mjs", + "a cache-read count without its raw input is refused rather than assumed", + [ + "record", + "--identity", + "cache-without-input", + "--engine", + "claude", + "--tier", + "routine", + "--started-at", + "2030-01-02T09:00:00Z", + "--ended-at", + "2030-01-02T10:00:00Z", + "--cached-input-tokens", + "850", + "--output-tokens", + "20", + "--ledger", + stage("budget/cache-without-input.jsonl", ""), + ], + { status: 2, stderr: /--cached-input-tokens requires --input-tokens and cannot exceed it/ }, + ) + check( + "automation-budget.mjs", + "a ledger row claiming more cache reads than raw input is rejected", + checkArgs( + "after-impossible-cache", + stage( + "budget/impossible-cache.jsonl", + `${JSON.stringify({ + identity: "impossible-cache", + engine: "claude", + tier: "routine", + startedAt: "2030-01-02T09:00:00.000Z", + endedAt: "2030-01-02T10:00:00.000Z", + inputTokens: 100, + cachedInputTokens: 101, + outputTokens: 20, + })}\n`, + ), + ), + { status: 3, stderr: /cachedInputTokens must not exceed inputTokens/ }, + ) + + /** + * The reservation lease. Every row below is built relative to the wall clock, because the + * lease is the only rule in this tool that reads it, and both of its edges have to hold: + * inside the lease a reservation still holds budget, past it the row releases itself. The + * unmeasured rows copy the exact shape the production ledger carries for a reservation + * written before `reserve` persisted `pending`: no pending key, no reserved figure, no + * tokens. That shape is why the fuse refused every codex launch for a full week. + */ + /** + * ABSOLUTE ages, never an offset derived from the tool's own constants. A fixture aged + * relative to the compiled-in lease can never fail when that lease moves, it moves with it, + * which is how raising the unclaimed lease from two hours to sixteen re-poisoned the real + * production ledger with the whole suite still green. These four hold the two arms between + * fixed walls: change either constant far enough to re-break a four hour old legacy row, or + * to expire a fourteen hour session that is still running, and a case goes red. + */ + const HOUR_MILLISECONDS = 60 * 60 * 1000 + const leaseResetAt = new Date(Date.now() + 24 * 60 * 60 * 1000).toISOString() + const leaseCheckArgs = (identity, ledger, invocationTokens) => [ + "check", + "--engine", + "codex", + "--identity", + identity, + "--tier", + "routine", + "--reset-at", + leaseResetAt, + "--warning-tokens", + "800000", + "--budget-tokens", + "1000000", + "--invocation-tokens", + String(invocationTokens), + "--ledger", + ledger, + "--json", + ] + /** + * The committed production row, read off disk rather than retyped, so the SHAPE this suite + * defends is the one a pre-C1 `reserve` actually wrote: no pending, no reservedTokens, no + * token measurements, no workerPid. Only the timestamps are moved, to the absolute age each + * case names in its own label. + */ + const LEGACY_FIXTURE = JSON.parse( + readFileSync(join(TOOLS_DIR, "__fixtures__", "legacy-reservation.jsonl"), "utf8").trim(), + ) + const legacyReservation = (identity, endedAgoMilliseconds) => { + const endedAt = new Date(Date.now() - endedAgoMilliseconds).toISOString() + return JSON.stringify({ + ...LEGACY_FIXTURE, + identity, + startedAt: new Date(Date.now() - endedAgoMilliseconds - 13_000).toISOString(), + endedAt, + accountContext: { ...LEGACY_FIXTURE.accountContext, observedAt: endedAt }, + }) + } + T( + "automation-budget.mjs: the committed legacy fixture still carries the pre-C1 reservation shape", + LEGACY_FIXTURE.engine === "codex" && + LEGACY_FIXTURE.tier === "routine" && + typeof LEGACY_FIXTURE.identity === "string" && + LEGACY_FIXTURE.accountContext?.scope === "account" && + ["pending", "reservedTokens", "inputTokens", "outputTokens", "workerPid", "cancelled"].every( + (field) => !Object.hasOwn(LEGACY_FIXTURE, field), + ), + `tools/__fixtures__/legacy-reservation.jsonl: ${JSON.stringify(LEGACY_FIXTURE)}`, + ) + const leasedReservation = (identity, endedAgoMilliseconds, reservedTokens, workerPid) => + JSON.stringify({ + identity, + engine: "codex", + tier: "routine", + startedAt: new Date(Date.now() - endedAgoMilliseconds - 13_000).toISOString(), + endedAt: new Date(Date.now() - endedAgoMilliseconds).toISOString(), + pending: true, + reservedTokens, + ...(workerPid === undefined ? {} : { workerPid }), + }) + const measuredInvocation = (identity, endedAgoMilliseconds, inputTokens, outputTokens) => + JSON.stringify({ + identity, + engine: "codex", + tier: "routine", + startedAt: new Date(Date.now() - endedAgoMilliseconds - 60_000).toISOString(), + endedAt: new Date(Date.now() - endedAgoMilliseconds).toISOString(), + inputTokens, + outputTokens, + }) + + check( + "automation-budget.mjs", + "a legacy reservation four hours old no longer refuses a launch the budget permits", + leaseCheckArgs( + "after-expired-legacy", + stage( + "budget/expired-legacy.jsonl", + `${legacyReservation("ORB-163:stranded", 4 * HOUR_MILLISECONDS)}\n`, + ), + 100000, + ), + { + status: 0, + stdout: /"status":"PROCEED"[\s\S]*"projectedTokens":100000[\s\S]*"pendingTokens":0,"missingIdentities":\[\],"expiredIdentities":\["ORB-163:stranded"\]/, + stderr: /reservation lease expired for identities ORB-163:stranded/, + }, + ) + check( + "automation-budget.mjs", + "a legacy reservation one hour old still fails the fuse closed", + leaseCheckArgs( + "after-live-legacy", + stage( + "budget/live-legacy.jsonl", + `${legacyReservation("ORB-163:in-flight", HOUR_MILLISECONDS)}\n`, + ), + 100000, + ), + { status: 3, stderr: /lack input or output tokens[\s\S]*ORB-163:in-flight/ }, + ) + check( + "automation-budget.mjs", + "an unclaimed reservation one hour old still holds its reserved tokens", + leaseCheckArgs( + "after-live-reservation", + stage( + "budget/live-reservation.jsonl", + `${leasedReservation("ORB-163:live", HOUR_MILLISECONDS, 250000)}\n`, + ), + 100000, + ), + { + status: 0, + stdout: /"status":"PROCEED"[\s\S]*"projectedTokens":350000[\s\S]*"pendingTokens":250000,"missingIdentities":\[\],"expiredIdentities":\[\]/, + }, + ) + check( + "automation-budget.mjs", + "an unclaimed reservation four hours old stops holding budget", + leaseCheckArgs( + "after-killed-launcher", + stage( + "budget/expired-reservation.jsonl", + `${leasedReservation("ORB-163:killed-launcher", 4 * HOUR_MILLISECONDS, 250000)}\n`, + ), + 100000, + ), + { + status: 0, + stdout: /"projectedTokens":100000[\s\S]*"pendingTokens":0,"missingIdentities":\[\],"expiredIdentities":\["ORB-163:killed-launcher"\]/, + }, + ) + check( + "automation-budget.mjs", + "a reservation whose worker process is gone expires well inside its lease", + leaseCheckArgs( + "after-dead-worker", + stage( + "budget/dead-worker.jsonl", + `${leasedReservation("ORB-163:dead-worker", 60_000, 250000, exitedProbePid())}\n`, + ), + 100000, + ), + { + status: 0, + stdout: /"projectedTokens":100000[\s\S]*"pendingTokens":0,"missingIdentities":\[\],"expiredIdentities":\["ORB-163:dead-worker"\]/, + stderr: /reservation lease expired for identities ORB-163:dead-worker/, + }, + ) + check( + "automation-budget.mjs", + "a live worker PID fourteen hours in still holds its tokens, because real sessions run that long", + leaseCheckArgs( + "after-live-worker", + stage( + "budget/live-worker.jsonl", + `${leasedReservation("ORB-163:live-worker", 14 * HOUR_MILLISECONDS, 250000, process.pid)}\n`, + ), + 100000, + ), + { + status: 0, + stdout: /"projectedTokens":350000[\s\S]*"pendingTokens":250000,"missingIdentities":\[\],"expiredIdentities":\[\]/, + }, + ) + check( + "automation-budget.mjs", + "a live worker PID eighteen hours in still expires, so a recycled PID can never poison the fuse forever", + leaseCheckArgs( + "after-recycled-pid", + stage( + "budget/recycled-pid.jsonl", + `${leasedReservation("ORB-163:recycled-pid", 18 * HOUR_MILLISECONDS, 250000, process.pid)}\n`, + ), + 100000, + ), + { + status: 0, + stdout: /"projectedTokens":100000[\s\S]*"pendingTokens":0,"missingIdentities":\[\],"expiredIdentities":\["ORB-163:recycled-pid"\]/, + stderr: /reservation lease expired for identities ORB-163:recycled-pid/, + }, + ) + const claimedLedger = stage("budget/claimed.jsonl", `${leasedReservation("ORB-163:to-claim", 60_000, 250000)}\n`) + const claimed = run("automation-budget.mjs", [ + "claim", + "--identity", + "ORB-163:to-claim", + "--engine", + "codex", + "--tier", + "routine", + "--started-at", + new Date(Date.now() - 73_000).toISOString(), + "--ended-at", + new Date().toISOString(), + "--invocation-tokens", + "250000", + "--worker-pid", + String(process.pid), + "--ledger", + claimedLedger, + "--json", + ]) + const claimedRows = existsSync(claimedLedger) + ? readFileSync(claimedLedger, "utf8").trim().split(/\r?\n/).filter(Boolean).map((line) => JSON.parse(line)) + : [] + T( + "automation-budget.mjs: claim attaches the running worker PID to an open reservation", + claimed.status === 0 && + /"status":"CLAIMED"/.test(claimed.stdout) && + claimedRows.length === 2 && + claimedRows[1]?.pending === true && + claimedRows[1]?.workerPid === process.pid && + claimedRows[1]?.reservedTokens === 250000, + `exit ${claimed.status}\n ${claimed.stderr}\n ${JSON.stringify(claimedRows)}`, + ) + check( + "automation-budget.mjs", + "claim refuses an identity whose latest record is not an open reservation", + [ + "claim", + "--identity", + "ORB-163:measured-already", + "--engine", + "codex", + "--tier", + "routine", + "--started-at", + new Date(Date.now() - 73_000).toISOString(), + "--ended-at", + new Date().toISOString(), + "--invocation-tokens", + "250000", + "--worker-pid", + String(process.pid), + "--ledger", + stage("budget/claim-closed.jsonl", `${measuredInvocation("ORB-163:measured-already", 60_000, 10, 5)}\n`), + ], + { status: 3, stderr: /is not an open reservation/ }, + ) + check( + "automation-budget.mjs", + "a ledger row carrying a worker PID without a reservation is rejected", + leaseCheckArgs( + "after-orphan-pid", + stage( + "budget/orphan-pid.jsonl", + `${JSON.stringify({ + identity: "ORB-163:orphan-pid", + engine: "codex", + tier: "routine", + startedAt: "2026-07-30T09:00:00.000Z", + endedAt: "2026-07-30T10:00:00.000Z", + inputTokens: 10, + outputTokens: 5, + workerPid: 1234, + })}\n`, + ), + 100000, + ), + { status: 3, stderr: /workerPid is only valid on a pending reservation/ }, + ) + check( + "automation-budget.mjs", + "a half-measured invocation keeps failing closed for the whole window, no lease applies", + leaseCheckArgs( + "after-half-measured", + stage( + "budget/half-measured.jsonl", + `${JSON.stringify({ + identity: "ORB-163:half-measured", + engine: "codex", + tier: "routine", + startedAt: new Date(Date.now() - 4 * HOUR_MILLISECONDS - 60_000).toISOString(), + endedAt: new Date(Date.now() - 4 * HOUR_MILLISECONDS).toISOString(), + inputTokens: 900, + })}\n`, + ), + 100000, + ), + { status: 3, stderr: /lack input or output tokens[\s\S]*ORB-163:half-measured/ }, + ) + check( + "automation-budget.mjs", + "an expired lease never softens a real token block", + leaseCheckArgs( + "after-expired-block", + stage( + "budget/expired-with-spend.jsonl", + [ + legacyReservation("ORB-163:stranded-beside-spend", 4 * HOUR_MILLISECONDS), + measuredInvocation("ORB-163:measured", 4 * HOUR_MILLISECONDS, 900000, 50000), + "", + ].join("\n"), + ), + 100000, + ), + { status: 4, stderr: /blocked:[\s\S]*projected spend 1050000 tokens/ }, + ) + const reportLedger = stage( "budget/report.jsonl", [ @@ -5338,7 +5635,7 @@ const automationBudgetCases = () => { { status: 0, stdout: - /"engine":"claude","inputTokens":400,"outputTokens":250,"totalTokens":650,"routineTokens":500,"reservedTokens":150,"missingIdentities":\["report-pending"\],"windowStart":"2030-01-01T00:00:00.000Z","resetsAt":"2030-01-08T00:00:00.000Z"/, + /"engine":"claude","inputTokens":400,"outputTokens":250,"totalTokens":650,"routineTokens":500,"reservedTokens":150,"pendingTokens":0,"missingIdentities":\["report-pending"\],"expiredIdentities":\[\],"windowStart":"2030-01-01T00:00:00.000Z","resetsAt":"2030-01-08T00:00:00.000Z"/, }, ) check( @@ -5348,7 +5645,7 @@ const automationBudgetCases = () => { { status: 0, stdout: - /^claude: 650 tokens \(400 input, 250 output; 500 routine, 150 reserved\); missing identities: report-pending; resets at 2030-01-08T00:00:00.000Z\r?\n$/, + /^claude: 650 tokens \(400 input, 250 output; 500 routine, 150 reserved, 0 pending\); missing identities: report-pending; expired reservations: none; resets at 2030-01-08T00:00:00.000Z\r?\n$/, }, ) @@ -5364,7 +5661,7 @@ const common = (identity) => [ tool, "reserve", "--engine", "claude", "--identity", identity, "--tier", "routine", "--started-at", "2030-01-02T09:00:00.000Z", "--ended-at", "2030-01-02T10:00:00.000Z", "--reset-at", "2030-01-08T00:00:00Z", "--warning-tokens", "800", - "--budget-tokens", "1000", "--invocation-tokens", "600", "--ledger", ledger, + "--budget-tokens", "1000", "--invocation-tokens", "400", "--ledger", ledger, ] const run = (identity, env = {}) => { const child = spawn(process.execPath, common(identity), { @@ -5402,16 +5699,18 @@ process.stdout.write(JSON.stringify(results)) ? readFileSync(atomicLedger, "utf8").trim().split(/\r?\n/).filter(Boolean).map((line) => JSON.parse(line)) : [] T( - "automation-budget.mjs: concurrent checks atomically reserve before another process can pass", + "automation-budget.mjs: concurrent reservations proceed atomically below the budget", atomic.status === 0 && atomicResults[0]?.status === 0 && - atomicResults[1]?.status === 3 && - /lack input or output tokens[\s\S]*atomic-a/.test(atomicResults[1]?.stderr ?? "") && - atomicRecords.length === 1 && - atomicRecords[0]?.identity === "atomic-a" && + atomicResults[1]?.status === 0 && + !/lack input or output tokens/.test(atomicResults[1]?.stderr ?? "") && + atomicRecords.length === 2 && + new Set(atomicRecords.map((record) => record.identity)).size === 2 && + atomicRecords.every((record) => record.pending === true && record.reservedTokens === 400) && !existsSync(`${atomicLedger}.lock`), `exit ${atomic.status}\n stdout: ${atomic.stdout}\n stderr: ${atomic.stderr}\n ledger: ${JSON.stringify(atomicRecords)}`, ) + const beforeCancel = run("automation-budget.mjs", checkArgs("atomic-before-cancel", atomicLedger, 600, ["--json"])) const cancelAtomic = run("automation-budget.mjs", [ "cancel", "--identity", @@ -5430,10 +5729,12 @@ process.stdout.write(JSON.stringify(results)) const afterCancel = run("automation-budget.mjs", checkArgs("atomic-after-cancel", atomicLedger, 600, ["--json"])) T( "automation-budget.mjs: append-only cancellation releases a reservation that never started", - cancelAtomic.status === 0 && + beforeCancel.status === 4 && + /"projectedTokens":1400[\s\S]*"pendingTokens":800,/.test(beforeCancel.stdout) && + cancelAtomic.status === 0 && afterCancel.status === 0 && - /"projectedTokens":600[\s\S]*"missingIdentities":\[\]/.test(afterCancel.stdout), - `cancel exit ${cancelAtomic.status}: ${cancelAtomic.stderr}\n check exit ${afterCancel.status}: ${afterCancel.stderr}\n ${afterCancel.stdout}`, + /"projectedTokens":1000[\s\S]*"pendingTokens":400,"missingIdentities":\[\]/.test(afterCancel.stdout), + `before exit ${beforeCancel.status}: ${beforeCancel.stdout}\n cancel exit ${cancelAtomic.status}: ${cancelAtomic.stderr}\n check exit ${afterCancel.status}: ${afterCancel.stderr}\n ${afterCancel.stdout}`, ) check( @@ -5597,6 +5898,176 @@ const mergeabilityCases = () => { T("mergeability.mjs: an unparseable forge result is HELD", unparseable.status === 1 && /HELD github-pull-request: GitHub pull-request lookup returned unparseable output/.test(unparseable.stdout), unparseable.stderr || unparseable.stdout) } +/** + * nudge-worker's whole surface is now a refusal, so every case here asserts the SAME thing the + * flag-by-flag suite asserted before the flags went away: this tool never delivers a mid-run turn. + * Each named case survives its flag's deletion because the invocation a caller would still try is + * exactly the one that must be refused, and the orca log proves nothing was sent. + */ +const nudgeWorkerCases = () => { + const noArgumentLog = join(root, "nudge-no-argument.log") + check( + "nudge-worker.mjs", + "headless workers explain that a live turn cannot be injected", + [], + { status: 1, stderr: /mid-run injection is unavailable[\s\S]*relaunch/ }, + { env: { ...orcaEnv([]), ORBIT_ORCA_LOG: noArgumentLog } }, + ) + T( + "nudge-worker.mjs: a refused nudge calls orca not at all", + !existsSync(noArgumentLog), + `orca was invoked: ${existsSync(noArgumentLog) ? readFileSync(noArgumentLog, "utf8") : ""}`, + ) + check("nudge-worker.mjs", "--help documents the fail-closed rule and the relaunch remedy", ["--help"], { status: 0, stdout: /unavailable for headless workers[\s\S]*Relaunch after exit[\s\S]*exit codes:/ }) + check("nudge-worker.mjs", "rejects multi-line text", ["--terminal", "t1", "--text", "first line\nsecond line"], { status: 2, stderr: /mid-run injection is unavailable/ }) + check("nudge-worker.mjs", "rejects --text together with --prompt-file", ["--terminal", "t1", "--text", "hi", "--prompt-file", stage("nudge-prompt.md", "body\n")], { status: 2, stderr: /mid-run injection is unavailable/ }) + check("nudge-worker.mjs", "rejects a non-positive --wait-attempts", ["--terminal", "t1", "--text", "hi", "--wait-attempts", "0"], { status: 2, stderr: /mid-run injection is unavailable/ }) + const dryRunLog = join(root, "nudge-dry-run.log") + check( + "nudge-worker.mjs", + "--dry-run calls orca not at all", + ["--terminal", "t1", "--text", "hi", "--dry-run"], + { status: 2, stderr: /mid-run injection is unavailable/ }, + { env: { ...orcaEnv([]), ORBIT_ORCA_LOG: dryRunLog } }, + ) + T( + "nudge-worker.mjs: --dry-run leaves no orca invocation behind", + !existsSync(dryRunLog), + `orca was invoked: ${existsSync(dryRunLog) ? readFileSync(dryRunLog, "utf8") : ""}`, + ) +} + +/** + * worker-watch cases against the PID model that replaced the repaint delta. Liveness is now a + * launcher-written PID the harness can prove alive or dead, but everything else the report is for + * survives: the Linear state beside liveness, a contract verdict that degrades visibly rather than + * vanishing, an empty fleet that says so, and a --repo filter that actually excludes. + */ +const stageWorkerWatch = (label, repoPath) => { + const base = join(root, "watch", label) + mkdirSync(join(base, "tools"), { recursive: true }) + mkdirSync(join(base, ".claude"), { recursive: true }) + writeFileSync( + join(base, ".claude", "orchestrator.json"), + JSON.stringify({ worker: "codex", maxParallelWorktrees: 4, repos: { ui: repoPath, api: join(root, "watch", "absent-api") } }), + ) + cpSync(join(TOOLS_DIR, "worker-watch.mjs"), join(base, "tools", "worker-watch.mjs")) + cpSync(join(TOOLS_DIR, "worker-status.mjs"), join(base, "tools", "worker-status.mjs")) + cpSync(join(TOOLS_DIR, "lib"), join(base, "tools", "lib"), { recursive: true }) + return join(base, "tools", "worker-watch.mjs") +} + +const stageWatchedWorktree = (label) => { + const path = join(root, "watch", "repos", label) + mkdirSync(path, { recursive: true }) + spawnSync("git", ["-C", path, "init", "--initial-branch=main"], { encoding: "utf8" }) + return path +} + +const workerWatchCases = () => { + const repoPath = join(root, "watch", "repos") + const tool = stageWorkerWatch("fleet", repoPath) + const watchPlan = (worktrees) => [ + { match: "worktree list", stdout: JSON.stringify({ ok: true, result: { worktrees } }) }, + { match: "linear issue ORB-75", stdout: JSON.stringify({ ok: true, result: { issue: { identifier: "ORB-75", state: { name: "In Progress" } } } }) }, + ] + const watched = (path) => ({ + path, + isMainWorktree: false, + isArchived: false, + branch: "refs/heads/feature/orb-75-prove-the-harness-gate", + linkedLinearIssue: "ORB-75", + baseRef: "main", + }) + + check( + "worker-watch.mjs", + "an empty fleet says so rather than printing nothing", + [], + { status: 0, stdout: /no Orca worktrees/ }, + { path: tool, env: orcaEnv(watchPlan([])) }, + ) + + const livePath = stageWatchedWorktree("live") + stageWorkerPidMarker(livePath, process.pid) + const live = check( + "worker-watch.mjs", + "a launcher PID that is still running is BUSY", + [], + { status: 0, stdout: /BUSY\s+ORB-75/ }, + { path: tool, env: orcaEnv(watchPlan([watched(livePath)])) }, + ) + T( + "worker-watch.mjs: the ticket's Linear state is reported alongside liveness", + /In Progress/.test(live.stdout), + live.stdout.slice(0, 400), + ) + T( + "worker-watch.mjs: an unreadable contract verdict is reported, never silently dropped", + /contract\s+unavailable/.test(live.stdout), + `worker-status ran against a checkout with no Orbit contract, so the verdict must degrade visibly\n ${live.stdout.slice(0, 400)}`, + ) + /** + * IDLE plus NOT MET is the pair that costs a run, so it is the row that has to say WHAT is + * unmet. worker-status.mjs already returns the list on stdout; reading only its exit code + * threw away the one thing an operator acts on, while /watch's own worked example promised it. + */ + const verdictTool = stageWorkerWatch("verdict", repoPath) + writeFileSync( + join(dirname(verdictTool), "worker-status.mjs"), + `#!/usr/bin/env node\nconsole.log(JSON.stringify({ issue: "ORB-75", unmet: ["commits", "pushed", "pr-open"], pullRequest: null, ok: false }))\nprocess.exit(1)\n`, + ) + const unmetReport = check( + "worker-watch.mjs", + "a NOT MET row names the unmet checklist rather than the bare verdict", + [], + { status: 0, stdout: /contract\s+NOT MET: commits, pushed, pr-open/ }, + { path: verdictTool, env: orcaEnv(watchPlan([watched(livePath)])) }, + ) + T( + "worker-watch.mjs: the JSON report carries the same unmet list the text line names", + /"unmet": \[\s*"commits",\s*"pushed",\s*"pr-open"\s*\]/.test( + run("worker-watch.mjs", ["--json"], { path: verdictTool, env: orcaEnv(watchPlan([watched(livePath)])) }).stdout, + ), + unmetReport.stdout.slice(0, 400), + ) + + const exitedPath = stageWatchedWorktree("exited") + stageWorkerPidMarker(exitedPath, exitedProbePid()) + check( + "worker-watch.mjs", + "a launcher PID that has exited is IDLE", + [], + { status: 0, stdout: /IDLE\s+ORB-75/ }, + { path: tool, env: orcaEnv(watchPlan([watched(exitedPath)])) }, + ) + + check( + "worker-watch.mjs", + "--repo actually excludes a worktree outside that repo", + ["--repo", "api"], + { status: 0, stdout: /no Orca worktrees for api/ }, + { path: tool, env: orcaEnv(watchPlan([watched(livePath)])) }, + ) + check( + "worker-watch.mjs", + "--repo keeps a worktree inside that repo", + ["--repo", "ui"], + { status: 0, stdout: /BUSY\s+ORB-75/ }, + { path: tool, env: orcaEnv(watchPlan([watched(livePath)])) }, + ) + check( + "worker-watch.mjs", + "the JSON report carries the PID liveness the text line summarises", + ["--json"], + { status: 0, stdout: /"liveness": "BUSY"[\s\S]*"pid": \d+,\s*"alive": true/ }, + { path: tool, env: orcaEnv(watchPlan([watched(livePath)])) }, + ) + check("worker-watch.mjs", "refuses a repo outside orchestrator.json", ["--repo", "zzz"], { status: 2, stderr: /--repo must be one of/ }, { path: tool }) + check("worker-watch.mjs", "refuses an unknown option instead of ignoring it", ["--lines", "8"], { status: 2, stderr: /unknown option/ }, { path: tool }) + check("worker-watch.mjs", "documents the JSON report mode", ["--help"], { status: 0, stdout: /--json/ }, { path: tool }) +} + const gateCases = { "mergeability.mjs": mergeabilityCases, "ai-quota.mjs": aiQuotaCases, diff --git a/tools/worker-watch.mjs b/tools/worker-watch.mjs index cc611e2cf..4f986b044 100644 --- a/tools/worker-watch.mjs +++ b/tools/worker-watch.mjs @@ -1,236 +1,81 @@ #!/usr/bin/env node -/** - * One screen answering "what is every child session doing right now". - * - * `worker-status.mjs` adjudicates DELIVERY from artifacts and is right to; nothing reported - * LIVENESS. Over the 2026-07-27 ORB-88 run the orchestrator hand-ran `orca terminal read` - * five times to answer "is this worker working, stuck, or asking a question", re-deriving the - * same two things each time, and both are hostile to read raw: `orca terminal read` flattens a - * TUI repaint, so a busy worker's tail arrives as thousands of characters of concatenated - * `Working` fragments, and a single read cannot distinguish a busy worker from an idle one at - * all. The cost was direct: Thomas twice noticed a stalled or duplicated child session before - * the orchestrator did. - * - * So this REPORTS liveness alongside the contract verdict and never replaces it, and it never - * acts: what to send a stalled worker is the orchestrator's judgement, not a watcher's. - */ - +/** Report worker liveness from launcher-owned PIDs. Orca worktrees remain required; terminals are optional. */ import { execFileSync, spawnSync } from "node:child_process" +import { existsSync, readFileSync } from "node:fs" +import { join, resolve } from "node:path" import { fileURLToPath } from "node:url" - import { readOrchestratorConfig } from "./lib/orchestrator-config.mjs" -import { REPAINT_SAMPLE_MS, classifyTerminals, pause, sampleTerminals } from "./lib/tui-repaint.mjs" - -const USAGE = `usage: worker-watch.mjs [options] - - --repo ui|api|landing only worktrees of that repo (default: every repo in orchestrator.json) - --lines how many meaningful output lines per worker (default: 8) - --no-contract skip the worker-status.mjs verdict, which costs a fetch + gh call per - worktree. Liveness only, for a fast look - --json emit the report as JSON instead of text - --help, -h print this usage and exit 0 - -Per Orca worktree: the Linear ticket, the branch, the ticket's Linear state, BUSY or IDLE -classified by repaint delta over ${REPAINT_SAMPLE_MS}ms, the last meaningful output lines with -repaint noise stripped, and the worker-status.mjs contract verdict. - -BUSY/IDLE is LIVENESS, never completion: an idle worker may be finished, stopped early, or -waiting on a question nobody will answer. The contract line is what says whether the work -landed. - -exit codes: 0 the report printed (including "no worktrees", which is a result), - 2 usage error, 3 an orca command failed` - -if (process.argv.includes("--help") || process.argv.includes("-h")) { - console.log(USAGE) - process.exit(0) -} const ORCA = process.env.ORCA_BIN || "C:\\Users\\thoma\\AppData\\Local\\Programs\\orca\\resources\\bin\\orca" -const WORKER_STATUS = fileURLToPath(new URL("./worker-status.mjs", import.meta.url)) - -/** - * What a TUI paints while it is thinking, and nothing else. A tail full of these is not output, - * it is the same frame redrawn: `orca terminal read` concatenates the frames with no separator, - * so a busy worker's tail is `WorkingWorkingWorking...` for thousands of characters. Stripping - * these leaves a line that is either real content or empty, and empty is what gets dropped. - * Deliberately matched loosely (`inter\\w*` covers the CLI's own `interupt` typo) and only used - * to DECIDE: what prints is the original line. - */ -const REPAINT_NOISE = /working|flowing|thinking|esc to inter\w*|ctrl\+?c|\b\d+s\b|\btokens?\b|[\u2800-\u28ff\u2500-\u257f\u2580-\u259f\u25a0-\u25ff]/gi -const ANSI = /\u001b\[[0-9;?]*[a-zA-Z]/g -/** Enough letters left after the noise to be a sentence rather than punctuation debris. */ -const MIN_MEANINGFUL_CHARS = 4 - -const fail = (code, message) => { - console.error(message) - process.exit(code) -} - -const argOf = (flag) => { - const index = process.argv.indexOf(flag) - return index === -1 ? null : process.argv[index + 1] -} - -const KNOWN_FLAGS = new Set(["--repo", "--lines", "--no-contract", "--json", "--help", "-h"]) -const unknown = process.argv.slice(2).filter((token) => token.startsWith("-") && !KNOWN_FLAGS.has(token)) -if (unknown.length > 0) fail(2, `${USAGE}\n\nunknown option(s): ${unknown.join(" ")}`) - -const orca = (args, { soft = false } = {}) => { - let raw - try { - raw = execFileSync(ORCA, [...args, "--json"], { encoding: "utf8", maxBuffer: 32 * 1024 * 1024 }) - } catch (error) { - if (soft) return null - const payload = error.stdout?.toString() ?? "" - return fail(3, `orca ${args.join(" ")} failed: ${payload.trim().slice(0, 300) || error.stderr?.toString().trim() || error.message}`) - } - let parsed - try { - parsed = JSON.parse(raw) - } catch { - if (soft) return null - return fail(3, `orca ${args.join(" ")} returned unparseable output: ${raw.slice(0, 300)}`) - } - if (parsed.ok === false) { - if (soft) return null - return fail(3, `orca ${args.join(" ")} failed: ${parsed.error?.message ?? "unknown orca error"}`) - } - return parsed.result ?? parsed +const WORKER_STATUS = new URL("./worker-status.mjs", import.meta.url) +const USAGE = `usage: worker-watch.mjs [--repo ui|api|landing] [--json] + + Per Orca worktree: BUSY or IDLE from the launcher-written worker PID, the linked ticket and its + Linear state, the branch, and the worker-status.mjs contract verdict. An empty fleet prints so. + + --repo report only worktrees under that repo's path from .claude/orchestrator.json + --json emit the report as JSON instead of one line per worktree + --help, -h print this usage and exit 0 + +exit codes: 0 the report printed, including an empty fleet, 2 usage or config error` +if (process.argv.includes("--help") || process.argv.includes("-h")) { console.log(USAGE); process.exit(0) } +const unknown = process.argv.slice(2).filter((value) => value.startsWith("-") && !["--repo", "--json"].includes(value)) +if (unknown.length) { console.error(`${USAGE}\n\nunknown option(s): ${unknown.join(" ")}`); process.exit(2) } +const json = process.argv.includes("--json") +const repo = process.argv.includes("--repo") ? process.argv[process.argv.indexOf("--repo") + 1] : null +const fail = (message) => { console.error(message); process.exit(2) } +if (repo === undefined) fail("--repo requires a value") +const config = readOrchestratorConfig() +if (repo && !config.repos?.[repo]) fail(`--repo must be one of: ${Object.keys(config.repos ?? {}).join(", ")}`) +const orca = (args) => { + const output = execFileSync(ORCA, [...args, "--json"], { encoding: "utf8" }) + const payload = JSON.parse(output) + if (payload.ok === false) throw new Error(payload.error?.message ?? "orca failure") + return payload.result ?? payload } - -const repoFilter = argOf("--repo") -const lines = Number(argOf("--lines") ?? 8) -const withContract = !process.argv.includes("--no-contract") -const asJson = process.argv.includes("--json") - -if (!Number.isInteger(lines) || lines < 1) fail(2, "--lines must be a positive integer") - -let config -try { - config = readOrchestratorConfig() -} catch (error) { - fail(2, error.message) +const alive = (pid) => { try { process.kill(pid, 0); return true } catch (error) { return error.code !== "ESRCH" } } +const gitDir = (path) => resolve(path, (spawnSync("git", ["-C", path, "rev-parse", "--git-dir"], { encoding: "utf8" }).stdout ?? "").trim()) +const pidsFor = (path) => { + const marker = join(gitDir(path), "orbit-worker-pids.jsonl") + if (!existsSync(marker)) return [] + return readFileSync(marker, "utf8").trim().split(/\r?\n/).filter(Boolean).flatMap((line) => { try { const row = JSON.parse(line); return row.worktreePath === path && Number.isInteger(row.pid) ? [row.pid] : [] } catch { return [] } }) } -const repos = config.repos ?? {} -if (repoFilter && !repos[repoFilter]) fail(2, `--repo must be one of: ${Object.keys(repos).join(", ")}`) - -const normalize = (path) => (path ?? "").replaceAll("\\", "/").replace(/\/+$/, "").toLowerCase() -const wanted = Object.entries(repos).filter(([key]) => !repoFilter || key === repoFilter) - const worktrees = orca(["worktree", "list"]).worktrees ?? [] - /** - * Which repo a worktree belongs to, by the repoId its own MAIN worktree carries. Orca's - * worktrees live outside the repo (`~/orca/workspaces/...`), so a path-prefix test cannot - * answer this. `projectId` is the second signal, for a fleet listed without its main worktree. + * A verdict that cannot be read is reported as unavailable, never folded into MET or NOT MET. + * The unmet list is the whole point of the NOT MET row: worker-status.mjs already returns + * `{ unmet: [...], pullRequest }` on stdout, and reading only the exit code threw away the one + * thing an operator acts on. IDLE plus NOT MET is the pair that costs a run, so it is exactly + * the row that must say WHAT is unmet. */ -const repoIdOf = new Map( - worktrees.filter((worktree) => worktree.isMainWorktree).map((worktree) => [normalize(worktree.path), worktree.repoId]), -) -const repoKeyOf = (worktree) => { - for (const [key, path] of wanted) { - const mainRepoId = repoIdOf.get(normalize(path)) - if (mainRepoId && worktree.repoId === mainRepoId) return key - if (normalize(worktree.projectId).endsWith(`/${normalize(path).split("/").pop()}`)) return key - } - return null -} - -const children = worktrees - .filter((worktree) => !worktree.isMainWorktree && !worktree.isArchived) - .map((worktree) => ({ worktree, repoKey: repoKeyOf(worktree) })) - .filter((entry) => entry.repoKey) - -if (children.length === 0) { - const scope = repoFilter ? `the ${repoFilter} repo` : `the Orbit repos (${Object.keys(repos).join(", ")})` - if (asJson) console.log(JSON.stringify({ worktrees: [], scope: repoFilter ?? null }, null, 2)) - else console.log(`no Orca worktrees in ${scope}; nothing is running`) - process.exit(0) -} - -/** Two samples one window apart: a running turn repaints continuously, an idle TUI emits nothing. */ -const before = sampleTerminals(orca) -pause(REPAINT_SAMPLE_MS) -const after = sampleTerminals(orca) -const liveness = classifyTerminals(before, after) -const terminals = orca(["terminal", "list"]).terminals ?? [] - -const meaningfulLines = (tail) => { - const kept = [] - for (const raw of tail) { - const line = raw.replace(ANSI, "").replace(/\s+/g, " ").trim() - if (!line) continue - const bare = line.replace(REPAINT_NOISE, "").replace(/[^\p{L}\p{N}]/gu, "") - if (bare.length < MIN_MEANINGFUL_CHARS) continue - if (kept[kept.length - 1] === line) continue - kept.push(line) +const contractVerdict = (exitCode) => (exitCode === 0 ? "MET" : exitCode === 1 ? "NOT MET" : "unavailable") +const contractDetail = (status) => { + if (status.status !== 0 && status.status !== 1) return { unmet: [], pullRequest: null } + try { + const verdict = JSON.parse(status.stdout) + return { unmet: Array.isArray(verdict.unmet) ? verdict.unmet : [], pullRequest: verdict.pullRequest ?? null } + } catch { + return { unmet: [], pullRequest: null } } - return kept.slice(-lines) } - -const contractVerdict = (path, issue, base) => { - if (!issue) return { state: "skipped", detail: "the worktree carries no linked Linear issue" } - const result = spawnSync(process.execPath, [WORKER_STATUS, "--worktree", path, "--issue", issue, "--base", base, "--json"], { - encoding: "utf8", - maxBuffer: 32 * 1024 * 1024, - }) - let verdict = null +const linearState = (issue) => { + if (!issue) return "(no ticket)" try { - verdict = JSON.parse(result.stdout) + const detail = orca(["linear", "issue", issue]) + return (detail.issue ?? detail)?.state?.name ?? "unknown" } catch { - /* worker-status writes its own failure to stderr; the state below carries it */ + return "unknown" } - if (verdict) return { state: verdict.ok ? "met" : "not-met", unmet: verdict.unmet, pullRequest: verdict.pullRequest } - return { state: "unavailable", detail: (result.stderr || "").trim().split("\n")[0]?.slice(0, 200) || `worker-status exited ${result.status}` } } - -const report = children.map(({ worktree, repoKey }) => { - const own = terminals.filter((terminal) => normalize(terminal.worktreePath) === normalize(worktree.path)) - const busy = own.some((terminal) => liveness.get(terminal.handle) === "BUSY") - const newest = own.slice().sort((first, second) => (second.lastOutputAt ?? 0) - (first.lastOutputAt ?? 0))[0] ?? null - const issue = worktree.linkedLinearIssue ?? null - const detail = issue ? orca(["linear", "issue", issue], { soft: true }) : null - const linearIssue = detail?.issue ?? detail - const tail = newest ? orca(["terminal", "read", "--terminal", newest.handle, "--limit", "200"], { soft: true })?.terminal?.tail ?? [] : [] - - return { - issue, - repo: repoKey, - path: worktree.path, - branch: (worktree.branch ?? "").replace(/^refs\/heads\//, ""), - linearState: linearIssue?.state?.name ?? linearIssue?.state ?? (issue ? "unknown" : null), - liveness: busy ? "BUSY" : "IDLE", - terminals: own.map((terminal) => ({ handle: terminal.handle, liveness: liveness.get(terminal.handle) ?? "IDLE", title: terminal.title })), - comment: worktree.comment || null, - lastOutput: meaningfulLines(tail), - contract: withContract ? contractVerdict(worktree.path, issue, worktree.baseRef ?? "main") : null, - } +const report = worktrees.filter((entry) => !entry.isMainWorktree && !entry.isArchived).filter((entry) => !repo || resolve(entry.path).startsWith(resolve(config.repos[repo]))).map((entry) => { + const pids = pidsFor(entry.path) + const workerAlive = pids.some(alive) + const status = entry.linkedLinearIssue + ? spawnSync(process.execPath, [fileURLToPath(WORKER_STATUS), "--worktree", entry.path, "--issue", entry.linkedLinearIssue, "--base", entry.baseRef ?? "main", "--json"], { encoding: "utf8" }) + : { status: null } + const { unmet, pullRequest } = contractDetail(status) + return { issue: entry.linkedLinearIssue ?? null, state: linearState(entry.linkedLinearIssue), path: entry.path, branch: entry.branch ?? "", liveness: workerAlive ? "BUSY" : "IDLE", workerPids: pids.map((pid) => ({ pid, alive: alive(pid) })), contractExit: status.status, contract: contractVerdict(status.status), unmet, pullRequest } }) - -if (asJson) { - console.log(JSON.stringify({ worktrees: report, scope: repoFilter ?? null }, null, 2)) - process.exit(0) -} - -console.log(`${report.length} Orca worktree(s)${repoFilter ? ` in ${repoFilter}` : ""}, liveness sampled over ${REPAINT_SAMPLE_MS}ms\n`) -for (const entry of report) { - console.log(`${entry.liveness} ${entry.issue ?? "(no ticket)"} ${entry.branch} [${entry.repo}]`) - console.log(` path ${entry.path}`) - if (entry.linearState) console.log(` linear ${entry.linearState}`) - if (entry.comment) console.log(` card ${entry.comment}`) - const terminalLines = entry.terminals.map((terminal) => `${terminal.handle} ${terminal.liveness}`).join("\n ") - console.log(` terminals ${terminalLines || "none (no TUI attached)"}`) - if (entry.contract) { - const line = - entry.contract.state === "met" - ? `CONTRACT MET (${entry.contract.pullRequest ?? "no PR recorded"})` - : entry.contract.state === "not-met" - ? `NOT MET: ${entry.contract.unmet.join(", ")}` - : `${entry.contract.state}: ${entry.contract.detail}` - console.log(` contract ${line}`) - } - console.log(` last output${entry.lastOutput.length === 0 ? " (nothing but repaint noise)" : ""}`) - for (const line of entry.lastOutput) console.log(` | ${line.slice(0, 160)}`) - console.log("") -} +if (json) console.log(JSON.stringify({ worktrees: report }, null, 2)) +else if (report.length === 0) console.log(`no Orca worktrees${repo ? ` for ${repo}` : ""}`) +else for (const entry of report) console.log(`${entry.liveness} ${entry.issue ?? "(no ticket)"} ${entry.state} ${entry.branch} pid(s): ${entry.workerPids.map((worker) => `${worker.pid}:${worker.alive ? "alive" : "exited"}`).join(", ") || "none"} contract ${entry.contract}${entry.unmet.length > 0 ? `: ${entry.unmet.join(", ")}` : ""}`)