From 95bee2781fd64b37e6bbe5ca0ba7b3d5ba1f739e Mon Sep 17 00:00:00 2001 From: lidge-jun Date: Tue, 13 Oct 2026 01:09:25 +0900 Subject: [PATCH 1/9] fix(ci): run the test runner on Bun 1.4.3 so spawnSync stops losing child exits (#4956) (#6986) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Squashed from #6986 for the 2.84.0 release train. Linux CI shard 1/4 batch 5 kept hitting its 120 s batch deadline (#4956; runs 38146205183 and 38162375606). The batch did not stall after its tests passed: it stalled inside `tests/ci-workflows/release-version-sources.test.ts`, in a `Bun.spawnSync(..., { timeout: 15000 })` that never returned. **Root cause (Bun 1.4.0, the CI test runner).** For the length of a call, `Bun.spawnSync` pointed the VM at its private wait loop. A finalizer that ran during the call (an earlier file's stdout/stderr sink, a stream reader) released its poll on that loop. The loop is cached for the life of the process, so its poll count drifted to zero. A later call then stopped polling for its child: the child became a zombie and the call spun at 100% CPU past its own `timeout`. In a multi-file `--isolate` process this depends on what earlier files left to finalize, which is why every file passed alone. Upstream fixed it in oven-sh/bun#44581, first released in Bun 1.4.3. Evidence: - **Hosted reproduction** (scratch branch, not part of this PR): running shard 1/4 batches 1–5 in sequence on Bun 1.4.0 stalled 8 of 252 batch-5 runs at the same test (run 38213764818). Each dump shows the same state: - the test process main thread in state R, symbolized as `spawn_maybe_sync` → `tick_with_timeout` → `tick_tasks_only`; - the CLI child a zombie; - its pidfd armed level-triggered in spawnSync's private epoll but never waited on. - **Hosted A/B:** the same sequence on Bun 1.4.3 had 0 stalls in 280 batch-5 runs and no crash in 1,400 batches (run 38215466714). Running the 12-file batch alone never stalled in 450 runs (run 38213203939). - **Local:** upstream's mechanism with one finalized poll hung past its own timeout in 10 of 12 runs on 1.4.0 and passed 12 of 12 on 1.4.3. **Changes** - `package.json` `testRunnerBun` 1.4.0 → 1.4.3. The shipped runtime (`dependencies.bun` 1.4.2) and Docker image are unchanged. The pin rationale is updated in the setup action, `ci.yml`, `Dockerfile`, `structure/runtime.md`, `CONTRIBUTING.md` and the docs-site contributing page, and the pin assertion in `install-scripts.test.ts` is updated. - New `tests/ci-workflows/test-runner-spawnsync-finalizer.test.ts` (+ child fixture): reproduces the mechanism in a child of the test-runner Bun under a bounded async parent (10 s deadline, 3 s reap grace, settles even if exit or EOF never arrive). It fails on 1.4.0 (one writer: never returns; three writers: timeout with empty output) and passes on 1.4.3. Skipped on Windows, like upstream's case. Closes #6984 --- .github/actions/setup-project-bun/action.yml | 6 +- .github/workflows/ci.yml | 5 +- CONTRIBUTING.md | 4 +- Dockerfile | 5 +- .../000_plan.md | 27 +++++ docs-site/src/content/docs/contributing.md | 4 +- package.json | 2 +- scripts/test-layout/layout.json | 2 +- src/claude/intercept/picker-listener.ts | 21 +++- src/codex/prompt-lock-claim.ts | 16 ++- src/codex/prompt-lock-owner.ts | 12 ++- structure/clients/claude-desktop.md | 9 +- structure/codex-home.md | 4 + structure/runtime.md | 6 +- tests/ci-workflows/ci-scope-gaps.test.ts | 2 +- tests/ci-workflows/install-scripts.test.ts | 10 +- .../test-runner-spawnsync-finalizer.test.ts | 98 +++++++++++++++++++ .../claude-picker-upload.test.ts | 25 +++-- .../codex-prompt-lock.test.ts | 35 +++++++ .../spawnsync-finalizer-drift-child.ts | 54 ++++++++++ tests/fixtures/test-layout-expected.json | 1 + tests/server/server-auth.test.ts | 6 +- 22 files changed, 318 insertions(+), 36 deletions(-) create mode 100644 devlog/_plan/261012_4956_spawnsync_exit_stall/000_plan.md create mode 100644 tests/ci-workflows/test-runner-spawnsync-finalizer.test.ts create mode 100644 tests/fixtures/spawnsync-finalizer-drift-child.ts diff --git a/.github/actions/setup-project-bun/action.yml b/.github/actions/setup-project-bun/action.yml index f758ad64ff8..1ab22dd1dc9 100644 --- a/.github/actions/setup-project-bun/action.yml +++ b/.github/actions/setup-project-bun/action.yml @@ -4,8 +4,10 @@ description: >- runtime (default), or testRunnerBun for test-runner. # Runtime 1.4.2 fixes #6684's Windows Bun.fetch stream truncation. The test runner -# stays on 1.4.0: #4821 reverted #4064 after 1.4.2's multi-file --isolate preload -# segfault; #6713 run 37631558263 reproduces it on Linux 1/4, macOS 2/2, Windows 9/9. +# skipped 1.4.2: #4821 reverted #4064 after its multi-file --isolate preload segfault +# (#6713 run 37631558263: Linux 1/4, macOS 2/2, Windows 9/9). It moved from 1.4.0 to +# 1.4.3 for oven-sh/bun#44581: 1.4.0's Bun.spawnSync could lose a child's exit after +# finalizers ran inside an earlier call and spin past its own timeout (#4956). # Rejoin only after the upstream runner crash is fixed and a green lane=all # dispatch proves the unified version; then delete testRunnerBun and this role input. inputs: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 34ebb9411c4..34de6c49240 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -550,8 +550,9 @@ jobs: printf 'scheduled run tests dev at %s (workflow %s)\n' "$sha" "$GITHUB_SHA" # Bun roles (#6713): runtime 1.4.2 fixes #6684's Windows Bun.fetch streaming; - # test-runner 1.4.0 avoids 1.4.2's multi-file --isolate preload segfault (#4821 - # reverted #4064; run 37631558263: Linux 1/4, macOS 2/2, Windows 9/9). + # the test runner skipped 1.4.2's multi-file --isolate preload segfault (#4821 + # reverted #4064; run 37631558263: Linux 1/4, macOS 2/2, Windows 9/9) and runs + # 1.4.3, whose oven-sh/bun#44581 stops Bun.spawnSync losing a child's exit (#4956). # Rejoin after the upstream runner crash is fixed and a green lane=all dispatch # proves the unified version, then delete testRunnerBun and the action's role input. # The suite, split by file across four Linux runners. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4c74e70a2d1..f6b200e609b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -61,8 +61,8 @@ A ready-for-review PR is the author's claim that the change is complete, underst ## Local validation and hooks -The suite uses Bun 1.4.0 (`testRunnerBun`) while the shipped runtime is 1.4.2. -`scripts/test.ts` finds a 1.4.0 binary on PATH or in `~/.bun/bin`, or uses +The suite uses Bun 1.4.3 (`testRunnerBun`) while the shipped runtime is 1.4.2. +`scripts/test.ts` finds a 1.4.3 binary on PATH or in `~/.bun/bin`, or uses `OCX_TEST_RUNNER_BUN`; install that version locally if it is missing. Run `bun run test` before review readiness. If the full local suite is too costly diff --git a/Dockerfile b/Dockerfile index c8e587fe934..7e36f2063a0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,10 @@ # syntax=docker/dockerfile:1 # Keep the runtime aligned with package.json and pin the multi-platform image index. -# Runtime 1.4.2 fixes #6684's Windows Bun.fetch streaming; CI tests stay on 1.4.0 +# Runtime 1.4.2 fixes #6684's Windows Bun.fetch streaming; CI tests run on 1.4.3 # because 1.4.2 segfaults reloading preloads in multi-file --isolate (#4821; -# #6713 run 37631558263: Linux 1/4, macOS 2/2, Windows 9/9). Rejoin after the +# #6713 run 37631558263: Linux 1/4, macOS 2/2, Windows 9/9) and 1.4.0's spawnSync +# can lose a child's exit (#4956, oven-sh/bun#44581). Rejoin after the # upstream crash fix passes a unified-version lane=all dispatch, then delete # package.json's testRunnerBun pin and setup-project-bun's role input. ARG BUN_IMAGE=oven/bun:1.4.2@sha256:9114c058aeae42162ee16dd5084b95fe9473970bb6bcb5b232ab1630f0546895 diff --git a/devlog/_plan/261012_4956_spawnsync_exit_stall/000_plan.md b/devlog/_plan/261012_4956_spawnsync_exit_stall/000_plan.md new file mode 100644 index 00000000000..9a7eef39518 --- /dev/null +++ b/devlog/_plan/261012_4956_spawnsync_exit_stall/000_plan.md @@ -0,0 +1,27 @@ +# #4956 — Bun 1.4.0 spawnSync loses a child exit inside multi-file `bun test --isolate` + +## Evidence + +- All three hosted stalls of Linux shard 1/4 batch 5/57 (run 38162375606 attempts 1 and 2, job 114537864404 / 114545178864; PR run 38146205183 job 114490736681) stop at the same test: `tests/ci-workflows/release-version-sources.test.ts` "the check CLI fails naming the drifted source and passes when they agree". Five tests of that file printed; its sixth never did and no summary line followed. The issue comment's "every test passed, then Bun failed to exit" misreads the log. +- That test calls `Bun.spawnSync([bun, cli], { timeout: 15000, killSignal: "SIGKILL" })` several times. About 112 s passed between the last printed result and the 120 s batch kill, far beyond the 15 s bound, so the child-process timeout did not return. The hosted log alone does not say which call stalled; the scratch reproduction below does. +- Bun 1.4.0 (34cbb9a40) `spawn_maybe_sync` waits on a private uWS loop and, for the call's duration, repoints `vm.event_loop_handle` at it. A GC finalizer that runs during the call (an earlier file's `process.stderr` sink, a pipe reader, a `KeepAlive`) resolves its loop through the VM and decrements the private loop's `num_polls`. That loop is cached for the life of the process. Once the drift cancels a later call's polls, `us_loop_run_bun_tick` returns without calling `epoll_wait`/`kevent`, the child's exit poll is never read, and after the timeout fires the following ticks carry no deadline: the call spins at 100% CPU forever. Upstream: oven-sh/bun#44581 (merged 2026-10-05, `13a98b0dbd`, fixes oven-sh/bun#34069), first released in Bun v1.4.3 (2026-10-10). +- Local reproduction (macOS arm64) with upstream's fixture, one finalized `Bun.stderr` writer, stderr piped, `BUN_JSC_slowPathAllocsBetweenGCs=3`, inner spawnSync timeout 2 s, outer 6 s alarm: Bun 1.4.0 hung past its own timeout in 10/12 runs at 100% CPU; Bun 1.4.3 returned the child's output in 12/12 (~115 ms). With three writers 1.4.0 returns only at the timeout with empty output (3/3), 1.4.3 returns output (3/3). +- Composition dependence: in the suite the finalized objects come from earlier files in the same process (`--isolate` retires realms but keeps the VM, the FilePoll hive and the cached private loop), which can explain why each file alone passes and attribution sweeps stay clean. The mechanism itself does not need other files: a single script reproduces it. +- Hosted reproduction (scratch branch `codex/4956-diag-repro`, not for merge): 450 isolated runs of the 12-file batch never stalled (run 38213203939), but running shard 1/4 batches 1-5 in sequence like CI did, on Bun 1.4.0 (run 38213764818): 8 of 252 batch-5 runs stalled at the same test. Every dump shows the test process main thread in state R with no wait channel, symbolized as `spawn_maybe_sync` (js_bun_spawn_bindings.rs:1982/1956) -> `tick_with_timeout` -> `tick_tasks_only`; the CLI child is a zombie whose parent is the test process; its pidfd is registered level-triggered (`events: 19`) in spawnSync's private epoll and readable, yet never waited on. That state is consistent with the `num_polls == 0` early return #44581 removes (the dumps show the loop never waiting, not the counter itself). Hosted 1.4.3 A/B: run 38215466714. + +## Change + +1. `package.json` `testRunnerBun`: 1.4.0 → 1.4.3. The shipped runtime (`dependencies.bun` 1.4.2) is unchanged; unifying it is a product decision outside this fix. +2. Regression test `tests/ci-workflows/test-runner-spawnsync-finalizer.test.ts` + child `tests/fixtures/spawnsync-finalizer-drift-child.ts`: runs upstream's mechanism in a child of the test-runner Bun with stderr piped, for one and three finalized writers, and asserts the child saw every writer collected during the first call and that the second call returned the grandchild's output without timing out. The parent never blocks: a 10 s deadline kills the child, a 3 s reap grace follows, and the case settles whether or not the exit or either pipe EOF arrives (unref on expiry, kill failure ignored, timers cleared on every path). Three writers is the deterministic red oracle on 1.4.0; one writer exercises the unbounded spin. Skipped on Windows like upstream's case (POSIX mechanism, uses `sh`/`true`). +3. `tests/ci-workflows/install-scripts.test.ts` asserts the exact pin (`testRunnerBun` 1.4.0 -> 1.4.3) and its comment keeps the 1.4.2 history and adds this reason. +4. Rationale text where the pin is explained: `.github/actions/setup-project-bun/action.yml`, `.github/workflows/ci.yml` Bun-roles comment, `structure/runtime.md`, `CONTRIBUTING.md`, `docs-site/src/content/docs/contributing.md`. Injected 1.4.0 fixture values in `test-runner-bun.test.ts` and `ci-scope-gaps.test.ts` and transport minimums stay as they are. +5. Layout registration in `scripts/test-layout/layout.json` and `tests/fixtures/test-layout-expected.json`. + +## Risks and gates + +- 1.4.2's multi-file `--isolate` preload segfault (#4821, #6713) is the reason for the pin. 1.4.3 must show no runtime crash across Cross-platform CI `lane=all` (Linux 4 shards, macOS 2, Windows 9, serial families) at the exact PR head. If it reappears, this patch is abandoned and re-planned: the runner pin, the regression contract (which correctly fails on 1.4.0) and a workaround proof (async spawn with a bounded kill at the spawnSync-with-timeout fixtures) move together. Restoring 1.4.0 while skipping or weakening the regression is not an acceptable fallback. +- Version-coupled tests (`tests/test-layout-tooling.test.ts` asserts `Bun.version === testRunnerBun`) follow automatically. + +## Verification + +`bun run typecheck`; focused tests under 1.4.3 via `OCX_TEST_RUNNER_BUN` (new regression red on 1.4.0 / green on 1.4.3, 20 fresh children per case; install-scripts, test-runner-bun, ci-scope-gaps, docs-bun-source-requirement, both layout guards, cold-spawn-warmup); bounded-settlement fault cases (never-exiting child, grandchild holding pipes) in scratch; the 12-file batch repeated under 1.4.3; hosted 1.4.3 sequence A/B; `bun run test` under 1.4.3; `bun run structure:check`; `bun run privacy:scan`; exact-head `lane=all` inspected per batch for preload crashes and missing summaries. Workflow/dependency-pin surfaces need the maintainer security review named in MAINTAINERS.md before merge. diff --git a/docs-site/src/content/docs/contributing.md b/docs-site/src/content/docs/contributing.md index 888b80644a2..aa8668df8c5 100644 --- a/docs-site/src/content/docs/contributing.md +++ b/docs-site/src/content/docs/contributing.md @@ -45,8 +45,8 @@ bun run privacy:scan # credential/privacy scan used by CI bun run prepare:package # refresh package launchers/assets ``` -The suite uses Bun 1.4.0 (`package.json`'s `testRunnerBun`) while the shipped runtime is 1.4.2. -`scripts/test.ts` finds a 1.4.0 binary on PATH or in `~/.bun/bin`, or uses `OCX_TEST_RUNNER_BUN`. +The suite uses Bun 1.4.3 (`package.json`'s `testRunnerBun`) while the shipped runtime is 1.4.2. +`scripts/test.ts` finds a 1.4.3 binary on PATH or in `~/.bun/bin`, or uses `OCX_TEST_RUNNER_BUN`. Install that version locally if it is missing; the resolver never downloads it. Run `bun run test` by default. If a full run is disproportionately expensive for the task size, diff --git a/package.json b/package.json index ac2c33fdf02..b8713f17c09 100644 --- a/package.json +++ b/package.json @@ -41,7 +41,7 @@ "trustedDependencies": [ "bun" ], - "testRunnerBun": "1.4.0", + "testRunnerBun": "1.4.3", "scripts": { "dev": "bun run src/cli/index.ts start", "dev:proxy": "bun run src/cli/index.ts start", diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index 04b42fc64fb..b0a5fd267ec 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -1817,7 +1817,7 @@ "terminal-guard-server.test.ts": "server", "terminal-guard.test.ts": "server", "test-home-guard.test.ts": "ci-workflows", - "test-runner.test.ts": "ci-workflows", "test-runner-bun.test.ts": "ci-workflows", "test-sandbox-cleanup.test.ts": "ci-workflows", "test-temp-quarantine.test.ts": "ci-workflows", + "test-runner.test.ts": "ci-workflows", "test-runner-bun.test.ts": "ci-workflows", "test-runner-spawnsync-finalizer.test.ts": "ci-workflows", "test-sandbox-cleanup.test.ts": "ci-workflows", "test-temp-quarantine.test.ts": "ci-workflows", "thought-signature-credential-scope.test.ts": "responses", "token-estimate.test.ts": "lib", "token-guardian.test.ts": "codex-integration", diff --git a/src/claude/intercept/picker-listener.ts b/src/claude/intercept/picker-listener.ts index 23e8f5e1c8d..571dda753cd 100644 --- a/src/claude/intercept/picker-listener.ts +++ b/src/claude/intercept/picker-listener.ts @@ -205,8 +205,18 @@ export async function startPickerListener(options: PickerListenerOptions): Promi } else req.destroy(); } }; + // An HTTP/1.1 reply sent while the upload is unfinished advertises Connection: close. Bun 1.4.3 + // ends such a response with socket.destroySoon() before our finish listeners run, which resets + // the splice before the reply reaches the client; keep it a graceful FIN that discards input. + // Only that closing socket is changed, and destroy() stays available for errors and shutdown. + const closesUnfinishedUpload = () => !h2 && uploading && !req.complete; + const keepClosingReplyIntact = () => { + const socket = req.socket; + socket.destroySoon = () => { req.resume(); socket.end(); return socket; }; + }; // Refusals answer with an empty response and a fixed log line that carries no request data. const refuse = (status: 400 | 503) => { + if (closesUnfinishedUpload()) keepClosingReplyIntact(); // Do not leave a refused, still-uploading stream behind after delivering its empty reply. if (uploading && !req.complete) { req.once("error", () => res.destroy()); @@ -236,6 +246,7 @@ export async function startPickerListener(options: PickerListenerOptions): Promi if (clientGone) return; if (res.headersSent) res.destroy(); else { + if (closesUnfinishedUpload()) keepClosingReplyIntact(); res.writeHead(502, { "Content-Length": "0", ...(!h2 && uploading && !req.complete ? { Connection: "close" } : {}) }); res.end(); log(502); } @@ -260,7 +271,10 @@ export async function startPickerListener(options: PickerListenerOptions): Promi // HTTP/2 has no reason phrase; its compat writeHead takes the same flat raw header array. if (h2) (relayRes as Http2ServerResponse).writeHead(status, raw as unknown as Record); else { - if (uploading && !req.complete) raw.push("Connection", "close"); + if (uploading && !req.complete) { + keepClosingReplyIntact(); + raw.push("Connection", "close"); + } res.writeHead(status, upRes.statusMessage, raw); } log(status); @@ -438,6 +452,8 @@ export async function startPickerListener(options: PickerListenerOptions): Promi const offersH2 = clientHelloOffersH2(head); if (offersH2 === null) return; client.off("data", onData); + // Keep reading an aborted upload after the backend reply FIN. + client.allowHalfOpen = true; // Paused, a FIN or later bytes wait in the readable buffer and follow `head` once piped. client.pause(); clearTimeout(deadline); @@ -450,6 +466,9 @@ export async function startPickerListener(options: PickerListenerOptions): Promi // its peer (reset, dial failure) tears the peer down instead of leaving it open. inner.once("close", () => { spliced.delete(inner); if (!client.writableEnded) client.destroy(); }); client.once("close", () => { if (!inner.writableEnded) inner.destroy(); }); + // The backend reply ended; discard ciphertext still coming from the aborted upload. + // Unpipe before resuming, so backpressure cannot hide the peer FIN. + inner.once("end", () => { client.unpipe(inner); client.resume(); }); inner.once("connect", () => { inner.setNoDelay(true); client.setNoDelay(true); diff --git a/src/codex/prompt-lock-claim.ts b/src/codex/prompt-lock-claim.ts index b7410450a7f..8c299a3d6ea 100644 --- a/src/codex/prompt-lock-claim.ts +++ b/src/codex/prompt-lock-claim.ts @@ -1,6 +1,6 @@ import { linkSync, mkdirSync, readdirSync, readFileSync, renameSync, rmdirSync, unlinkSync, writeFileSync } from "node:fs"; import { dirname, join } from "node:path"; -import { ownEvidence, ownerState, safeNamespace, type OwnerDeps, type OwnerEvidence } from "./prompt-lock-owner"; +import { ownEvidence, ownerState, safeNamespace, safeNamespaceStat, type OwnerDeps, type OwnerEvidence } from "./prompt-lock-owner"; import { ChangedLock as ChangedClaim, checkEntry, checkIdentity, guarded, type Entry, type Namespace } from "./prompt-lock-evidence"; import { LockFileBusy, lockFileOperation } from "./prompt-lock-io"; @@ -41,8 +41,18 @@ export function withLockClaim( if (error instanceof LockFileBusy) return { ok: false }; if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; } - if (!safeNamespace(directory, "directory", deps)) throw new UnsafeLockNamespace(directory); - const parents = [root, { path: directory, stat: io(() => deps.lstat(directory)) }]; + // A peer's cleanup removes the empty claims directory, so it can vanish between our mkdir and + // this read. That is contention, not an unsafe namespace and not a crash: report it as busy. + // One observation is both validated and kept as the parent witness, so a replacement that + // appears after a missing read can never be trusted without its own ownership check. + let directoryStat: ReturnType; + try { directoryStat = io(() => deps.lstat(directory)); } + catch (error) { + if (error instanceof LockFileBusy || (error as NodeJS.ErrnoException).code === "ENOENT") return { ok: false }; + throw error; + } + if (!safeNamespaceStat(directoryStat, "directory", deps)) throw new UnsafeLockNamespace(directory); + const parents = [root, { path: directory, stat: directoryStat }]; const claimDirectoryGone = (): boolean => { try { const now = deps.lstat(directory); return now.dev !== parents[1].stat.dev || now.ino !== parents[1].stat.ino; } catch { return true; } diff --git a/src/codex/prompt-lock-owner.ts b/src/codex/prompt-lock-owner.ts index b2286ba7534..4dbbc18ffb0 100644 --- a/src/codex/prompt-lock-owner.ts +++ b/src/codex/prompt-lock-owner.ts @@ -121,12 +121,16 @@ export function ownerState(record: OwnerEvidence | null, deps: OwnerDeps): "dead return deps.isProcessAlive(record.pid) === false ? "dead" : "live"; } catch { return "live"; } } +/** Type, no symlink, and current-user ownership on POSIX, judged on one captured observation. */ +export function safeNamespaceStat(stat: LockStat, kind: "file" | "directory", deps: OwnerDeps): boolean { + const uid = deps.uid(); + return !stat.isSymbolicLink() && (kind === "file" ? stat.isFile() : stat.isDirectory()) + && (deps.platform === "win32" || (uid !== undefined + && stat.uid === (typeof stat.uid === "bigint" ? BigInt(uid) : uid))); +} export function safeNamespace(path: string, kind: "file" | "directory", deps: OwnerDeps, missing = false): boolean { try { - const stat = lockFileOperation(() => deps.lstat(path), deps.platform), uid = deps.uid(); - return !stat.isSymbolicLink() && (kind === "file" ? stat.isFile() : stat.isDirectory()) - && (deps.platform === "win32" || (uid !== undefined - && stat.uid === (typeof stat.uid === "bigint" ? BigInt(uid) : uid))); + return safeNamespaceStat(lockFileOperation(() => deps.lstat(path), deps.platform), kind, deps); } catch (error) { if (error instanceof LockFileBusy) throw error; return missing && (error as NodeJS.ErrnoException).code === "ENOENT"; diff --git a/structure/clients/claude-desktop.md b/structure/clients/claude-desktop.md index f2b6e80270a..98d075d280b 100644 --- a/structure/clients/claude-desktop.md +++ b/structure/clients/claude-desktop.md @@ -232,7 +232,14 @@ HTTP/1.1 refusals, generated 502s and early replies advertise `Connection: close the listener unpipes the upstream upload, reads and discards remaining input, and ends the socket gracefully so queued response bytes flush. Errors and premature downstream closes still destroy the request. Draining adds no upload deadline or byte cap and does not wait for request-body -completion before ending the response side. Body completion removes the input-cleanup listeners, +completion before ending the response side. Bun 1.4.3's HTTP server ends a `Connection: close` +response with `socket.destroySoon()` before the listener's own finish handler runs, which reset +the splice before the reply reached the client; every HTTP/1.1 reply that closes unfinished input +(refusal, generated 502, early upstream reply) therefore replaces that socket's `destroySoon` with +the same resume-and-`end()`. The TCP front +enables half-open reads once a ClientHello is recognized and, when the backend side ends, unpipes +and keeps discarding client ciphertext, so a client still writing an abandoned upload reads the +reply and FIN and can close (Bun 1.4.2 otherwise left it open). Body completion removes the input-cleanup listeners, leaving long-lived responses and SSE subscriptions independent of upload cleanup. Completed HTTP/1.1 uploads retain keep-alive on generated 502s and can reuse the same client socket. `tests/claude-integration/claude-picker-upload.test.ts` covers headers-only refusals on both protocols, byte-exact early replies under backpressure, completed uploads with SSE, cancellation, diff --git a/structure/codex-home.md b/structure/codex-home.md index 4e89e552e1f..35d34d0aad0 100644 --- a/structure/codex-home.md +++ b/structure/codex-home.md @@ -323,6 +323,10 @@ config parent. Both leases release on refusal or completion. PID/token file and bakery ticket. A contender still choosing makes peers refuse, and a later contender sees an earlier published ticket. Dead reservations are removed by their unique filename; stale observation cannot rename a live successor's reusable lock path. +A peer's cleanup removes the empty claims directory, so a claims directory that vanishes between a +contender's `mkdir` and its read is contention (`locked`), never an unsafe namespace or a crash. +That single read is both the ownership/type check and the parent witness, so a replacement that +appears after a missing read is never trusted. Lock and claim records include host evidence and optional process-start evidence. Takeover requires a matching local host and two dead-PID observations; a reused PID with a different known start identity remains busy. Unknown start identity disables only that comparison: live or unknown PID liveness diff --git a/structure/runtime.md b/structure/runtime.md index 3ac1b73f729..205c288d8b2 100644 --- a/structure/runtime.md +++ b/structure/runtime.md @@ -4,9 +4,9 @@ The minute sweep checks persisted activation deadlines locally; only missing dea ## Bun runtime and test runner -`package.json` pins the shipped npm runtime (`dependencies.bun`) to 1.4.2 and the test runner (`testRunnerBun`) to 1.4.0; `@types/bun` remains unchanged. Runtime 1.4.2 fixes #6684's Windows Bun.fetch truncation after the first cleartext upstream body chunk. -The runner pin avoids 1.4.2's multi-file `bun test --isolate` preload reload segfault (`load_preloads -> JSModuleLoader::loadModule -> JSPromise::status`, address 0x10): #4064 introduced 1.4.2 on 2026-09-09 and #4821 reverted it on 2026-09-17. -#6713 run 37631558263 reproduces it on Linux 1/4, macOS 2/2 and Windows 9/9; single-file success does not prove the multi-file runner works. +`package.json` pins the shipped npm runtime (`dependencies.bun`) to 1.4.2 and the test runner (`testRunnerBun`) to 1.4.3; `@types/bun` remains unchanged. Runtime 1.4.2 fixes #6684's Windows Bun.fetch truncation after the first cleartext upstream body chunk. +The runner pin skipped 1.4.2's multi-file `bun test --isolate` preload reload segfault (`load_preloads -> JSModuleLoader::loadModule -> JSPromise::status`, address 0x10): #4064 introduced 1.4.2 on 2026-09-09 and #4821 reverted it on 2026-09-17. It left 1.4.0 for 1.4.3 because of oven-sh/bun#44581 (#4956). Bun 1.4.0's `Bun.spawnSync` pointed the VM at its private wait loop, so a finalizer that ran during one call (an earlier file's stdout/stderr sink, a stream reader) released its poll on that loop; the cached loop's poll count drifted to zero and a later call stopped polling for its child, spinning past its own `timeout` until the 120-second batch deadline. In a multi-file `--isolate` process that depends on what earlier files left to finalize, so single-file attribution stayed clean. `tests/ci-workflows/test-runner-spawnsync-finalizer.test.ts` reproduces the mechanism in a child under a bounded parent and fails on 1.4.0. +#6713 run 37631558263 reproduces that 1.4.2 preload segfault on Linux 1/4, macOS 2/2 and Windows 9/9; single-file success does not prove the multi-file runner works. `.github/actions/setup-project-bun/action.yml` defaults to role `runtime` (`dependencies.bun`); `test-runner` resolves `testRunnerBun`. Unknown roles and missing or invalid pins fail closed. `.github/workflows/ci.yml` verifies both resolved and installed versions in `setup-action`; `tests/ci-workflows/ci-scope-gaps.test.ts` guards job roles. Test jobs, including single-file development-version validation, select `test-runner`. They also build dashboard fixtures; `gates` uploads a dashboard preview and checks release-helper syntax on that pin. diff --git a/tests/ci-workflows/ci-scope-gaps.test.ts b/tests/ci-workflows/ci-scope-gaps.test.ts index 0169eae7f96..370dea1bf49 100644 --- a/tests/ci-workflows/ci-scope-gaps.test.ts +++ b/tests/ci-workflows/ci-scope-gaps.test.ts @@ -62,7 +62,7 @@ describe("an edit to the setup action alone", () => { }); }); -// #6713: user runtimes and compiled artifacts need 1.4.2; isolate runners need 1.4.0. +// #6713: user runtimes and compiled artifacts need 1.4.2; isolate runners need the testRunnerBun pin (1.4.3, #4956). // Check the active setup at each test command, so a later role switch cannot evade the guard. const runsBunTests = (step: Step): boolean => /\bbun\s+test\b|\bbun\s+run\s+test(?::changed)?(?:\s|$)|\bbun\s+(?:run\s+)?scripts\/test\.ts\b|scripts\/ci\/run-bun-test-batches\.sh/.test( diff --git a/tests/ci-workflows/install-scripts.test.ts b/tests/ci-workflows/install-scripts.test.ts index e529af33ff0..55b21004ff5 100644 --- a/tests/ci-workflows/install-scripts.test.ts +++ b/tests/ci-workflows/install-scripts.test.ts @@ -67,7 +67,7 @@ describe("install scripts", () => { expect(pkg.exports?.["."]?.bun).toBe("./src/index.ts"); expect(pkg.exports?.["."]?.default).toBe("./bin/package-main.mjs"); // Bun 1.4.2 fixes the Windows fetch streaming regression reported in #6684. - // CI's testRunnerBun stays on 1.4.0: 1.4.2 segfaults while RE-LOADING the bunfig + // CI's testRunnerBun skipped 1.4.2: it segfaults while RE-LOADING the bunfig // preload that `--isolate` re-enters once per test file: // `load_preloads -> JSModuleLoader::loadModule -> JSPromise::status` dereferences a dead // promise and the process dies with "Segmentation fault at address 0x10". It is a crash in @@ -87,10 +87,16 @@ describe("install scripts", () => { // #4821 (2026-09-17) reverted #4064 (2026-09-09). #6713 run 37631558263 reproduced // the crash on Linux 1/4, macOS 2/2 and Windows 9/9; individual files pass. // Today's diagnostic sweep cannot turn a crashed primary batch green. + // + // testRunnerBun then moved from 1.4.0 to 1.4.3 for oven-sh/bun#44581 (#4956): 1.4.0's + // Bun.spawnSync lost a child's exit once finalizers released polls on its private wait loop, + // and spun past its own timeout. Scratch run 38213764818 stalled 8 of 252 Linux shard 1/4 + // batch-5 runs that way (CLI child a zombie, its pidfd armed but never polled). + // tests/ci-workflows/test-runner-spawnsync-finalizer.test.ts keeps the runner above it. // Rejoin after the upstream runner crash is fixed and a green `lane=all` dispatch // proves the unified version; then delete testRunnerBun and the setup action's role. expect(pkg.dependencies?.bun).toBe("1.4.2"); - expect(pkg.testRunnerBun).toBe("1.4.0"); + expect(pkg.testRunnerBun).toBe("1.4.3"); expect(pkg.dependencies?.zod).toBe("4.4.3"); expect(pkg.devDependencies?.typescript).toBe("7.0.2"); expect(pkg.devDependencies?.["@types/bun"]).toBe("1.4.0"); diff --git a/tests/ci-workflows/test-runner-spawnsync-finalizer.test.ts b/tests/ci-workflows/test-runner-spawnsync-finalizer.test.ts new file mode 100644 index 00000000000..a7e590f1a2c --- /dev/null +++ b/tests/ci-workflows/test-runner-spawnsync-finalizer.test.ts @@ -0,0 +1,98 @@ +import { describe, expect, test } from "bun:test"; +import { fixturePath } from "../helpers/repo-root"; +import { SPAWN_BUDGET_MS } from "../helpers/test-budget"; + +/** + * The test runner's Bun must deliver a spawnSync child's exit after finalizers ran inside an + * earlier spawnSync (#4956). + * + * Bun 1.4.0 loses it: the call spins at full CPU past its own `timeout`, so a bounded + * `Bun.spawnSync` in one file can hold a whole multi-file `--isolate` batch until the batch + * deadline kills it, after every earlier file passed and with nothing to attribute. That is + * how Linux shard 1/4 batch 5 timed out in runs 38146205183 and 38162375606 inside + * release-version-sources.test.ts. Bun 1.4.3 carries the upstream fix (oven-sh/bun#44581); + * `package.json` `testRunnerBun` must stay at or above it. + * + * The child runs the mechanism directly rather than hoping a batch composition triggers it. + * One finalized poll can make the second call spin forever on 1.4.0; three make it return only at + * its timeout with no output, every time. `collectedDuringCall` keeps the case honest: if the collection + * ever stops landing inside the first call, the assertion names that instead of passing. + * + * The parent waits asynchronously under its own deadline, so a regressed runtime fails this + * test instead of wedging the batch it runs in. The child shells out to `sh` and `true`; + * upstream skips its case on Windows for the same reason. + */ + +const CHILD = fixturePath("spawnsync-finalizer-drift-child.ts"); +/** Far above the ~0.1 s the fixed runtime needs and the 2 s inner timeout a regressed one hits. */ +const OUTER_DEADLINE_MS = 10_000; +/** After the deadline's SIGKILL, how long the exit and both pipe EOFs get before the case gives up on them. */ +const REAP_GRACE_MS = 3_000; + +type DriftOutcome = + | { kind: "settled"; exitCode: number; stdout: string; stderr: string } + | { kind: "deadline"; reaped: boolean } + | { kind: "read-error"; error: string; reaped: boolean }; + +/** + * Bounded on every path: the deadline kills the child and the case settles after a short reap + * grace whether or not the kill is delivered, the exit is observed, or a pipe reaches EOF. A + * regressed runtime therefore fails this case instead of holding its batch, which is the very + * failure the case exists to catch. + */ +async function runDriftChild(writers: number): Promise { + const child = Bun.spawn([process.execPath, CHILD], { + env: { ...process.env, OCX_DRIFT_WRITERS: String(writers), BUN_JSC_slowPathAllocsBetweenGCs: "3" }, + stdin: "ignore", + stdout: "pipe", + // Must be a pipe: a Bun.stderr writer registers a poll only when stderr is pollable. + stderr: "pipe", + }); + const work = Promise.all([new Response(child.stdout).text(), new Response(child.stderr).text(), child.exited]) + .then(value => ({ ok: true as const, value }), (error: unknown) => ({ ok: false as const, error: String(error) })); + const settledWork = work.then(() => true); + let deadline: ReturnType | undefined; + const expired = new Promise<"deadline">(resolve => { deadline = setTimeout(() => resolve("deadline"), OUTER_DEADLINE_MS); }); + let readError: string | undefined; + try { + const first = await Promise.race([work, expired]); + if (first !== "deadline" && first.ok) { + const [stdout, stderr, exitCode] = first.value; + return { kind: "settled", exitCode, stdout: stdout.trim(), stderr: stderr.trim().slice(-2_000) }; + } + // A pipe read that failed leaves the child unobserved; it gets the same bounded cleanup. + if (first !== "deadline") readError = first.error; + } finally { + clearTimeout(deadline); + } + try { child.kill("SIGKILL"); } catch { /* A failed kill must not extend the wait. */ } + let grace: ReturnType | undefined; + const reaped = await Promise.race([ + settledWork, + new Promise(resolve => { grace = setTimeout(() => resolve(false), REAP_GRACE_MS); }), + ]); + clearTimeout(grace); + // A child or pipe that outlived its kill must not keep this test process alive either. + if (!reaped) child.unref(); + return readError === undefined ? { kind: "deadline", reaped } : { kind: "read-error", error: readError, reaped }; +} + +describe.skipIf(process.platform === "win32")("test-runner Bun spawnSync after finalizers inside spawnSync", () => { + for (const writers of [1, 3]) { + test(`a child exit is still polled after ${writers} finalized poll(s)`, async () => { + const result = await runDriftChild(writers); + const report = JSON.stringify(result); + if (result.kind === "deadline") { + throw new Error(`spawnSync never returned within ${OUTER_DEADLINE_MS} ms (Bun ${Bun.version}): ${report}`); + } + if (result.kind === "read-error") throw new Error(`reading the drift child's output failed: ${report}`); + expect(result.exitCode, report).toBe(0); + const parsed = JSON.parse(result.stdout) as Record; + expect({ + collectedDuringCall: parsed.collectedDuringCall, + stdout: parsed.stdout, + exitedDueToTimeout: parsed.exitedDueToTimeout, + }, report).toEqual({ collectedDuringCall: writers, stdout: "polled\n", exitedDueToTimeout: false }); + }, SPAWN_BUDGET_MS); + } +}); diff --git a/tests/claude-integration/claude-picker-upload.test.ts b/tests/claude-integration/claude-picker-upload.test.ts index 18c4ea0eec5..8d4cd0d1859 100644 --- a/tests/claude-integration/claude-picker-upload.test.ts +++ b/tests/claude-integration/claude-picker-upload.test.ts @@ -212,7 +212,10 @@ for (const protocol of ["h1", "h2"] as const) { } finally { await wait(f.close()); } }); - bounded(`${protocol} upstream failure delivers 502 and releases an unfinished upload`, async wait => { + // "buffered" queues 4 MiB of an advertised 32 MiB body, so the generated 502 races input still in + // flight (Bun 1.4.3 reset that reply through the splice before the client could read it). + for (const input of protocol === "h1" ? ["small", "buffered"] as const : ["small"] as const) + bounded(`${protocol} upstream failure delivers 502 and releases ${input === "buffered" ? "a buffered" : "an"} unfinished upload`, async wait => { const f = await fixture((req, res) => { req.on("error", () => res.destroy()); req.resume(); if (req.url === "/failed") req.socket.destroy(); else res.end("ordinary"); @@ -221,10 +224,10 @@ for (const protocol of ["h1", "h2"] as const) { const session = f.h2(); const upload = protocol === "h2" ? session.request({ ":method": "POST", ":path": "/failed", ":authority": "claude.ai" }) - : f.h1("/failed", "POST", { "Content-Length": "100" }); + : f.h1("/failed", "POST", { "Content-Length": input === "buffered" ? String(32 * 1024 * 1024) : "100" }); const closed = protocol === "h2" ? observeClose(upload) : observeSocketClose(upload as ClientRequest); const result = protocol === "h2" ? h2Result(upload as ClientHttp2Stream) : h1Result(upload as ClientRequest); - upload.write("x"); + upload.write(input === "buffered" ? Buffer.alloc(4 * 1024 * 1024, 0x61) : "x"); expect(await wait(result)).toEqual({ status: 502, text: "" }); await wait(closed); if (protocol === "h2") expect((upload as ClientHttp2Stream).rstCode).toBe(constants.NGHTTP2_NO_ERROR); const next = session.request({ ":method": "POST", ":path": "/next", ":authority": "claude.ai" }); @@ -347,9 +350,16 @@ bounded("h2 cancellation releases aggregate admission once without weakening the }); for (const protocol of ["h1", "h2"] as const) { - bounded(`${protocol} early upstream response closes an unfinished upload without losing the reply`, async wait => { + // "buffered" answers early while 4 MiB of an advertised 32 MiB body is still queued. + for (const input of protocol === "h1" ? ["small", "buffered"] as const : ["small"] as const) + bounded(`${protocol} early upstream response closes ${input === "buffered" ? "a buffered" : "an"} unfinished upload without losing the reply`, async wait => { const f = await fixture((req, res) => { - req.on("error", () => res.destroy()); + // Answer like a real origin that replies early: keep reading the body and close with a FIN. + // Bun's own server would close this unfinished request with destroySoon(), a reset that + // Windows delivers before the reply bytes, so the relay could never read what it should relay. + req.on("error", () => res.destroy()); req.resume(); + const socket = req.socket; + socket.destroySoon = () => { socket.end(); return socket; }; if (req.url === "/early") { res.writeHead(413, { "Content-Length": "5" }); res.end("early"); } else res.end("ordinary"); }); @@ -357,9 +367,10 @@ for (const protocol of ["h1", "h2"] as const) { const session = f.h2(); const upload = protocol === "h2" ? session.request({ ":method": "POST", ":path": "/early", ":authority": "claude.ai" }) - : f.h1("/early", "POST", { "Content-Length": "100" }); + : f.h1("/early", "POST", { "Content-Length": input === "buffered" ? String(32 * 1024 * 1024) : "100" }); const result = protocol === "h2" ? h2Result(upload as ClientHttp2Stream) : h1Result(upload as ClientRequest); - const closed = protocol === "h2" ? observeClose(upload) : observeSocketClose(upload as ClientRequest); upload.write("x"); + const closed = protocol === "h2" ? observeClose(upload) : observeSocketClose(upload as ClientRequest); + upload.write(input === "buffered" ? Buffer.alloc(4 * 1024 * 1024, 0x61) : "x"); expect(await wait(result)).toEqual({ status: 413, text: "early" }); await wait(closed); if (protocol === "h2") expect((upload as ClientHttp2Stream).rstCode).toBe(constants.NGHTTP2_NO_ERROR); diff --git a/tests/codex-integration/codex-prompt-lock.test.ts b/tests/codex-integration/codex-prompt-lock.test.ts index d06f0d9de73..7cc149660e5 100644 --- a/tests/codex-integration/codex-prompt-lock.test.ts +++ b/tests/codex-integration/codex-prompt-lock.test.ts @@ -111,6 +111,41 @@ describe("staleness", () => { }); describe("interleavings", () => { + // A peer's cleanup rmdirs the empty claims directory; it can vanish between our mkdir and our + // reads of it. That is contention: the contender must report "locked", not crash or call it unsafe. + for (const read of [1, 2] as const) { + test(`a claims directory removed by a peer before read ${read} reports contention`, () => { + const path = lockPath(), claims = path + ".claims"; + let reads = 0; + const deps: LockDeps = { ...alive, lstat: target => { + if (target === claims && ++reads === read) rmSync(claims, { recursive: true, force: true }); + return ownerDefaults.lstat(target); + } }; + expect(tryAcquire(path, deps)).toEqual({ ok: false, error: "locked" }); + expect(existsSync(path)).toBe(false); + expect(tryAcquire(path, alive).ok).toBe(true); + }); + } + // The read that sees the directory is also the one judged: a foreign replacement that appears + // after a read found it missing is never trusted as the claims namespace or run in. + test.skipIf(process.platform === "win32")("a foreign directory replacing a removed claims directory is never trusted", () => { + const path = lockPath(), claims = path + ".claims", fs = require("node:fs") as typeof import("node:fs"); + let reads = 0; + const deps: LockDeps = { ...alive, lstat: target => { + if (target !== claims) return ownerDefaults.lstat(target); + reads += 1; + // First read: a peer has just removed the directory. Later reads: a foreign one replaced it. + if (reads === 1) rmSync(claims, { recursive: true, force: true }); + else if (!existsSync(claims)) fs.mkdirSync(claims); + const stat = ownerDefaults.lstat(target); + Object.defineProperty(stat, "uid", { value: BigInt((process.getuid?.() ?? 0) + 1) }); + return stat; + } }; + expect(tryAcquire(path, deps)).toEqual({ ok: false, error: "locked" }); + expect(existsSync(path)).toBe(false); + expect(readdirSync(join(path, ".."))).not.toContain("opencodex-prompt.lock"); + }); + test("real concurrent contenders admit at most one live owner and leave retries usable", async () => { const path = lockPath(), go = path + ".go", stop = path + ".stop"; writeFileSync(path, JSON.stringify({ ...ownEvidence(ownerDefaults), token: "old", pid: 999999999, acquiredAt: 0 })); diff --git a/tests/fixtures/spawnsync-finalizer-drift-child.ts b/tests/fixtures/spawnsync-finalizer-drift-child.ts new file mode 100644 index 00000000000..d3fed42941f --- /dev/null +++ b/tests/fixtures/spawnsync-finalizer-drift-child.ts @@ -0,0 +1,54 @@ +// Child for tests/ci-workflows/test-runner-spawnsync-finalizer.test.ts (#4956). +// +// Bun 1.4.0 points the VM at Bun.spawnSync's private event loop for the length of the call, so a +// FilePoll that a garbage collection finalizes during the call is released against that loop +// instead of the thread's. The private loop is cached for the life of the process, so its poll +// count drifts down; once the drift cancels a later call's polls, that call never polls for its +// child again and spins, past its own timeout (oven-sh/bun#34069, fixed by oven-sh/bun#44581 in +// Bun 1.4.3). In a multi-file `bun test --isolate` process the finalized objects are earlier files' +// stdout/stderr sinks and stream readers, which is how a 15-second spawnSync bound in +// release-version-sources.test.ts stalled shard 1/4 batch 5 until the 120-second batch deadline. +// +// This reproduces it deterministically, after upstream's own regression case: stderr is a pipe, so +// each Bun.stderr writer registers a poll; the first spawnSync collects them while it waits; the +// second has to poll for a grandchild that outlives the registration of its polls. + +const state = globalThis as typeof globalThis & { driftWriters?: unknown[] | null }; +const count = Number(process.env.OCX_DRIFT_WRITERS ?? "1"); +const refs: WeakRef[] = []; +state.driftWriters = []; +for (let i = 0; i < count; i++) { + const writer = Bun.stderr.writer(); + writer.write(""); + writer.flush(); + state.driftWriters.push(writer); + refs.push(new WeakRef(writer)); +} +// A WeakRef keeps its target alive until the job that created it ends. +await Bun.sleep(0); +Bun.spawnSync({ + cmd: ["true"], + stdout: "pipe", + stderr: "ignore", + // maxBuffer is the last option spawnSync reads; collecting here lands the finalizers inside the call. + get maxBuffer() { + Bun.gc(true); + state.driftWriters = null; + return undefined; + }, +}); +const collectedDuringCall = refs.filter(ref => ref.deref() === undefined).length; +const started = performance.now(); +const second = Bun.spawnSync({ + cmd: ["sh", "-c", "sleep 0.1; echo polled"], + stdout: "pipe", + stderr: "pipe", + timeout: 2_000, +}); +console.log(JSON.stringify({ + runtime: Bun.version, + collectedDuringCall, + stdout: second.stdout.toString(), + exitedDueToTimeout: second.exitedDueToTimeout ?? false, + elapsedMs: Math.round(performance.now() - started), +})); diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index 58d33431d9f..88ce20ca193 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -1101,6 +1101,7 @@ "tencent-siliconflow-providers.test.ts": "gui", "terminal-continuation-owner-rotation.test.ts": "adapters", "terminal-guard-server.test.ts": "server", "terminal-guard.test.ts": "server", "test-home-guard.test.ts": "ci-workflows", "test-runner.test.ts": "ci-workflows", "test-runner-bun.test.ts": "ci-workflows", + "test-runner-spawnsync-finalizer.test.ts": "ci-workflows", "test-sandbox-cleanup.test.ts": "ci-workflows", "test-temp-quarantine.test.ts": "ci-workflows", "thought-signature-credential-scope.test.ts": "responses", "token-estimate.test.ts": "lib", "token-guardian.test.ts": "codex-integration", "tool-argument-integers.test.ts": "adapters", "tool-catalog-nudge.test.ts": "adapters", diff --git a/tests/server/server-auth.test.ts b/tests/server/server-auth.test.ts index 12ca927bc09..6fa2f8d87d6 100644 --- a/tests/server/server-auth.test.ts +++ b/tests/server/server-auth.test.ts @@ -3687,10 +3687,12 @@ describe("server local API auth", () => { expect(response.status).toBe(502); // #914: a dead-port refusal is pre-connection — host-wide, not account // evidence. Account health stays untouched; the (provider, host) ledger - // records the failure instead. + // records the failure instead, under whichever refusal code this Bun's fetch + // raised: 1.4.0 says ConnectionRefused, 1.4.3 says ECONNREFUSED, and + // PRE_CONNECT_REACHABILITY_CODES classifies both alike. expect(getCodexUpstreamHealth("pool-a")).toBeNull(); expect(getUpstreamHostHealth(upstreamHostHealthKey("openai", "https://chatgpt.com"))) - .toMatchObject({ consecutiveFailures: 1, lastFailureCode: "ConnectionRefused" }); + .toMatchObject({ consecutiveFailures: 1, lastFailureCode: expect.stringMatching(/^(?:ConnectionRefused|ECONNREFUSED)$/) }); } finally { await server.stop(true); } From c220eec4b8e63e5bb3ffd6a33157bf7c1b43db46 Mon Sep 17 00:00:00 2001 From: lidge-jun Date: Tue, 13 Oct 2026 01:09:27 +0900 Subject: [PATCH 2/9] ci(test): refresh shard durations from a full dispatch and give Windows its own table (#6982) (#6994) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Squashed from #6994 for the 2.84.0 release train. Refreshes the per-file duration tables that balance the hosted test shards (#6982). Windows gets its own table, because no single table balanced all three platforms. **Problem.** `scripts/ci/test-durations.tsv` was generated once from Linux run 35816902207 (2026-09-23). The suite now has 2,242 files, and 684 of them had no row, so each weighed the 157 ms median. Every shard printed the same prediction while the measured test time diverged. In the reference run below, Linux shards measured 452–572 s, macOS 745–816 s and Windows 838–1279 s (each Windows shard was predicted at 151–152 s). **Source data.** Green dev `lane=all` run [38250055508](https://github.com/lidge-jun/opencodex/actions/runs/38250055508) at `dc45bb2ca9`, which is also this PR's base. All 16 suite shards succeeded: Linux `test 1/4`–`4/4`, `macos 1/3`–`3/3`, `windows 1/9`–`9/9`. Both tables were written by the existing `bun scripts/ci/test-durations.ts refresh --source ... ...` from those job logs; nothing was hand-edited. | Table | Logs | Rows (before → after) | Re-measured / new / dropped | Fallback median for files without a row | |---|---|---|---|---| | `test-durations.tsv` (Linux, macOS) | 4 Linux + 3 macOS shards | 1,558 → 2,242 | 1,558 / 684 / 0 | 157 ms → 185 ms | | `test-durations.windows.tsv` (new) | 9 Windows shards | — → 2,242 | — / 2,242 / — | 428 ms | Files measured on several platforms record the mean of their samples, as the tool already did. The median fallback still applies to any file added after this refresh and is printed in each shard's log. **Why two tables.** I first fed all three platforms' logs into one table, which is the tool's normal path. Windows per-file cost is not a constant multiple of Linux or macOS cost: across the suite it averages about 4.7× Linux, and the ratio varies widely by file. A single table refreshed from all 16 logs made Linux worse than the stale table in all three evaluations and macOS worse in two of three (1 s better in the third). A Linux+macOS table left Windows worse in two of three. Estimated maximum shard time, in seconds (this run / out-of-sample run 38214884055 / out-of-sample run 38162375606): | Platform | Stale table | Linux+macOS table | Platform's own table | All three averaged into one table | |---|---|---|---|---| | Linux 4 shards | 572 / 551 / 550 | **526 / 509 / 530** | 518 / 514 / 534 | 588 / 570 / 590 | Closes #6982 --- scripts/ci/run-bun-test-batches.sh | 13 +- scripts/ci/test-durations.ts | 37 +- scripts/ci/test-durations.tsv | 3766 ++++++++++------- scripts/ci/test-durations.windows.tsv | 2246 ++++++++++ structure/ops/cross-platform-ci.md | 10 +- structure/ops/docs-and-release.md | 2 +- .../ci-workflows/ci-crash-disposition.test.ts | 6 + tests/ci-workflows/ci-shard-balance.test.ts | 85 +- 8 files changed, 4594 insertions(+), 1571 deletions(-) create mode 100644 scripts/ci/test-durations.windows.tsv diff --git a/scripts/ci/run-bun-test-batches.sh b/scripts/ci/run-bun-test-batches.sh index 023f4ba0576..9786ddaf898 100644 --- a/scripts/ci/run-bun-test-batches.sh +++ b/scripts/ci/run-bun-test-batches.sh @@ -294,7 +294,14 @@ done < <( # drops files is the one failure here that stays green. Each shard still runs its files in sorted # order. Refresh the table with scripts/ci/test-durations.ts from hosted job logs. batch_script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -readonly DURATIONS_FILE="${BUN_TEST_DURATIONS_FILE:-$batch_script_dir/test-durations.tsv}" +# Windows runners weigh files with their own table. Per-file cost there is not a constant multiple of +# Linux or macOS cost (process start and filesystem work dominate different files), and one shared +# table balanced either Linux and macOS or Windows, never all three (#6982). +default_durations_file="$batch_script_dir/test-durations.tsv" +if [[ "${RUNNER_OS:-}" == "Windows" && -f "$batch_script_dir/test-durations.windows.tsv" ]]; then + default_durations_file="$batch_script_dir/test-durations.windows.tsv" +fi +readonly DURATIONS_FILE="${BUN_TEST_DURATIONS_FILE:-$default_durations_file}" durations_source="/dev/null" if [[ -f "$DURATIONS_FILE" ]]; then durations_source="$DURATIONS_FILE" @@ -403,7 +410,9 @@ if (( pending_count > 0 )); then fi readonly TOTAL_BATCHES=${#BATCH_STARTS[@]} echo "Shard ${SHARD_SPEC}: ${#SELECTED_FILES[@]} files in ${TOTAL_BATCHES} primary Bun processes (scope ${TEST_FILE_SCOPE}, batch size <= ${BATCH_SIZE}, timeout ${BATCH_TIMEOUT_SECONDS}s)." -echo "Predicted shard time from recorded durations: $((predicted_ms / 1000))s; a file without a record weighs ${FALLBACK_MS}ms and a batch closes before ${BATCH_BUDGET_MS}ms." +durations_label="no duration table" +[[ "$durations_source" == "/dev/null" ]] || durations_label="${durations_source##*/}" +echo "Predicted shard time from recorded durations (${durations_label}): $((predicted_ms / 1000))s; a file without a record weighs ${FALLBACK_MS}ms and a batch closes before ${BATCH_BUDGET_MS}ms." echo "Nothing here is retried. A test failure, a process timeout and a Bun runtime crash each fail this shard on their first occurrence." echo "A timeout or a crash is additionally swept one file per process for attribution, after the shard has already failed; that sweep cannot turn it green." diff --git a/scripts/ci/test-durations.ts b/scripts/ci/test-durations.ts index b6c056bf4d3..2cdaa6eb321 100644 --- a/scripts/ci/test-durations.ts +++ b/scripts/ci/test-durations.ts @@ -9,25 +9,38 @@ * header, for the first file of a process) to its own end, so process start and module loading are * charged to the file that caused them. * - * Refresh from a green run on dev (all four Linux shards): + * Two tables are kept. Linux and macOS shards share `test-durations.tsv`; Windows shards weigh files + * with `test-durations.windows.tsv`, which the batch runner selects when RUNNER_OS is Windows. Per-file + * cost on Windows is not a constant multiple of the other two, so one table balanced either Linux and + * macOS or Windows, never all three (#6982). A file measured in several logs records the mean of its + * samples, which is why each table reads only its own platforms' logs. * - * gh run view --log > .tmp/ci-run.log - * bun scripts/ci/test-durations.ts refresh --source "run " .tmp/ci-run.log + * Refresh both from one green `lane=all` dispatch on dev. `gh run view --log` refuses a run this + * large, so read one job at a time: + * + * gh run view --job --log > .tmp/.log (every suite shard) + * bun scripts/ci/test-durations.ts refresh --source "run , ..." + * bun scripts/ci/test-durations.ts refresh --table scripts/ci/test-durations.windows.tsv \ + * --source "run , ..." + * + * Windows shards print their group headers with backslashes (`tests\a\b.test.ts:`); those paths are + * stored with forward slashes, the form `run-bun-test-batches.sh` looks up. * * Files measured in the logs replace their rows; rows for files that still exist are kept; rows for * files that no longer exist are dropped. Attribution sweeps (a failed shard re-running files one at a * time) are ignored because they do not measure the batch shape the table is used for. */ import { existsSync, readFileSync, writeFileSync } from "node:fs"; -import { dirname, join } from "node:path"; +import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; const REPO_ROOT = dirname(dirname(dirname(fileURLToPath(import.meta.url)))); export const DURATIONS_TABLE = join(REPO_ROOT, "scripts", "ci", "test-durations.tsv"); +export const WINDOWS_DURATIONS_TABLE = join(REPO_ROOT, "scripts", "ci", "test-durations.windows.tsv"); const LOG_LINE = /^(?:(.*?)\t[^\t]*\t)?\uFEFF?(\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d)(\.\d+)?Z (.*)$/; const BATCH_HEADER = /^##\[group\]shard \S+ batch \d+\/\d+(.*)$/; -const FILE_HEADER = /^##\[group\](tests\/.+):$/; +const FILE_HEADER = /^##\[group\](tests[\\/].+):$/; const ANSI = /\u001b\[[0-9;]*m/g; function timestampMs(seconds: string, fraction: string | undefined): number { @@ -60,7 +73,7 @@ export function parseJobLog(text: string): Map { } const file = FILE_HEADER.exec(body); if (file) { - state.file = state.attribution || state.previousEnd === null ? null : file[1]!; + state.file = state.attribution || state.previousEnd === null ? null : file[1]!.replaceAll("\\", "/"); continue; } if (body.startsWith("##[endgroup]") && state.file !== null && state.previousEnd !== null) { @@ -119,13 +132,15 @@ export function renderTable(durations: ReadonlyMap, source: stri if (import.meta.main) { const [command, ...rest] = process.argv.slice(2); let source = "unspecified"; + let table = DURATIONS_TABLE; const logs: string[] = []; for (let index = 0; index < rest.length; index += 1) { if (rest[index] === "--source") source = rest[++index] ?? source; + else if (rest[index] === "--table") table = resolve(rest[++index] ?? table); else logs.push(rest[index]!); } if (command !== "refresh" || logs.length === 0) { - console.error("usage: bun scripts/ci/test-durations.ts refresh [--source ] ..."); + console.error("usage: bun scripts/ci/test-durations.ts refresh [--table ] [--source ] ..."); process.exit(64); } const measured = new Map(); @@ -135,11 +150,11 @@ if (import.meta.main) { } } if (measured.size === 0) { - console.error("No per-file durations found; pass hosted logs of the Linux test shards."); + console.error("No per-file durations found; pass hosted logs of the suite shards."); process.exit(1); } - const previous = existsSync(DURATIONS_TABLE) ? parseTable(readFileSync(DURATIONS_TABLE, "utf8")) : new Map(); + const previous = existsSync(table) ? parseTable(readFileSync(table, "utf8")) : new Map(); const merged = mergeDurations(previous, measured, path => existsSync(join(REPO_ROOT, path))); - writeFileSync(DURATIONS_TABLE, renderTable(merged, source)); - console.log(`Recorded ${merged.size} files (${measured.size} measured) in ${DURATIONS_TABLE}.`); + writeFileSync(table, renderTable(merged, source)); + console.log(`Recorded ${merged.size} files (${measured.size} measured) in ${table}.`); } diff --git a/scripts/ci/test-durations.tsv b/scripts/ci/test-durations.tsv index b66d5022526..580ae237b20 100644 --- a/scripts/ci/test-durations.tsv +++ b/scripts/ci/test-durations.tsv @@ -1,1562 +1,2246 @@ # Per-file Bun test durations in milliseconds, read from hosted CI job logs. # Consumed by scripts/ci/run-bun-test-batches.sh to balance shards; a file without a row weighs the median. # Regenerate with scripts/ci/test-durations.ts (usage in its header); do not edit by hand. -# Source: run 35816902207, Linux test 1/4-4/4 -399 tests/adapters/abort-race.test.ts -524 tests/adapters/adapter-buffered-tool-conformance.test.ts -126 tests/adapters/adapter-error-inline.test.ts -1699 tests/adapters/adapter-inner-send-budget-wiring.test.ts -2899 tests/adapters/adapter-inner-send-budget.test.ts -153 tests/adapters/adapter-input-media-guard.test.ts -224 tests/adapters/adapter-registry-authority.test.ts -635 tests/adapters/adapter-tool-conformance.test.ts -147 tests/adapters/adapter-usage.test.ts -271 tests/adapters/anthropic/anthropic-account-pool.test.ts -86 tests/adapters/anthropic/anthropic-agentrouter-language-framing.test.ts -131 tests/adapters/anthropic/anthropic-baseurl-override.test.ts -1290 tests/adapters/anthropic/anthropic-compatible-stream.test.ts -93 tests/adapters/anthropic/anthropic-empty-content.test.ts -146 tests/adapters/anthropic/anthropic-eof-tolerance.test.ts -296 tests/adapters/anthropic/anthropic-error-body.test.ts -107 tests/adapters/anthropic/anthropic-error-stop-reason.test.ts -114 tests/adapters/anthropic/anthropic-fast-speed.test.ts -100 tests/adapters/anthropic/anthropic-hardening.test.ts -213 tests/adapters/anthropic/anthropic-image-guard.test.ts -3066 tests/adapters/anthropic/anthropic-image-normalize.test.ts -807 tests/adapters/anthropic/anthropic-image-retry-e2e.test.ts -201 tests/adapters/anthropic/anthropic-image-retry.test.ts -100 tests/adapters/anthropic/anthropic-parallel-tool-disable.test.ts -110 tests/adapters/anthropic/anthropic-pool-toggle-copy.test.ts -302 tests/adapters/anthropic/anthropic-quota-dispatch.test.ts -206 tests/adapters/anthropic/anthropic-ratelimit-headers.test.ts -153 tests/adapters/anthropic/anthropic-reasoning.test.ts -125 tests/adapters/anthropic/anthropic-reset-grants.test.ts -179 tests/adapters/anthropic/anthropic-sidecar-account-failover.test.ts -85 tests/adapters/anthropic/anthropic-stream-hardening.test.ts -88 tests/adapters/anthropic/anthropic-tail-guard.test.ts -182 tests/adapters/anthropic/anthropic-thinking-signature.test.ts -97 tests/adapters/anthropic/anthropic-tool-call-id.test.ts -175 tests/adapters/anthropic/anthropic-tool-declaration-constraints.test.ts -104 tests/adapters/anthropic/anthropic-tool-schema.test.ts -254 tests/adapters/bridge-legacy-shell-normalization.test.ts -3485 tests/adapters/bridge-lifecycle.test.ts -113 tests/adapters/bridge-nonstreaming-terminal.test.ts -281 tests/adapters/bridge-raw-reasoning-hidden.test.ts +# Source: run 38250055508 (dev dc45bb2ca9, lane=all), Linux test 1/4-4/4 and macos 1/3-3/3 +271 tests/adapters/abort-race.test.ts +643 tests/adapters/adapter-buffered-tool-conformance.test.ts +270 tests/adapters/adapter-error-inline.test.ts +1781 tests/adapters/adapter-inner-send-budget-wiring.test.ts +2955 tests/adapters/adapter-inner-send-budget.test.ts +234 tests/adapters/adapter-input-media-guard.test.ts +756 tests/adapters/adapter-registry-authority.test.ts +1782 tests/adapters/adapter-tool-conformance.test.ts +240 tests/adapters/adapter-usage.test.ts +779 tests/adapters/anthropic/anthropic-429-policy.test.ts +98 tests/adapters/anthropic/anthropic-account-metadata.test.ts +205 tests/adapters/anthropic/anthropic-account-pause-outbound.test.ts +456 tests/adapters/anthropic/anthropic-account-pause.test.ts +524 tests/adapters/anthropic/anthropic-account-pool.test.ts +873 tests/adapters/anthropic/anthropic-account-threshold.test.ts +104 tests/adapters/anthropic/anthropic-agentrouter-language-framing.test.ts +169 tests/adapters/anthropic/anthropic-baseurl-override.test.ts +104 tests/adapters/anthropic/anthropic-beta-allowlist.test.ts +95 tests/adapters/anthropic/anthropic-client-identity.test.ts +314 tests/adapters/anthropic/anthropic-combo-account-cooldown.test.ts +2635 tests/adapters/anthropic/anthropic-compatible-stream.test.ts +267 tests/adapters/anthropic/anthropic-cooldown-recovery.test.ts +123 tests/adapters/anthropic/anthropic-empty-content.test.ts +160 tests/adapters/anthropic/anthropic-eof-tolerance.test.ts +488 tests/adapters/anthropic/anthropic-error-body.test.ts +99 tests/adapters/anthropic/anthropic-error-stop-reason.test.ts +615 tests/adapters/anthropic/anthropic-fast-opt-in.test.ts +113 tests/adapters/anthropic/anthropic-fast-speed.test.ts +151 tests/adapters/anthropic/anthropic-haiku-5-5-contract.test.ts +119 tests/adapters/anthropic/anthropic-hardening.test.ts +181 tests/adapters/anthropic/anthropic-image-guard.test.ts +4497 tests/adapters/anthropic/anthropic-image-normalize.test.ts +892 tests/adapters/anthropic/anthropic-image-retry-e2e.test.ts +315 tests/adapters/anthropic/anthropic-image-retry.test.ts +888 tests/adapters/anthropic/anthropic-instance-isolation.test.ts +555 tests/adapters/anthropic/anthropic-instance-pool-parity.test.ts +1215 tests/adapters/anthropic/anthropic-instance-quota.test.ts +2265 tests/adapters/anthropic/anthropic-instance-recovery.test.ts +101 tests/adapters/anthropic/anthropic-message-stop-termination.test.ts +126 tests/adapters/anthropic/anthropic-messages-passthrough-oauth.test.ts +89 tests/adapters/anthropic/anthropic-messages-passthrough.test.ts +931 tests/adapters/anthropic/anthropic-model-routes.test.ts +572 tests/adapters/anthropic/anthropic-model-weekly-admission.test.ts +94 tests/adapters/anthropic/anthropic-opaque-strip.test.ts +93 tests/adapters/anthropic/anthropic-output-maxima.test.ts +108 tests/adapters/anthropic/anthropic-parallel-tool-disable.test.ts +186 tests/adapters/anthropic/anthropic-pool-toggle-copy.test.ts +1011 tests/adapters/anthropic/anthropic-quota-dispatch.test.ts +372 tests/adapters/anthropic/anthropic-ratelimit-headers.test.ts +245 tests/adapters/anthropic/anthropic-reasoning.test.ts +520 tests/adapters/anthropic/anthropic-reset-grants.test.ts +396 tests/adapters/anthropic/anthropic-revoked-token-boundaries.test.ts +298 tests/adapters/anthropic/anthropic-revoked-token-continuation.test.ts +366 tests/adapters/anthropic/anthropic-revoked-token-sidecars.test.ts +1437 tests/adapters/anthropic/anthropic-revoked-token.test.ts +240 tests/adapters/anthropic/anthropic-sidecar-account-failover.test.ts +85 tests/adapters/anthropic/anthropic-sonnet-5-5-contract.test.ts +108 tests/adapters/anthropic/anthropic-stream-hardening.test.ts +94 tests/adapters/anthropic/anthropic-tail-guard.test.ts +190 tests/adapters/anthropic/anthropic-thinking-signature.test.ts +124 tests/adapters/anthropic/anthropic-thinking-usage.test.ts +140 tests/adapters/anthropic/anthropic-tool-call-id.test.ts +217 tests/adapters/anthropic/anthropic-tool-declaration-constraints.test.ts +127 tests/adapters/anthropic/anthropic-tool-schema.test.ts +609 tests/adapters/anthropic/anthropic2-fast-parity.test.ts +156 tests/adapters/bridge-legacy-shell-normalization.test.ts +3563 tests/adapters/bridge-lifecycle.test.ts +190 tests/adapters/bridge-nonstreaming-terminal.test.ts +210 tests/adapters/bridge-raw-reasoning-hidden.test.ts 91 tests/adapters/bridge-reasoning-replay-batch.test.ts -92 tests/adapters/bridge-terminal-singleness.test.ts -198 tests/adapters/bridge.test.ts -139 tests/adapters/buffered-response-shape-guards.test.ts -89 tests/adapters/coding-agent-tool-result-images.test.ts -270 tests/adapters/empty-tool-output-annotation.test.ts -363 tests/adapters/exec-tool-result-normalize.test.ts -123 tests/adapters/google/antigravity-baseurl-override.test.ts -152 tests/adapters/google/antigravity-static-catalog.test.ts -877 tests/adapters/google/gcp-adc.test.ts -117 tests/adapters/google/gemini-37-flash-migration.test.ts -139 tests/adapters/google/gemini-web-search.test.ts -137 tests/adapters/google/google-adapter.test.ts -4106 tests/adapters/google/google-antigravity-oauth.test.ts -2533 tests/adapters/google/google-antigravity-replay.test.ts -135 tests/adapters/google/google-antigravity-wire.test.ts -102 tests/adapters/google/google-buffered-stop-reason.test.ts -108 tests/adapters/google/google-claude-prefill-guard.test.ts -112 tests/adapters/google/google-empty-content.test.ts -85 tests/adapters/google/google-errors.test.ts -121 tests/adapters/google/google-hardening.test.ts -156 tests/adapters/google/google-models-listing.test.ts -93 tests/adapters/google/google-output-clamp.test.ts -87 tests/adapters/google/google-provider-metadata-roundtrip.test.ts -141 tests/adapters/google/google-signature-history-roundtrip.test.ts -105 tests/adapters/google/google-strict-tool-validated-mode.test.ts -99 tests/adapters/google/google-structured-output.test.ts -92 tests/adapters/google/google-tool-result-adjacency.test.ts -128 tests/adapters/google/google-tool-schema-contract.test.ts -103 tests/adapters/google/google-tool-schema.test.ts -434 tests/adapters/google/google-vertex-http.test.ts -108 tests/adapters/google/google-vertex-stream.test.ts -98 tests/adapters/google/google-vertex-thought-signature.test.ts -103 tests/adapters/google/google-wire-compiler.test.ts -133 tests/adapters/google/google-wire-shape.test.ts -143 tests/adapters/google/vertex-catalog.test.ts -236 tests/adapters/identity-neutralize.test.ts -528 tests/adapters/key-failover.test.ts -128 tests/adapters/openai/inline-think-boundaries.test.ts -1962 tests/adapters/openai/openai-api-virtual-models.test.ts -302 tests/adapters/openai/openai-chat-bounded-tool-names.test.ts -103 tests/adapters/openai/openai-chat-dangling-toolcalls.test.ts -103 tests/adapters/openai/openai-chat-developer-position.test.ts -100 tests/adapters/openai/openai-chat-eof.test.ts -360 tests/adapters/openai/openai-chat-hardening.test.ts -6406 tests/adapters/openai/openai-chat-image-normalization.test.ts -125 tests/adapters/openai/openai-chat-inline-think-tags.test.ts -124 tests/adapters/openai/openai-chat-invalid-tool-call-diagnostics.test.ts -142 tests/adapters/openai/openai-chat-model-suffix.test.ts -418 tests/adapters/openai/openai-chat-native-policy.test.ts -240 tests/adapters/openai/openai-chat-parallel-stream.test.ts -265 tests/adapters/openai/openai-chat-path-override.test.ts -111 tests/adapters/openai/openai-chat-reasoning-wire-policy.test.ts -149 tests/adapters/openai/openai-chat-system-order.test.ts -90 tests/adapters/openai/openai-chat-tool-result-images.test.ts -92 tests/adapters/openai/openai-chat-url.test.ts -98 tests/adapters/openai/openai-chat-video-part.test.ts -1062 tests/adapters/openai/openai-provider-option-e2e.test.ts -100 tests/adapters/openai/openai-provider-option-migration.test.ts -105 tests/adapters/openai/openai-provider-option-startup.test.ts -92 tests/adapters/openai/openai-provider-option-tooling.test.ts -91 tests/adapters/openai/openai-provider-option.test.ts -122 tests/adapters/physical-send.test.ts -104 tests/adapters/reasoning-replay-identity.test.ts -115 tests/adapters/reasoning-replay-robustness.test.ts -154 tests/adapters/routed-agent-messages.test.ts -115 tests/adapters/run-turn-queue.test.ts -314 tests/adapters/terminal-continuation-owner-rotation.test.ts -104 tests/adapters/tool-argument-integers.test.ts -81 tests/adapters/tool-catalog-nudge.test.ts -92 tests/adapters/tool-choice-performance.test.ts -3069 tests/adapters/translator-budget.test.ts -108 tests/adapters/upstream-http-error.test.ts -2576 tests/ci-workflows/assert-mergeable-review.test.ts -194 tests/ci-workflows/brand-favicons.test.ts -94 tests/ci-workflows/build-desktop-icon-set.test.ts -107 tests/ci-workflows/build-release-changelog.test.ts -245 tests/ci-workflows/bump-dev-version.test.ts -100 tests/ci-workflows/bun-runtime.test.ts -84 tests/ci-workflows/ci-bun-crash-classifier.test.ts +206 tests/adapters/bridge-stall.test.ts +89 tests/adapters/bridge-terminal-singleness.test.ts +270 tests/adapters/bridge.test.ts +129 tests/adapters/buffered-response-shape-guards.test.ts +123 tests/adapters/coding-agent-tool-result-images.test.ts +371 tests/adapters/empty-tool-output-annotation.test.ts +201 tests/adapters/exec-tool-result-normalize.test.ts +161 tests/adapters/google/antigravity-baseurl-override.test.ts +97 tests/adapters/google/antigravity-discovered-families.test.ts +108 tests/adapters/google/antigravity-refusal.test.ts +104 tests/adapters/google/antigravity-static-catalog.test.ts +840 tests/adapters/google/gcp-adc.test.ts +103 tests/adapters/google/gemini-37-flash-migration.test.ts +145 tests/adapters/google/gemini-web-search.test.ts +128 tests/adapters/google/google-adapter.test.ts +4160 tests/adapters/google/google-antigravity-oauth.test.ts +2646 tests/adapters/google/google-antigravity-replay.test.ts +112 tests/adapters/google/google-antigravity-wire.test.ts +86 tests/adapters/google/google-buffered-stop-reason.test.ts +93 tests/adapters/google/google-claude-prefill-guard.test.ts +102 tests/adapters/google/google-empty-content.test.ts +98 tests/adapters/google/google-errors.test.ts +118 tests/adapters/google/google-hardening.test.ts +245 tests/adapters/google/google-models-listing.test.ts +87 tests/adapters/google/google-output-clamp.test.ts +88 tests/adapters/google/google-provider-metadata-roundtrip.test.ts +145 tests/adapters/google/google-signature-history-roundtrip.test.ts +98 tests/adapters/google/google-strict-tool-validated-mode.test.ts +96 tests/adapters/google/google-structured-output.test.ts +99 tests/adapters/google/google-tool-result-adjacency.test.ts +164 tests/adapters/google/google-tool-schema-contract.test.ts +118 tests/adapters/google/google-tool-schema.test.ts +3143 tests/adapters/google/google-vertex-http.test.ts +101 tests/adapters/google/google-vertex-stream.test.ts +114 tests/adapters/google/google-vertex-thought-signature.test.ts +85 tests/adapters/google/google-wire-compiler.test.ts +131 tests/adapters/google/google-wire-shape.test.ts +119 tests/adapters/google/vertex-catalog.test.ts +197 tests/adapters/identity-neutralize.test.ts +391 tests/adapters/identity-subagent.test.ts +629 tests/adapters/key-failover.test.ts +137 tests/adapters/openai/inline-think-boundaries.test.ts +2190 tests/adapters/openai/openai-api-virtual-models.test.ts +276 tests/adapters/openai/openai-chat-bounded-tool-names.test.ts +89 tests/adapters/openai/openai-chat-dangling-toolcalls.test.ts +102 tests/adapters/openai/openai-chat-developer-position.test.ts +166 tests/adapters/openai/openai-chat-eof.test.ts +102 tests/adapters/openai/openai-chat-glm-summary.test.ts +357 tests/adapters/openai/openai-chat-hardening.test.ts +6490 tests/adapters/openai/openai-chat-image-normalization.test.ts +144 tests/adapters/openai/openai-chat-inline-think-tags.test.ts +94 tests/adapters/openai/openai-chat-invalid-tool-call-diagnostics.test.ts +111 tests/adapters/openai/openai-chat-model-suffix.test.ts +387 tests/adapters/openai/openai-chat-native-policy.test.ts +94 tests/adapters/openai/openai-chat-parallel-stream.test.ts +113 tests/adapters/openai/openai-chat-path-override.test.ts +95 tests/adapters/openai/openai-chat-qwen38-leading-system.test.ts +143 tests/adapters/openai/openai-chat-reasoning-wire-policy.test.ts +252 tests/adapters/openai/openai-chat-sanitization-review-regressions.test.ts +117 tests/adapters/openai/openai-chat-serialized-tool-call-content.test.ts +298 tests/adapters/openai/openai-chat-serialized-tool-call-hold-bound.test.ts +225 tests/adapters/openai/openai-chat-serialized-tool-call-scaling.test.ts +154 tests/adapters/openai/openai-chat-serialized-tool-call-think.test.ts +129 tests/adapters/openai/openai-chat-system-order.test.ts +127 tests/adapters/openai/openai-chat-tool-call-id-remint.test.ts +87 tests/adapters/openai/openai-chat-tool-result-images.test.ts +86 tests/adapters/openai/openai-chat-url.test.ts +82 tests/adapters/openai/openai-chat-video-part.test.ts +168 tests/adapters/openai/openai-empty-assistant-history.test.ts +1275 tests/adapters/openai/openai-provider-option-e2e.test.ts +106 tests/adapters/openai/openai-provider-option-migration.test.ts +97 tests/adapters/openai/openai-provider-option-startup.test.ts +146 tests/adapters/openai/openai-provider-option-tooling.test.ts +94 tests/adapters/openai/openai-provider-option.test.ts +142 tests/adapters/physical-send.test.ts +160 tests/adapters/reasoning-replay-identity.test.ts +450 tests/adapters/reasoning-replay-robustness.test.ts +227 tests/adapters/routed-agent-messages.test.ts +143 tests/adapters/run-turn-queue.test.ts +520 tests/adapters/terminal-continuation-owner-rotation.test.ts +102 tests/adapters/tool-argument-integers.test.ts +211 tests/adapters/tool-catalog-nudge.test.ts +128 tests/adapters/tool-choice-performance.test.ts +154 tests/adapters/tool-envelope-echo-whole-line.test.ts +287 tests/adapters/translator-budget-lifetime.test.ts +5036 tests/adapters/translator-budget.test.ts +217 tests/adapters/upstream-http-error.test.ts +4323 tests/ci-workflows/assert-mergeable-review.test.ts +108 tests/ci-workflows/brand-favicons.test.ts +104 tests/ci-workflows/build-desktop-icon-set.test.ts +167 tests/ci-workflows/build-release-changelog.test.ts +298 tests/ci-workflows/bump-dev-version.test.ts +320 tests/ci-workflows/bun-runtime.test.ts +112 tests/ci-workflows/ci-bun-crash-classifier.test.ts 89 tests/ci-workflows/ci-concurrency-groups.test.ts -422 tests/ci-workflows/ci-crash-disposition.test.ts -3005 tests/ci-workflows/ci-review-lanes.test.ts -237 tests/ci-workflows/ci-scope-reduction.test.ts -180 tests/ci-workflows/ci-structure-gate.test.ts -766 tests/ci-workflows/ci-workflows.test.ts -88 tests/ci-workflows/cleanup-orphaned-workflows.test.ts -93 tests/ci-workflows/closed-pr-branch-cleanup.test.ts -6512 tests/ci-workflows/cold-spawn-warmup.test.ts -238 tests/ci-workflows/compatibility-version.test.ts -91 tests/ci-workflows/docs-429-failover-claims.test.ts -89 tests/ci-workflows/docs-bun-source-requirement.test.ts -111 tests/ci-workflows/docs-developer-role-policy.test.ts -79 tests/ci-workflows/docs-gui-screenshot-policy.test.ts -221 tests/ci-workflows/docs-link-targets.test.ts -84 tests/ci-workflows/docs-provider-billing-claims.test.ts -121 tests/ci-workflows/docs-provider-discovery-limits.test.ts -98 tests/ci-workflows/docs-provider-preset-counts.test.ts -112 tests/ci-workflows/docs-readme-translation-parity.test.ts -202 tests/ci-workflows/docs-remote-hub-claims.test.ts -181 tests/ci-workflows/dsh-path-contract.test.ts -352 tests/ci-workflows/dsh-rc6-compat-script.test.ts -200 tests/ci-workflows/dsh-writer-lock.test.ts -98 tests/ci-workflows/exhaustive-deps-suppression.test.ts -341 tests/ci-workflows/file-size-ratchet.test.ts -265 tests/ci-workflows/fixture-dir-uniqueness.test.ts -179 tests/ci-workflows/install-scripts.test.ts -116 tests/ci-workflows/installed-gate-drivers.test.ts -96 tests/ci-workflows/keyring-smoke.test.ts -2948 tests/ci-workflows/macos-serial-lanes.test.ts -2704 tests/ci-workflows/package-tree-integrity.test.ts -259 tests/ci-workflows/package-tree-restart-ownership.test.ts -139 tests/ci-workflows/pr-readiness-reattest.test.ts -174 tests/ci-workflows/privacy-scan-asset-names.test.ts -93 tests/ci-workflows/privacy-scan-meta-key.test.ts -94 tests/ci-workflows/privacy-scan-ssh-endpoint.test.ts -127 tests/ci-workflows/release-desktop-scripts.test.ts -12261 tests/ci-workflows/release-helper.test.ts -96 tests/ci-workflows/release-notes.test.ts -91 tests/ci-workflows/release-pipeline-contract.test.ts -179 tests/ci-workflows/release-resume-identity.test.ts -194 tests/ci-workflows/release-version-line.test.ts -221 tests/ci-workflows/release-version-sources.test.ts -276 tests/ci-workflows/repo-hygiene.test.ts -383 tests/ci-workflows/repo-import-resolution.test.ts -383 tests/ci-workflows/setup-hooks.test.ts -103 tests/ci-workflows/skill-ocx.test.ts -468 tests/ci-workflows/structure-ssot.test.ts -40405 tests/ci-workflows/test-home-guard.test.ts -17910 tests/ci-workflows/test-runner.test.ts -93 tests/ci-workflows/test-sandbox-cleanup.test.ts -183 tests/ci-workflows/version-line.test.ts -108 tests/ci-workflows/warmup-registration.test.ts -95 tests/ci-workflows/zz-ci-api-usage-isolation.test.ts -188 tests/ci-workflows/zz-ci-storage-policy-isolation.test.ts -124 tests/ci-workflows/zz-pr-coderabbit-readiness-revalidation.test.ts -2836 tests/claude-integration/claude-529-mapping.test.ts -109 tests/claude-integration/claude-agent-startup-sync.test.ts -124 tests/claude-integration/claude-agents-inject-client.test.ts -139 tests/claude-integration/claude-agents-inject.test.ts -150 tests/claude-integration/claude-alias.test.ts -83 tests/claude-integration/claude-auth-detect.test.ts -115 tests/claude-integration/claude-auth-mode.test.ts -105 tests/claude-integration/claude-authmode-migration.test.ts -142 tests/claude-integration/claude-cli.test.ts -219 tests/claude-integration/claude-code-thought-signature-scope.test.ts -96 tests/claude-integration/claude-compatibility.test.ts -131 tests/claude-integration/claude-context-windows.test.ts -117 tests/claude-integration/claude-desktop-1m.test.ts -570 tests/claude-integration/claude-desktop-cli.test.ts -330 tests/claude-integration/claude-desktop-config-path.test.ts -439 tests/claude-integration/claude-desktop-discovery.test.ts -522 tests/claude-integration/claude-desktop-first-party.test.ts -121 tests/claude-integration/claude-desktop-mode-explanation.test.ts -140 tests/claude-integration/claude-desktop-native-context.test.ts -96 tests/claude-integration/claude-desktop-policy.test.ts -5687 tests/claude-integration/claude-desktop-remote-hub.test.ts -183 tests/claude-integration/claude-dotenv-provenance-transport.test.ts -106 tests/claude-integration/claude-gateway-cache.test.ts -408 tests/claude-integration/claude-inbound-cache-stabilize.test.ts -100 tests/claude-integration/claude-inbound-debug.test.ts -157 tests/claude-integration/claude-inbound.test.ts -217 tests/claude-integration/claude-intercept-local-ca.test.ts -159 tests/claude-integration/claude-intercept-proxy.test.ts -199 tests/claude-integration/claude-intercept-settings.test.ts -1761 tests/claude-integration/claude-management-api.test.ts -4557 tests/claude-integration/claude-messages-endpoint.test.ts -122 tests/claude-integration/claude-model-info.test.ts -864 tests/claude-integration/claude-models-discovery.test.ts -2441 tests/claude-integration/claude-native-affinity.test.ts -3021 tests/claude-integration/claude-native-passthrough.test.ts -1031 tests/claude-integration/claude-outbound.test.ts -126 tests/claude-integration/claude-shell-hook.test.ts -379 tests/claude-integration/claude-sidecar-override.test.ts -108 tests/claude-integration/claude-source-envelope.test.ts -122 tests/claude-integration/claude-system-env-auto.test.ts -107 tests/cli/agent-driven.test.ts -107 tests/cli/cli-account-cancel-flow.test.ts -80 tests/cli/cli-account-orca-import.test.ts -136 tests/cli/cli-account-pin-drain.test.ts -118 tests/cli/cli-account-pool-verbs.test.ts -136 tests/cli/cli-account-threshold.test.ts -829 tests/cli/cli-account.test.ts -134 tests/cli/cli-capabilities.test.ts +6280 tests/ci-workflows/ci-crash-disposition.test.ts +810 tests/ci-workflows/ci-gui-typecheck-gate.test.ts +1421 tests/ci-workflows/ci-privacy-gate.test.ts +9066 tests/ci-workflows/ci-review-lanes.test.ts +1204 tests/ci-workflows/ci-scope-gaps.test.ts +279 tests/ci-workflows/ci-scope-reduction.test.ts +751 tests/ci-workflows/ci-shard-balance.test.ts +115 tests/ci-workflows/ci-structure-gate.test.ts +895 tests/ci-workflows/ci-workflows.test.ts +119 tests/ci-workflows/cleanup-orphaned-workflows.test.ts +78 tests/ci-workflows/closed-pr-branch-cleanup.test.ts +6959 tests/ci-workflows/cold-spawn-warmup.test.ts +271 tests/ci-workflows/compatibility-version.test.ts +73 tests/ci-workflows/docs-429-failover-claims.test.ts +97 tests/ci-workflows/docs-bun-source-requirement.test.ts +127 tests/ci-workflows/docs-developer-role-policy.test.ts +137 tests/ci-workflows/docs-gjc-reasoning-export.test.ts +94 tests/ci-workflows/docs-gui-screenshot-policy.test.ts +511 tests/ci-workflows/docs-link-targets.test.ts +75 tests/ci-workflows/docs-provider-billing-claims.test.ts +116 tests/ci-workflows/docs-provider-discovery-limits.test.ts +83 tests/ci-workflows/docs-provider-preset-counts.test.ts +174 tests/ci-workflows/docs-readme-memory-inventory.test.ts +110 tests/ci-workflows/docs-readme-translation-parity.test.ts +109 tests/ci-workflows/docs-remote-hub-claims.test.ts +133 tests/ci-workflows/dsh-path-contract.test.ts +357 tests/ci-workflows/dsh-rc6-compat-script.test.ts +208 tests/ci-workflows/dsh-writer-lock.test.ts +141 tests/ci-workflows/exhaustive-deps-suppression.test.ts +778 tests/ci-workflows/file-size-ratchet.test.ts +659 tests/ci-workflows/fixture-dir-uniqueness.test.ts +90 tests/ci-workflows/fixture-teardown-helper.test.ts +350 tests/ci-workflows/install-scripts.test.ts +154 tests/ci-workflows/installed-gate-drivers.test.ts +123 tests/ci-workflows/keyring-smoke.test.ts +154 tests/ci-workflows/linux-desktop-packaged-ci.test.ts +146 tests/ci-workflows/linux-desktop-packaged-e2e.test.ts +7684 tests/ci-workflows/macos-serial-lanes.test.ts +2814 tests/ci-workflows/package-tree-fenced-restart.test.ts +2823 tests/ci-workflows/package-tree-integrity.test.ts +190 tests/ci-workflows/package-tree-restart-ownership.test.ts +176 tests/ci-workflows/package-tree-retarget.test.ts +160 tests/ci-workflows/pr-readiness-reattest.test.ts +99 tests/ci-workflows/privacy-scan-asset-names.test.ts +97 tests/ci-workflows/privacy-scan-meta-key.test.ts +93 tests/ci-workflows/privacy-scan-ssh-endpoint.test.ts +126 tests/ci-workflows/release-desktop-scripts.test.ts +13586 tests/ci-workflows/release-helper.test.ts +86 tests/ci-workflows/release-notes.test.ts +183 tests/ci-workflows/release-outcome-report.test.ts +135 tests/ci-workflows/release-pipeline-contract.test.ts +2884 tests/ci-workflows/release-preflight.test.ts +230 tests/ci-workflows/release-resume-identity.test.ts +131 tests/ci-workflows/release-version-line.test.ts +282 tests/ci-workflows/release-version-sources.test.ts +422 tests/ci-workflows/repo-hygiene.test.ts +669 tests/ci-workflows/repo-import-resolution.test.ts +604 tests/ci-workflows/setup-hooks.test.ts +824 tests/ci-workflows/skill-ocx-generated.test.ts +222 tests/ci-workflows/skill-ocx-workflows.test.ts +149 tests/ci-workflows/skill-ocx.test.ts +771 tests/ci-workflows/structure-ssot.test.ts +43547 tests/ci-workflows/test-home-guard.test.ts +298 tests/ci-workflows/test-runner-bun.test.ts +18264 tests/ci-workflows/test-runner.test.ts +141 tests/ci-workflows/test-sandbox-cleanup.test.ts +84 tests/ci-workflows/test-temp-quarantine.test.ts +101 tests/ci-workflows/version-line.test.ts +91 tests/ci-workflows/warmup-registration.test.ts +104 tests/ci-workflows/zz-ci-api-usage-isolation.test.ts +149 tests/ci-workflows/zz-ci-storage-policy-isolation.test.ts +158 tests/ci-workflows/zz-pr-coderabbit-readiness-revalidation.test.ts +2115 tests/claude-integration/anthropic2-native-routing.test.ts +2834 tests/claude-integration/claude-529-mapping.test.ts +142 tests/claude-integration/claude-agent-startup-sync.test.ts +101 tests/claude-integration/claude-agents-inject-client.test.ts +129 tests/claude-integration/claude-agents-inject.test.ts +212 tests/claude-integration/claude-alias.test.ts +107 tests/claude-integration/claude-auth-detect.test.ts +150 tests/claude-integration/claude-auth-mode.test.ts +94 tests/claude-integration/claude-authmode-migration.test.ts +265 tests/claude-integration/claude-cli-picker-surface.test.ts +576 tests/claude-integration/claude-cli-picker.test.ts +137 tests/claude-integration/claude-cli.test.ts +177 tests/claude-integration/claude-code-thought-signature-scope.test.ts +173 tests/claude-integration/claude-compatibility.test.ts +641 tests/claude-integration/claude-config-home-isolation.test.ts +137 tests/claude-integration/claude-context-accounting.test.ts +144 tests/claude-integration/claude-context-windows.test.ts +128 tests/claude-integration/claude-desktop-1m.test.ts +885 tests/claude-integration/claude-desktop-cli.test.ts +573 tests/claude-integration/claude-desktop-config-path.test.ts +568 tests/claude-integration/claude-desktop-discovery.test.ts +434 tests/claude-integration/claude-desktop-first-party-guards.test.ts +991 tests/claude-integration/claude-desktop-first-party.test.ts +135 tests/claude-integration/claude-desktop-mode-explanation.test.ts +124 tests/claude-integration/claude-desktop-native-context.test.ts +172 tests/claude-integration/claude-desktop-picker-profile.test.ts +555 tests/claude-integration/claude-desktop-picker-routes.test.ts +328 tests/claude-integration/claude-desktop-picker.test.ts +83 tests/claude-integration/claude-desktop-policy.test.ts +7157 tests/claude-integration/claude-desktop-remote-hub.test.ts +108 tests/claude-integration/claude-desktop-system-proxy.test.ts +325 tests/claude-integration/claude-devin-output-order.test.ts +220 tests/claude-integration/claude-dotenv-provenance-transport.test.ts +540 tests/claude-integration/claude-estimate-projection.test.ts +385 tests/claude-integration/claude-first-party-agent-sync.test.ts +220 tests/claude-integration/claude-first-party-union.test.ts +159 tests/claude-integration/claude-gateway-cache.test.ts +371 tests/claude-integration/claude-inbound-cache-stabilize.test.ts +106 tests/claude-integration/claude-inbound-debug.test.ts +106 tests/claude-integration/claude-inbound-token-footer.test.ts +229 tests/claude-integration/claude-inbound-tool-reference.test.ts +176 tests/claude-integration/claude-inbound.test.ts +89 tests/claude-integration/claude-intercept-cleanup.test.ts +199 tests/claude-integration/claude-intercept-client-class.test.ts +193 tests/claude-integration/claude-intercept-local-ca-files.test.ts +188 tests/claude-integration/claude-intercept-local-ca-windows.test.ts +254 tests/claude-integration/claude-intercept-local-ca.test.ts +276 tests/claude-integration/claude-intercept-model-bindings.test.ts +302 tests/claude-integration/claude-intercept-no-proxy.test.ts +315 tests/claude-integration/claude-intercept-on-demand.test.ts +186 tests/claude-integration/claude-intercept-proxy-auth.test.ts +350 tests/claude-integration/claude-intercept-proxy.test.ts +115 tests/claude-integration/claude-intercept-settings.test.ts +174 tests/claude-integration/claude-long-context-eligibility.test.ts +5002 tests/claude-integration/claude-management-api.test.ts +6099 tests/claude-integration/claude-messages-endpoint.test.ts +414 tests/claude-integration/claude-messages-thread.test.ts +179 tests/claude-integration/claude-model-info.test.ts +1095 tests/claude-integration/claude-models-discovery.test.ts +2628 tests/claude-integration/claude-native-affinity.test.ts +10397 tests/claude-integration/claude-native-passthrough.test.ts +194 tests/claude-integration/claude-native-rate-limit-headers.test.ts +156 tests/claude-integration/claude-native-reasoning-continuity.test.ts +478 tests/claude-integration/claude-native-reasoning-provenance.test.ts +3884 tests/claude-integration/claude-outbound.test.ts +154 tests/claude-integration/claude-output-schema-strict.test.ts +316 tests/claude-integration/claude-picker-bootstrap.test.ts +633 tests/claude-integration/claude-picker-ca-store.test.ts +733 tests/claude-integration/claude-picker-ca.test.ts +1321 tests/claude-integration/claude-picker-listener.test.ts +115 tests/claude-integration/claude-picker-models.test.ts +1517 tests/claude-integration/claude-picker-recovery.test.ts +809 tests/claude-integration/claude-picker-runtime.test.ts +1924 tests/claude-integration/claude-picker-startup.test.ts +116 tests/claude-integration/claude-picker-trust.test.ts +1263 tests/claude-integration/claude-picker-upload.test.ts +2041 tests/claude-integration/claude-prompt-too-long.test.ts +121 tests/claude-integration/claude-shell-hook.test.ts +131 tests/claude-integration/claude-sidecar-override.test.ts +116 tests/claude-integration/claude-source-envelope.test.ts +125 tests/claude-integration/claude-subagent-model-force.test.ts +147 tests/claude-integration/claude-system-env-auto.test.ts +251 tests/claude-integration/messages-native-decline-trace.test.ts +2085 tests/claude-integration/messages-native-oauth.test.ts +365 tests/claude-integration/messages-native-opaque-state.test.ts +744 tests/claude-integration/messages-native-scope.test.ts +2539 tests/claude-integration/messages-native.test.ts +2167 tests/claude-integration/messages-request-id-endpoint.test.ts +133 tests/claude-integration/messages-request-id-headers.test.ts +1465 tests/claude-integration/messages-revoked-token.test.ts +248 tests/claude-integration/messages-surface-matrix.test.ts +100 tests/cli/agent-driven.test.ts +209 tests/cli/agent-subagent-force.test.ts +134 tests/cli/claude-config-first-party.test.ts +15356 tests/cli/cli-access-audio-live.test.ts +558 tests/cli/cli-access-audio.test.ts +170 tests/cli/cli-access-data-plane.test.ts +139 tests/cli/cli-access-key-guard.test.ts +308 tests/cli/cli-access-rename.test.ts +141 tests/cli/cli-account-alias-target.test.ts +188 tests/cli/cli-account-cancel-flow.test.ts +222 tests/cli/cli-account-key-quota.test.ts +137 tests/cli/cli-account-kiro-device.test.ts +637 tests/cli/cli-account-login-options.test.ts +99 tests/cli/cli-account-orca-import.test.ts +122 tests/cli/cli-account-pin-drain.test.ts +980 tests/cli/cli-account-policy.test.ts +140 tests/cli/cli-account-pool-verbs.test.ts +141 tests/cli/cli-account-threshold.test.ts +122 tests/cli/cli-account-verify-status.test.ts +1267 tests/cli/cli-account.test.ts +192 tests/cli/cli-agent-runtime-settings.test.ts +294 tests/cli/cli-agent-settings.test.ts +173 tests/cli/cli-alias-json.test.ts +115 tests/cli/cli-anthropic-account-threshold.test.ts +194 tests/cli/cli-anthropic2-account.test.ts +214 tests/cli/cli-api-protocols.test.ts +199 tests/cli/cli-aside-sync.test.ts +149 tests/cli/cli-capabilities-arguments.test.ts +194 tests/cli/cli-capabilities.test.ts +116 tests/cli/cli-capability-access-workflows.test.ts +206 tests/cli/cli-capability-account-workflows.test.ts +174 tests/cli/cli-capability-agent-workflows.test.ts +3495 tests/cli/cli-capability-data.test.ts +175 tests/cli/cli-capability-integration-workflows.test.ts +247 tests/cli/cli-capability-lab-workflows.test.ts +189 tests/cli/cli-capability-operation-workflows.test.ts +156 tests/cli/cli-capability-provider-workflows.test.ts +103 tests/cli/cli-catalog-command-result.test.ts 100 tests/cli/cli-catalog-prewarm.test.ts -139 tests/cli/cli-codex-cli-update.test.ts -123 tests/cli/cli-codex-log-guard-compact.test.ts -126 tests/cli/cli-codex-log-guard-protection.test.ts -115 tests/cli/cli-codex-log-guard.test.ts -106 tests/cli/cli-companion.test.ts -4458 tests/cli/cli-config-command.test.ts -2354 tests/cli/cli-config-show-client.test.ts -7261 tests/cli/cli-connect-readiness.test.ts -218 tests/cli/cli-dispatch.test.ts -129 tests/cli/cli-dto-fidelity.test.ts -230 tests/cli/cli-effort.test.ts -642 tests/cli/cli-export-command.test.ts -110 tests/cli/cli-head.test.ts -371 tests/cli/cli-headless-parity.test.ts -12045 tests/cli/cli-help.test.ts -133 tests/cli/cli-json-contract.test.ts -112 tests/cli/cli-management-auth.test.ts -122 tests/cli/cli-models-free-only.test.ts -143 tests/cli/cli-models-price.test.ts -167 tests/cli/cli-models-reasoning.test.ts -115 tests/cli/cli-models-runtime-dispatch.test.ts -11305 tests/cli/cli-models.test.ts -216 tests/cli/cli-native-profile.test.ts -111 tests/cli/cli-observe-logs.test.ts -15141 tests/cli/cli-provider.test.ts -864 tests/cli/cli-ready-subprocess.test.ts -193 tests/cli/cli-ready.test.ts -116 tests/cli/cli-registry.test.ts -2786 tests/cli/cli-resolve-subprocess.test.ts -98 tests/cli/cli-resolve.test.ts -3534 tests/cli/cli-restart-health.test.ts -2957 tests/cli/cli-restore-back.test.ts -4480 tests/cli/cli-start-auxiliary-bind.test.ts -4594 tests/cli/cli-start-journal-order.test.ts -2577 tests/cli/cli-status-hub-state.test.ts -14493 tests/cli/cli-status-json.test.ts -127 tests/cli/cli-status-oauth-health.test.ts -133 tests/cli/cli-stop-json.test.ts -135 tests/cli/cli-storage-inspect.test.ts -140 tests/cli/cli-transport-honesty.test.ts -153 tests/cli/cli-usage-hub.test.ts -146 tests/cli/cli-usage-report.test.ts -100 tests/cli/cli-version-skew.test.ts -131 tests/cli/ensure-desired-integrations-race.test.ts -149 tests/cli/hub-gated-local-clients.test.ts -92 tests/cli/hub-invite.test.ts -97 tests/cli/interactive-confirm.test.ts -111 tests/cli/model-selection-guidance.test.ts -971 tests/cli/ocx-launcher-runtime.test.ts -97 tests/cli/ocx-launcher-source.test.ts -100 tests/cli/ocx-run.test.ts -1150 tests/cli/restore-completes-shared-teardown.test.ts -411 tests/cli/route-explainability.test.ts -102 tests/cli/star-deferral.test.ts -78 tests/cli/start-args.test.ts -77 tests/cli/start-ownership-publication.test.ts -98 tests/cli/system-restart-client.test.ts -153 tests/cli/uninstall.test.ts -148 tests/clients/aside-profile-identity.test.ts -281 tests/clients/aside-profile-paths.test.ts -1825 tests/clients/aside-profile-sync-owner.test.ts -574 tests/clients/aside-profiles.test.ts -76 tests/clients/client-catalog-compatibility.test.ts -19498 tests/clients/client-connect.test.ts -121 tests/clients/client-export-modality-enum.test.ts -92 tests/clients/client-fingerprint.test.ts -94 tests/clients/client-hub-relay.test.ts -111 tests/clients/client-hub-state.test.ts -114 tests/clients/client-hub-usage.test.ts -368 tests/clients/client-lifecycle-lock.test.ts -247 tests/clients/client-machine-listener.test.ts -119 tests/clients/client-runtime.test.ts -250 tests/clients/cline-client.test.ts -200 tests/clients/cline-writer.test.ts -161 tests/clients/desktop-3p-guard.test.ts -167 tests/clients/desktop-3p-removal.test.ts -187 tests/clients/desktop-3p.test.ts -102 tests/clients/desktop-app-restart-posix.test.ts -201 tests/clients/desktop-app-restart.test.ts -74 tests/clients/desktop-cli-contracts.test.ts -86 tests/clients/desktop-exit-ownership.test.ts -85 tests/clients/desktop-install-identity.test.ts -556 tests/clients/desktop-profile.test.ts -81 tests/clients/desktop-proxy-direct-transport.test.ts -341 tests/clients/desktop-remote-store.test.ts -98 tests/clients/desktop-restart-handoff.test.ts -89 tests/clients/desktop-runtime-identity.test.ts -89 tests/clients/desktop-start-at-login-default.test.ts -85 tests/clients/desktop-startup-surface.test.ts +546 tests/cli/cli-claude-desktop-profile.test.ts +143 tests/cli/cli-client-start-fence.test.ts +1788 tests/cli/cli-client-start-owner-e2e.test.ts +514 tests/cli/cli-codex-cli-update.test.ts +141 tests/cli/cli-codex-log-guard-compact.test.ts +140 tests/cli/cli-codex-log-guard-protection.test.ts +112 tests/cli/cli-codex-log-guard.test.ts +212 tests/cli/cli-combo-parity.test.ts +138 tests/cli/cli-combo-partial-update.test.ts +317 tests/cli/cli-combo-stats.test.ts +208 tests/cli/cli-companion-timeline.test.ts +172 tests/cli/cli-companion-usage.test.ts +166 tests/cli/cli-companion.test.ts +1117 tests/cli/cli-config-bom.test.ts +5689 tests/cli/cli-config-command.test.ts +1945 tests/cli/cli-config-default-show.test.ts +2596 tests/cli/cli-config-show-client.test.ts +7251 tests/cli/cli-connect-readiness.test.ts +252 tests/cli/cli-dispatch.test.ts +175 tests/cli/cli-dto-fidelity.test.ts +149 tests/cli/cli-effort-slug.test.ts +482 tests/cli/cli-effort.test.ts +195 tests/cli/cli-ensure-client-state.test.ts +110 tests/cli/cli-ensure-readiness.test.ts +970 tests/cli/cli-export-command.test.ts +102 tests/cli/cli-head.test.ts +770 tests/cli/cli-headless-parity.test.ts +3575 tests/cli/cli-help-navigation.test.ts +17678 tests/cli/cli-help-paths.test.ts +6049 tests/cli/cli-help-recovery.test.ts +13973 tests/cli/cli-help.test.ts +1584 tests/cli/cli-init-port.test.ts +141 tests/cli/cli-injection-follow.test.ts +194 tests/cli/cli-integer-options.test.ts +190 tests/cli/cli-integration-cursor.test.ts +308 tests/cli/cli-integration-droid.test.ts +331 tests/cli/cli-integration-journal.test.ts +457 tests/cli/cli-integration-preview.test.ts +147 tests/cli/cli-json-contract.test.ts +444 tests/cli/cli-json-input.test.ts +124 tests/cli/cli-kiro-auto-selection.test.ts +229 tests/cli/cli-link-enrollment.test.ts +309 tests/cli/cli-link-force.test.ts +152 tests/cli/cli-link.test.ts +503 tests/cli/cli-log-follow.test.ts +706 tests/cli/cli-log-view-filter.test.ts +195 tests/cli/cli-logout-runtime.test.ts +107 tests/cli/cli-logs-request-id.test.ts +114 tests/cli/cli-management-auth.test.ts +399 tests/cli/cli-models-custom-runtime.test.ts +192 tests/cli/cli-models-display-name.test.ts +181 tests/cli/cli-models-free-only.test.ts +203 tests/cli/cli-models-local-result.test.ts +245 tests/cli/cli-models-order.test.ts +264 tests/cli/cli-models-price.test.ts +180 tests/cli/cli-models-reasoning.test.ts +194 tests/cli/cli-models-runtime-dispatch.test.ts +165 tests/cli/cli-models-set.test.ts +14554 tests/cli/cli-models.test.ts +169 tests/cli/cli-native-profile.test.ts +152 tests/cli/cli-observe-logs.test.ts +209 tests/cli/cli-observe-snapshot.test.ts +141 tests/cli/cli-path-diagnostics.test.ts +303 tests/cli/cli-provider-batch.test.ts +229 tests/cli/cli-provider-lifecycle-runtime.test.ts +221 tests/cli/cli-provider-result.test.ts +205 tests/cli/cli-provider-settings.test.ts +348 tests/cli/cli-provider-sync-result.test.ts +24697 tests/cli/cli-provider.test.ts +2075 tests/cli/cli-ready-subprocess.test.ts +114 tests/cli/cli-ready.test.ts +133 tests/cli/cli-registry.test.ts +318 tests/cli/cli-remote-workspace-hub.test.ts +3405 tests/cli/cli-resolve-subprocess.test.ts +111 tests/cli/cli-resolve.test.ts +132 tests/cli/cli-restart-handoff.test.ts +4087 tests/cli/cli-restart-health.test.ts +3385 tests/cli/cli-restore-back.test.ts +144 tests/cli/cli-route-policy-not-found.test.ts +1439 tests/cli/cli-route-policy-write.test.ts +6917 tests/cli/cli-start-auxiliary-bind.test.ts +14624 tests/cli/cli-start-journal-order.test.ts +94 tests/cli/cli-start-routing-heal-wiring.test.ts +137 tests/cli/cli-status-codex-routing-drift.test.ts +3045 tests/cli/cli-status-hub-state.test.ts +17271 tests/cli/cli-status-json.test.ts +166 tests/cli/cli-status-oauth-health.test.ts +1945 tests/cli/cli-status-startup-health.test.ts +181 tests/cli/cli-stop-json.test.ts +133 tests/cli/cli-storage-inspect.test.ts +303 tests/cli/cli-storage-policy-fields.test.ts +301 tests/cli/cli-system-health.test.ts +159 tests/cli/cli-system-settings-parity.test.ts +157 tests/cli/cli-transport-honesty.test.ts +2081 tests/cli/cli-update-badge.test.ts +653 tests/cli/cli-update-restart-admission.test.ts +127 tests/cli/cli-update-restart-child.test.ts +83 tests/cli/cli-update-restart-desktop-service.test.ts +244 tests/cli/cli-update-restart-home.test.ts +128 tests/cli/cli-update-restart-service-record.test.ts +93 tests/cli/cli-update-restart-supervision.test.ts +1158 tests/cli/cli-update-restart-transport.test.ts +742 tests/cli/cli-update-restart.test.ts +172 tests/cli/cli-usage-hub.test.ts +183 tests/cli/cli-usage-model-search.test.ts +216 tests/cli/cli-usage-report.test.ts +325 tests/cli/cli-usage-scope.test.ts +843 tests/cli/cli-v2-runtime.test.ts +22021 tests/cli/cli-version-skew.test.ts +146 tests/cli/ensure-desired-integrations-race.test.ts +1659 tests/cli/hub-disable.test.ts +322 tests/cli/hub-gated-local-clients.test.ts +151 tests/cli/hub-invite.test.ts +123 tests/cli/interactive-confirm.test.ts +112 tests/cli/model-selection-guidance.test.ts +2581 tests/cli/ocx-launcher-desktop-handoff.test.ts +5174 tests/cli/ocx-launcher-runtime.test.ts +109 tests/cli/ocx-launcher-source.test.ts +176 tests/cli/ocx-run.test.ts +1510 tests/cli/restore-completes-shared-teardown.test.ts +449 tests/cli/route-explainability.test.ts +16556 tests/cli/sibling-home-client-sync.test.ts +111 tests/cli/star-deferral.test.ts +86 tests/cli/start-args.test.ts +113 tests/cli/start-ownership-publication.test.ts +177 tests/cli/system-restart-client-package-tree.test.ts +118 tests/cli/system-restart-client.test.ts +353 tests/cli/uninstall.test.ts +127 tests/clients/aside-profile-identity.test.ts +214 tests/clients/aside-profile-paths.test.ts +2245 tests/clients/aside-profile-sync-owner.test.ts +179 tests/clients/aside-profiles-catalog-refresh.test.ts +602 tests/clients/aside-profiles.test.ts +83 tests/clients/client-catalog-compatibility.test.ts +21146 tests/clients/client-connect.test.ts +241 tests/clients/client-export-effective-metadata.test.ts +597 tests/clients/client-export-live-wire.test.ts +140 tests/clients/client-export-modality-enum.test.ts +119 tests/clients/client-export-reasoning-controls.test.ts +114 tests/clients/client-export-reasoning-metadata.test.ts +85 tests/clients/client-fingerprint.test.ts +126 tests/clients/client-hub-relay.test.ts +48 tests/clients/client-hub-state.test.ts +212 tests/clients/client-hub-usage.test.ts +408 tests/clients/client-lifecycle-lock.test.ts +435 tests/clients/client-link-connect.test.ts +312 tests/clients/client-link-relay.test.ts +5936 tests/clients/client-link-runtime.test.ts +233 tests/clients/client-link-state.test.ts +91 tests/clients/client-link-status.test.ts +304 tests/clients/client-link-teardown.test.ts +4537 tests/clients/client-link-tunnel.test.ts +5331 tests/clients/client-machine-listener.test.ts +130 tests/clients/client-runtime.test.ts +124 tests/clients/cline-client.test.ts +229 tests/clients/cline-writer.test.ts +191 tests/clients/command-code-client-contract.test.ts +156 tests/clients/command-code-client.test.ts +121 tests/clients/desktop-3p-guard.test.ts +211 tests/clients/desktop-3p-removal.test.ts +340 tests/clients/desktop-3p.test.ts +140 tests/clients/desktop-app-restart-posix.test.ts +140 tests/clients/desktop-app-restart.test.ts +1984 tests/clients/desktop-app-server-shim-launcher.test.ts +112 tests/clients/desktop-app-server-shim.test.ts +129 tests/clients/desktop-chatgpt-config.test.ts +99 tests/clients/desktop-cli-command-surface.test.ts +154 tests/clients/desktop-cli-contracts.test.ts +106 tests/clients/desktop-exit-ownership.test.ts +211 tests/clients/desktop-host-visibility.test.ts +103 tests/clients/desktop-install-identity.test.ts +647 tests/clients/desktop-profile.test.ts +105 tests/clients/desktop-proxy-direct-transport.test.ts +468 tests/clients/desktop-remote-store.test.ts +84 tests/clients/desktop-restart-handoff.test.ts +73 tests/clients/desktop-runtime-identity.test.ts +92 tests/clients/desktop-start-at-login-default.test.ts +101 tests/clients/desktop-startup-surface.test.ts +684 tests/clients/desktop-supervised-restart.test.ts 96 tests/clients/desktop-tray-availability.test.ts -99 tests/clients/desktop-widget-entry.test.ts -297 tests/clients/integrations-current-store.test.ts -163 tests/clients/integrations-journal.test.ts -139 tests/clients/integrations-merge.test.ts -195 tests/clients/integrations-serialize.test.ts -146 tests/clients/integrations-state.test.ts -184 tests/clients/integrations-superseded-store.test.ts -154 tests/clients/integrations-writer-frozen-config.test.ts -286 tests/clients/integrations-writer.test.ts -143 tests/clients/mutation-plan.test.ts -123 tests/clients/omo-client.test.ts -261 tests/clients/omp-path-contract.test.ts -91 tests/clients/omp-yaml-source-inline-comments.test.ts -115 tests/clients/pi-path-contract.test.ts -115 tests/clients/prime-client.test.ts -163 tests/clients/raycast-client.test.ts -87 tests/clients/raycast-detect.test.ts -6922 tests/clients/remote-catalog.test.ts -99 tests/clients/remote-control-prototype.test.ts -171 tests/clients/remote-workspace-activation.test.ts -115 tests/clients/remote-workspace-agent-wire.test.ts -192 tests/clients/remote-workspace-app-server.integration.test.ts -187 tests/clients/remote-workspace-claude.integration.test.ts -124 tests/clients/remote-workspace-cli-runtimes.test.ts -145 tests/clients/remote-workspace-cli.test.ts -2469 tests/clients/remote-workspace-codex-runtime.test.ts -128 tests/clients/remote-workspace-command-runner.test.ts -98 tests/clients/remote-workspace-device.test.ts -100 tests/clients/remote-workspace-hub.test.ts -85 tests/clients/remote-workspace-linux-confinement.test.ts -172 tests/clients/remote-workspace-management.test.ts -212 tests/clients/remote-workspace-platform.test.ts -78 tests/clients/remote-workspace-protocol.test.ts -87 tests/clients/remote-workspace-rpc-framing.test.ts -79 tests/clients/remote-workspace-secret-store.test.ts -709 tests/clients/remote-workspace-server.test.ts -104 tests/clients/remote-workspace-session-binding.test.ts -129 tests/clients/remote-workspace-sessions.test.ts -198 tests/clients/remote-workspace-tool-bridge.test.ts -163 tests/clients/remote-workspace.test.ts -401 tests/clients/sync-client-integrations.test.ts -550 tests/codex-integration/active-registry-admission.test.ts -137 tests/codex-integration/app-owned-memory.test.ts -3457 tests/codex-integration/bearer-admission-routed-provider.test.ts -147 tests/codex-integration/catalog-auto-refresh-scheduler.test.ts -115 tests/codex-integration/catalog-cursor-search.test.ts -147 tests/codex-integration/catalog-duplicate-slug-dedup.test.ts -142 tests/codex-integration/catalog-free-pricing-status.test.ts -4072 tests/codex-integration/catalog-full-picker-order.test.ts -139 tests/codex-integration/catalog-gated-native-suppression-reason.test.ts -131 tests/codex-integration/catalog-go-exact-efforts.test.ts -116 tests/codex-integration/catalog-hub-context-window.test.ts -181 tests/codex-integration/catalog-input-modality-enum.test.ts -126 tests/codex-integration/catalog-llamacpp-capabilities.test.ts -775 tests/codex-integration/catalog-modelalias-unique-sync.test.ts -153 tests/codex-integration/catalog-oauth-observation.test.ts -136 tests/codex-integration/catalog-opencode-go-context-window.test.ts -918 tests/codex-integration/catalog-remote-pull.test.ts -117 tests/codex-integration/catalog-retain-models.test.ts -110 tests/codex-integration/catalog-seed-window-fill.test.ts -187 tests/codex-integration/catalog-slug-uniqueness-boundary.test.ts -126 tests/codex-integration/catalog-verbosity-default.test.ts -143 tests/codex-integration/catalog-vision-sidecar-modalities.test.ts -115 tests/codex-integration/catalog-zero-credit-picker.test.ts -4512 tests/codex-integration/client-injection-guard.test.ts -172 tests/codex-integration/codex-account-delete-atomicity.test.ts -89 tests/codex-integration/codex-account-label.test.ts -83 tests/codex-integration/codex-account-namespaces.test.ts -173 tests/codex-integration/codex-account-selection-preferences.test.ts -151 tests/codex-integration/codex-account-store-refresh-classification.test.ts -886 tests/codex-integration/codex-account-store.test.ts -473 tests/codex-integration/codex-account-threshold-api.test.ts -255 tests/codex-integration/codex-account-threshold-auth.test.ts -304 tests/codex-integration/codex-account-threshold-routing.test.ts -153 tests/codex-integration/codex-account-unusable-reason.test.ts -133 tests/codex-integration/codex-admission-primitives.test.ts -150 tests/codex-integration/codex-admission.test.ts -105 tests/codex-integration/codex-affinity-debug.test.ts -79 tests/codex-integration/codex-app-server-path-spaces.test.ts -846 tests/codex-integration/codex-app-server-processes.test.ts -99 tests/codex-integration/codex-app-server-restart-service.test.ts -1925 tests/codex-integration/codex-auth-api.test.ts -110 tests/codex-integration/codex-auth-cancel-ownership.test.ts -138 tests/codex-integration/codex-auth-collision.test.ts -674 tests/codex-integration/codex-auth-context.test.ts -114 tests/codex-integration/codex-catalog-admission.test.ts -108 tests/codex-integration/codex-catalog-exclusions.test.ts -108 tests/codex-integration/codex-catalog-golden.test.ts -132 tests/codex-integration/codex-catalog-ladders.test.ts -123 tests/codex-integration/codex-catalog-model-picker-order.test.ts -83 tests/codex-integration/codex-catalog-refresh-status.test.ts -4572 tests/codex-integration/codex-catalog-restore.test.ts -18585 tests/codex-integration/codex-catalog-sync-hardening.test.ts -107 tests/codex-integration/codex-catalog-write-serialization.test.ts -103 tests/codex-integration/codex-catalog-writer.test.ts -1857 tests/codex-integration/codex-catalog.test.ts -125 tests/codex-integration/codex-cli-install-provenance.test.ts -104 tests/codex-integration/codex-cli-installation-identity.test.ts -79 tests/codex-integration/codex-cli-installation-targets.test.ts -89 tests/codex-integration/codex-cli-update-launcher-policy.test.ts -965 tests/codex-integration/codex-cli-update-zero-effect.test.ts -86 tests/codex-integration/codex-cli-windows-installation-files.test.ts -15251 tests/codex-integration/codex-composed-acceptance.test.ts -475 tests/codex-integration/codex-config-generation.test.ts -266 tests/codex-integration/codex-context-owner.test.ts -3803 tests/codex-integration/codex-convergence-account-selectors.test.ts -538 tests/codex-integration/codex-convergence-contract.test.ts -265 tests/codex-integration/codex-cooldown-recovery.test.ts -170 tests/codex-integration/codex-coordinator-doctor.test.ts -153 tests/codex-integration/codex-desired-state.test.ts -97 tests/codex-integration/codex-entitlement-identity-read-fence.test.ts -382 tests/codex-integration/codex-envkey-admission-substitution.test.ts -119 tests/codex-integration/codex-exec-invocation.test.ts -97 tests/codex-integration/codex-features-cache.test.ts -88 tests/codex-integration/codex-features-residual.test.ts -200 tests/codex-integration/codex-filesystem-evidence.test.ts -399 tests/codex-integration/codex-gather-authority.test.ts -2798 tests/codex-integration/codex-history-job.test.ts -151 tests/codex-integration/codex-history-lock.test.ts -1661 tests/codex-integration/codex-history-provider.test.ts -303 tests/codex-integration/codex-history-reachability.test.ts -93 tests/codex-integration/codex-history-worker-boundary.test.ts -1227 tests/codex-integration/codex-history-worker.test.ts -111 tests/codex-integration/codex-history-writer.test.ts -121 tests/codex-integration/codex-home-wsl.test.ts -257 tests/codex-integration/codex-inject-history-wording.test.ts -70464 tests/codex-integration/codex-inject-integration.test.ts -1122 tests/codex-integration/codex-inject-retained-table.test.ts -6724 tests/codex-integration/codex-inject-v1-reconcile.test.ts -9100 tests/codex-integration/codex-inject-write-lock.test.ts -138 tests/codex-integration/codex-inject.test.ts -91 tests/codex-integration/codex-injected-marker.test.ts -121 tests/codex-integration/codex-integration-record.test.ts -17011 tests/codex-integration/codex-journal.test.ts -100 tests/codex-integration/codex-legacy-config-keys.test.ts -213 tests/codex-integration/codex-lineage-placement.test.ts -143 tests/codex-integration/codex-log-guard-coderabbit.test.ts +98 tests/clients/desktop-update-surface.test.ts +96 tests/clients/desktop-widget-entry.test.ts +183 tests/clients/droid-client.test.ts +181 tests/clients/droid-managed-reasoning-defaults.test.ts +161 tests/clients/droid-normalized-reasoning-defaults.test.ts +309 tests/clients/integrations-current-store.test.ts +467 tests/clients/integrations-dsh-profile-patch.test.ts +372 tests/clients/integrations-hermes-affinity.test.ts +209 tests/clients/integrations-journal.test.ts +153 tests/clients/integrations-merge.test.ts +98 tests/clients/integrations-serialize.test.ts +143 tests/clients/integrations-state.test.ts +168 tests/clients/integrations-superseded-store.test.ts +222 tests/clients/integrations-writer-frozen-config.test.ts +160 tests/clients/integrations-writer-guard.test.ts +322 tests/clients/integrations-writer.test.ts +186 tests/clients/kilo-client.test.ts +147 tests/clients/lazycodex-detection.test.ts +82 tests/clients/link-admission-wait.test.ts +115 tests/clients/link-boundary.test.ts +91 tests/clients/link-compensation.test.ts +135 tests/clients/link-fingerprint.test.ts +144 tests/clients/link-ports.test.ts +295 tests/clients/link-relay-bound-transport.test.ts +133 tests/clients/link-routes.test.ts +767 tests/clients/link-ssh-argv-discovery.test.ts +742 tests/clients/link-ssh-argv.test.ts +191 tests/clients/link-ssh-config.test.ts +579 tests/clients/link-ssh-runner.test.ts +103 tests/clients/link-status-projection.test.ts +78 tests/clients/link-store.test.ts +1740 tests/clients/link-supervisor.test.ts +134 tests/clients/link-tunnel-state.test.ts +187 tests/clients/mutation-plan.test.ts +118 tests/clients/omo-client.test.ts +328 tests/clients/omo-role-models.test.ts +92 tests/clients/omp-path-contract.test.ts +136 tests/clients/omp-yaml-source-inline-comments.test.ts +230 tests/clients/opencode-kilo-refresh.test.ts +203 tests/clients/pi-path-contract.test.ts +132 tests/clients/prime-client.test.ts +236 tests/clients/raycast-client.test.ts +73 tests/clients/raycast-detect.test.ts +6959 tests/clients/remote-catalog.test.ts +104 tests/clients/remote-control-prototype.test.ts +246 tests/clients/remote-workspace-activation.test.ts +158 tests/clients/remote-workspace-agent-wire.test.ts +92 tests/clients/remote-workspace-app-server.integration.test.ts +99 tests/clients/remote-workspace-claude.integration.test.ts +132 tests/clients/remote-workspace-cli-runtimes.test.ts +201 tests/clients/remote-workspace-cli.test.ts +1424 tests/clients/remote-workspace-codex-runtime.test.ts +145 tests/clients/remote-workspace-command-runner.test.ts +188 tests/clients/remote-workspace-device.test.ts +103 tests/clients/remote-workspace-hub.test.ts +63 tests/clients/remote-workspace-linux-confinement.test.ts +177 tests/clients/remote-workspace-management.test.ts +98 tests/clients/remote-workspace-platform.test.ts +98 tests/clients/remote-workspace-protocol.test.ts +180 tests/clients/remote-workspace-rpc-framing.test.ts +244 tests/clients/remote-workspace-secret-store.test.ts +745 tests/clients/remote-workspace-server.test.ts +91 tests/clients/remote-workspace-session-binding.test.ts +146 tests/clients/remote-workspace-sessions.test.ts +137 tests/clients/remote-workspace-tool-bridge.test.ts +412 tests/clients/remote-workspace.test.ts +589 tests/clients/sync-client-integrations-unattended.test.ts +460 tests/clients/sync-client-integrations.test.ts +645 tests/codex-integration/active-registry-admission.test.ts +1359 tests/codex-integration/active-turn-disconnect.test.ts +175 tests/codex-integration/anthropic2-catalog.test.ts +113 tests/codex-integration/app-owned-memory.test.ts +4456 tests/codex-integration/bearer-admission-routed-provider.test.ts +4172 tests/codex-integration/catalog-async-lifetime.test.ts +332 tests/codex-integration/catalog-auto-refresh-client-fanout.test.ts +1677 tests/codex-integration/catalog-auto-refresh-scheduler.test.ts +4210 tests/codex-integration/catalog-auto-review-row.test.ts +101 tests/codex-integration/catalog-cursor-search.test.ts +128 tests/codex-integration/catalog-custom-ordering.test.ts +123 tests/codex-integration/catalog-duplicate-slug-dedup.test.ts +191 tests/codex-integration/catalog-free-pricing-status.test.ts +6152 tests/codex-integration/catalog-full-picker-order.test.ts +105 tests/codex-integration/catalog-gated-native-suppression-reason.test.ts +152 tests/codex-integration/catalog-go-exact-efforts.test.ts +207 tests/codex-integration/catalog-hub-context-window.test.ts +233 tests/codex-integration/catalog-input-modality-enum.test.ts +285 tests/codex-integration/catalog-llamacpp-capabilities.test.ts +959 tests/codex-integration/catalog-modelalias-unique-sync.test.ts +239 tests/codex-integration/catalog-oauth-observation.test.ts +129 tests/codex-integration/catalog-opencode-go-context-window.test.ts +100 tests/codex-integration/catalog-opengateway-hints.test.ts +1341 tests/codex-integration/catalog-remote-pull.test.ts +139 tests/codex-integration/catalog-retain-models.test.ts +192 tests/codex-integration/catalog-routed-comp-hash.test.ts +105 tests/codex-integration/catalog-seed-window-fill.test.ts +61 tests/codex-integration/catalog-slug-uniqueness-boundary.test.ts +107 tests/codex-integration/catalog-verbosity-default.test.ts +121 tests/codex-integration/catalog-vision-sidecar-modalities.test.ts +186 tests/codex-integration/catalog-zero-credit-picker.test.ts +6556 tests/codex-integration/client-injection-guard.test.ts +291 tests/codex-integration/code-mode-collab-surface.test.ts +139 tests/codex-integration/codex-account-clear-paused.test.ts +216 tests/codex-integration/codex-account-delete-atomicity.test.ts +78 tests/codex-integration/codex-account-label.test.ts +106 tests/codex-integration/codex-account-namespaces.test.ts +401 tests/codex-integration/codex-account-selection-atomicity.test.ts +296 tests/codex-integration/codex-account-selection-preferences.test.ts +235 tests/codex-integration/codex-account-store-refresh-classification.test.ts +1015 tests/codex-integration/codex-account-store.test.ts +431 tests/codex-integration/codex-account-threshold-api.test.ts +335 tests/codex-integration/codex-account-threshold-auth.test.ts +191 tests/codex-integration/codex-account-threshold-routing.test.ts +141 tests/codex-integration/codex-account-unusable-reason.test.ts +371 tests/codex-integration/codex-admission-primitives.test.ts +151 tests/codex-integration/codex-admission.test.ts +139 tests/codex-integration/codex-affinity-debug.test.ts +159 tests/codex-integration/codex-app-server-path-spaces.test.ts +1163 tests/codex-integration/codex-app-server-processes.test.ts +179 tests/codex-integration/codex-app-server-restart-service.test.ts +1906 tests/codex-integration/codex-auth-api.test.ts +132 tests/codex-integration/codex-auth-cancel-ownership.test.ts +150 tests/codex-integration/codex-auth-collision.test.ts +1074 tests/codex-integration/codex-auth-context.test.ts +194 tests/codex-integration/codex-auth-release.test.ts +164 tests/codex-integration/codex-catalog-admission.test.ts +118 tests/codex-integration/codex-catalog-exclusions.test.ts +139 tests/codex-integration/codex-catalog-golden.test.ts +458 tests/codex-integration/codex-catalog-heal-integration.test.ts +162 tests/codex-integration/codex-catalog-identity-neutral.test.ts +197 tests/codex-integration/codex-catalog-intent.test.ts +148 tests/codex-integration/codex-catalog-ladders.test.ts +132 tests/codex-integration/codex-catalog-model-picker-order.test.ts +140 tests/codex-integration/codex-catalog-refresh-status.test.ts +1959 tests/codex-integration/codex-catalog-restore.test.ts +168 tests/codex-integration/codex-catalog-routed-removal.test.ts +193 tests/codex-integration/codex-catalog-self-heal.test.ts +35417 tests/codex-integration/codex-catalog-sync-hardening.test.ts +842 tests/codex-integration/codex-catalog-write-audit.test.ts +148 tests/codex-integration/codex-catalog-write-serialization.test.ts +526 tests/codex-integration/codex-catalog-writer.test.ts +2403 tests/codex-integration/codex-catalog.test.ts +169 tests/codex-integration/codex-cli-install-provenance.test.ts +100 tests/codex-integration/codex-cli-installation-identity.test.ts +127 tests/codex-integration/codex-cli-installation-targets.test.ts +104 tests/codex-integration/codex-cli-update-launcher-policy.test.ts +1197 tests/codex-integration/codex-cli-update-zero-effect.test.ts +92 tests/codex-integration/codex-cli-windows-installation-files.test.ts +17660 tests/codex-integration/codex-composed-acceptance.test.ts +3114 tests/codex-integration/codex-config-drift-heal.test.ts +607 tests/codex-integration/codex-config-generation.test.ts +4282 tests/codex-integration/codex-config-preservation.test.ts +8640 tests/codex-integration/codex-config-write-lock.test.ts +291 tests/codex-integration/codex-context-owner.test.ts +6039 tests/codex-integration/codex-convergence-account-selectors.test.ts +989 tests/codex-integration/codex-convergence-contract.test.ts +311 tests/codex-integration/codex-cooldown-recovery.test.ts +211 tests/codex-integration/codex-coordinator-doctor.test.ts +260 tests/codex-integration/codex-credits-after-limit-main.test.ts +240 tests/codex-integration/codex-credits-after-limit.test.ts +180 tests/codex-integration/codex-credits-probes.test.ts +271 tests/codex-integration/codex-credits-settings.test.ts +100 tests/codex-integration/codex-credits.test.ts +309 tests/codex-integration/codex-desired-state.test.ts +181 tests/codex-integration/codex-entitlement-identity-read-fence.test.ts +640 tests/codex-integration/codex-envkey-admission-substitution.test.ts +104 tests/codex-integration/codex-exec-invocation.test.ts +136 tests/codex-integration/codex-failure-window.test.ts +107 tests/codex-integration/codex-features-cache.test.ts +118 tests/codex-integration/codex-features-residual.test.ts +172 tests/codex-integration/codex-filesystem-evidence.test.ts +140 tests/codex-integration/codex-forward-access-programs-writer.test.ts +404 tests/codex-integration/codex-gather-authority.test.ts +2970 tests/codex-integration/codex-history-job.test.ts +177 tests/codex-integration/codex-history-lock.test.ts +2579 tests/codex-integration/codex-history-provider.test.ts +633 tests/codex-integration/codex-history-reachability.test.ts +89 tests/codex-integration/codex-history-worker-boundary.test.ts +2139 tests/codex-integration/codex-history-worker.test.ts +128 tests/codex-integration/codex-history-writer.test.ts +14059 tests/codex-integration/codex-home-owner-lifecycle.test.ts +2478 tests/codex-integration/codex-home-owner.test.ts +146 tests/codex-integration/codex-home-wsl-local-state.test.ts +158 tests/codex-integration/codex-home-wsl.test.ts +343 tests/codex-integration/codex-idle-window.test.ts +109 tests/codex-integration/codex-inject-history-wording.test.ts +88831 tests/codex-integration/codex-inject-integration.test.ts +2787 tests/codex-integration/codex-inject-missing-config.test.ts +1627 tests/codex-integration/codex-inject-retained-table.test.ts +9251 tests/codex-integration/codex-inject-v1-reconcile.test.ts +11411 tests/codex-integration/codex-inject-write-lock.test.ts +205 tests/codex-integration/codex-inject.test.ts +87 tests/codex-integration/codex-injected-marker.test.ts +143 tests/codex-integration/codex-integration-record.test.ts +5561 tests/codex-integration/codex-journal-recovery.test.ts +19811 tests/codex-integration/codex-journal.test.ts +137 tests/codex-integration/codex-legacy-config-keys.test.ts +207 tests/codex-integration/codex-lineage-placement.test.ts +248 tests/codex-integration/codex-linked-account-pause.test.ts +139 tests/codex-integration/codex-log-guard-coderabbit.test.ts 98 tests/codex-integration/codex-log-guard-doctor-coderabbit.test.ts -183 tests/codex-integration/codex-log-guard-doctor-protection.test.ts -94 tests/codex-integration/codex-log-guard-doctor.test.ts -225 tests/codex-integration/codex-log-guard-inspect.test.ts -78 tests/codex-integration/codex-log-guard-lock.test.ts -735 tests/codex-integration/codex-log-guard-maintenance-coderabbit.test.ts -1146 tests/codex-integration/codex-log-guard-maintenance.test.ts -76 tests/codex-integration/codex-log-guard-policy.test.ts -78 tests/codex-integration/codex-log-guard-processes.test.ts -212 tests/codex-integration/codex-log-guard-protection.test.ts -208 tests/codex-integration/codex-log-guard-status-zero-write.test.ts -189 tests/codex-integration/codex-main-account-refresh.test.ts -227 tests/codex-integration/codex-main-rotation.test.ts -146 tests/codex-integration/codex-management-convergence.test.ts -176 tests/codex-integration/codex-metadata-integrity.test.ts -173 tests/codex-integration/codex-model-availability-error.test.ts -201 tests/codex-integration/codex-model-denial-evidence.test.ts -92 tests/codex-integration/codex-model-entitlement-admission.test.ts -214 tests/codex-integration/codex-model-entitlements.test.ts -194 tests/codex-integration/codex-models-cache-invalidate.test.ts -345 tests/codex-integration/codex-native-residue.test.ts -137 tests/codex-integration/codex-pin-drain-projection.test.ts -145 tests/codex-integration/codex-plan.test.ts -568 tests/codex-integration/codex-plugins-doctor.test.ts -178 tests/codex-integration/codex-pool-plan-exclusion.test.ts -82 tests/codex-integration/codex-pool-refresh-backoff.test.ts -536 tests/codex-integration/codex-pool-rotation.test.ts -262 tests/codex-integration/codex-priority-failback.test.ts -119 tests/codex-integration/codex-prompt-adopt.test.ts -126 tests/codex-integration/codex-prompt-base-variants.test.ts -91 tests/codex-integration/codex-prompt-journal.test.ts -101 tests/codex-integration/codex-prompt-layers-read.test.ts -264 tests/codex-integration/codex-prompt-layers-write.test.ts -86 tests/codex-integration/codex-prompt-layers.test.ts -98 tests/codex-integration/codex-prompt-lock.test.ts -1370 tests/codex-integration/codex-prompt-route.test.ts -856 tests/codex-integration/codex-prompt-text-probe.test.ts -97 tests/codex-integration/codex-provider-table-retention.test.ts -200 tests/codex-integration/codex-quota-auto-refresh-main-admission.test.ts -316 tests/codex-integration/codex-quota-auto-refresh.test.ts -95 tests/codex-integration/codex-quota-capacity.test.ts -194 tests/codex-integration/codex-quota-history.test.ts -131 tests/codex-integration/codex-quota-parser-parity.test.ts -361 tests/codex-integration/codex-quota-prime.test.ts -181 tests/codex-integration/codex-quota-rejection.test.ts -162 tests/codex-integration/codex-refresh.test.ts -1519 tests/codex-integration/codex-reset-credit-auto-redeem.test.ts -634 tests/codex-integration/codex-reset-credit-operation-ledger.test.ts -307 tests/codex-integration/codex-reset-credit-recovery.test.ts -91 tests/codex-integration/codex-restart-contract-parity.test.ts -271 tests/codex-integration/codex-restart-route.test.ts -5829 tests/codex-integration/codex-restore-app-rewrite.test.ts -7237 tests/codex-integration/codex-retained-root-serialization.test.ts -147 tests/codex-integration/codex-routing-cache-affinity-detour.test.ts -2801 tests/codex-integration/codex-routing.test.ts -141 tests/codex-integration/codex-runtime.test.ts -507 tests/codex-integration/codex-service-manager-probe-hardening.test.ts -156 tests/codex-integration/codex-service-manager-probe.test.ts -16277 tests/codex-integration/codex-shim-autorestore.test.ts -405 tests/codex-integration/codex-shim-destroyed-probe.test.ts -93 tests/codex-integration/codex-shim-ensure-failure.test.ts -10961 tests/codex-integration/codex-shim-readiness.test.ts -25412 tests/codex-integration/codex-shim.test.ts -134 tests/codex-integration/codex-signin-lockout.test.ts -604 tests/codex-integration/codex-spark-visibility.test.ts -180 tests/codex-integration/codex-sqlite-home.test.ts -2023 tests/codex-integration/codex-sync-api.test.ts -108 tests/codex-integration/codex-sync-response.test.ts -122 tests/codex-integration/codex-tool-mode.test.ts -701 tests/codex-integration/codex-transition-state-adoption.test.ts -97 tests/codex-integration/codex-transition-state-first-use-regression.test.ts -1878 tests/codex-integration/codex-transition-state-race.test.ts -287 tests/codex-integration/codex-transition-state.test.ts -121 tests/codex-integration/codex-user-identity.test.ts -528 tests/codex-integration/codex-v2-gate.test.ts -505 tests/codex-integration/codex-warmup.test.ts -89 tests/codex-integration/codex-websocket-registry.test.ts -3148 tests/codex-integration/codex-write-lock.test.ts -192 tests/codex-integration/combo-authoritative-reset.test.ts -325 tests/codex-integration/combos.test.ts -649 tests/codex-integration/compatibility-manifest.test.ts -91 tests/codex-integration/context-compat.test.ts -84 tests/codex-integration/custom-model-catalog-migration.test.ts -1093 tests/codex-integration/doctor.test.ts -237 tests/codex-integration/effort-policy.test.ts -146 tests/codex-integration/fast-row-listing.test.ts -131 tests/codex-integration/fast-row.test.ts -176 tests/codex-integration/gather-routed-models-single-flight.test.ts -129 tests/codex-integration/gpt6-native-rows.test.ts -87 tests/codex-integration/history-migration-guardian.test.ts -104 tests/codex-integration/history-ocx-compaction-recovery.test.ts -87 tests/codex-integration/history-paginated-openai-compat.test.ts -2318 tests/codex-integration/history-paginated-transition-destinations.test.ts -215 tests/codex-integration/injection-model-api.test.ts -5248 tests/codex-integration/issue-452-empty-503.test.ts -2788 tests/codex-integration/issue-702-expired-replay-state.test.ts -623 tests/codex-integration/issue-914-transport-attribution.test.ts -17598 tests/codex-integration/main-account-hard-lock-auth.test.ts -88 tests/codex-integration/main-account-hard-lock-policy.test.ts -153 tests/codex-integration/main-account-hard-lock-recovery.test.ts -129 tests/codex-integration/main-device-reauth-api.test.ts -188 tests/codex-integration/main-device-reauth.test.ts -160 tests/codex-integration/main-quota-evidence-validation.test.ts -878 tests/codex-integration/main-quota-provenance.test.ts -269 tests/codex-integration/main-quota-window-observation.test.ts -94 tests/codex-integration/model-cache-generation-tombstone.test.ts -89 tests/codex-integration/model-cache.test.ts -156 tests/codex-integration/model-display-names-management-api.test.ts -243 tests/codex-integration/model-metadata-sync.test.ts -839 tests/codex-integration/model-pinned-effort.test.ts -564 tests/codex-integration/model-visibility-management-api.test.ts -634 tests/codex-integration/multi-agent-compat.test.ts -325 tests/codex-integration/multi-agent-keep-native-v1.test.ts -139 tests/codex-integration/native-alias-maintainer-regressions.test.ts -254 tests/codex-integration/native-claude-code-toggle.test.ts -353 tests/codex-integration/native-claude-desktop-toggle.test.ts -2112 tests/codex-integration/native-codex-toggle.test.ts -408 tests/codex-integration/native-grok-toggle.test.ts -195 tests/codex-integration/native-main-auth-temp.test.ts -90 tests/codex-integration/native-main-claim-cache.test.ts -161 tests/codex-integration/native-main-claim.test.ts -7905 tests/codex-integration/native-main-owner-lifetime.test.ts -344 tests/codex-integration/native-model-toggle.test.ts -135 tests/codex-integration/native-profile-api.test.ts -8866 tests/codex-integration/native-profile-crash-boundaries.test.ts -2226 tests/codex-integration/native-profile-drain-server.test.ts -3526 tests/codex-integration/native-profile-manager.test.ts -487 tests/codex-integration/native-profile-processes.test.ts +113 tests/codex-integration/codex-log-guard-doctor-protection.test.ts +62 tests/codex-integration/codex-log-guard-doctor.test.ts +296 tests/codex-integration/codex-log-guard-inspect.test.ts +113 tests/codex-integration/codex-log-guard-lock.test.ts +460 tests/codex-integration/codex-log-guard-maintenance-coderabbit.test.ts +702 tests/codex-integration/codex-log-guard-maintenance.test.ts +99 tests/codex-integration/codex-log-guard-policy.test.ts +74 tests/codex-integration/codex-log-guard-processes.test.ts +203 tests/codex-integration/codex-log-guard-protection.test.ts +98 tests/codex-integration/codex-log-guard-status-zero-write.test.ts +216 tests/codex-integration/codex-main-account-refresh.test.ts +394 tests/codex-integration/codex-main-grant-refusal.test.ts +391 tests/codex-integration/codex-main-rotation.test.ts +149 tests/codex-integration/codex-main-token-invalidated.test.ts +127 tests/codex-integration/codex-management-convergence.test.ts +155 tests/codex-integration/codex-metadata-integrity.test.ts +12449 tests/codex-integration/codex-missing-home.test.ts +119 tests/codex-integration/codex-model-availability-error.test.ts +271 tests/codex-integration/codex-model-denial-evidence.test.ts +102 tests/codex-integration/codex-model-entitlement-admission.test.ts +112 tests/codex-integration/codex-model-entitlements-program-shape.test.ts +347 tests/codex-integration/codex-model-entitlements.test.ts +209 tests/codex-integration/codex-models-cache-invalidate.test.ts +985 tests/codex-integration/codex-native-availability-nux.test.ts +1438 tests/codex-integration/codex-native-residue.test.ts +182 tests/codex-integration/codex-pin-drain-projection.test.ts +134 tests/codex-integration/codex-plan.test.ts +705 tests/codex-integration/codex-plugins-doctor.test.ts +340 tests/codex-integration/codex-pool-credit-policy.test.ts +201 tests/codex-integration/codex-pool-plan-exclusion.test.ts +97 tests/codex-integration/codex-pool-refresh-backoff.test.ts +212 tests/codex-integration/codex-pool-request-owned-main.test.ts +710 tests/codex-integration/codex-pool-rotation.test.ts +339 tests/codex-integration/codex-priority-failback.test.ts +183 tests/codex-integration/codex-prompt-adopt.test.ts +464 tests/codex-integration/codex-prompt-base-variants.test.ts +144 tests/codex-integration/codex-prompt-journal.test.ts +108 tests/codex-integration/codex-prompt-layers-read.test.ts +262 tests/codex-integration/codex-prompt-layers-write.test.ts +80 tests/codex-integration/codex-prompt-layers.test.ts +344 tests/codex-integration/codex-prompt-lock-sharing.test.ts +635 tests/codex-integration/codex-prompt-lock.test.ts +1328 tests/codex-integration/codex-prompt-route.test.ts +319 tests/codex-integration/codex-prompt-scalar-scope.test.ts +946 tests/codex-integration/codex-prompt-text-probe.test.ts +930 tests/codex-integration/codex-prompt-writes.test.ts +1047 tests/codex-integration/codex-provider-table-retention.test.ts +130 tests/codex-integration/codex-quota-auto-refresh-generation.test.ts +202 tests/codex-integration/codex-quota-auto-refresh-main-admission.test.ts +358 tests/codex-integration/codex-quota-auto-refresh.test.ts +82 tests/codex-integration/codex-quota-capacity.test.ts +296 tests/codex-integration/codex-quota-history.test.ts +108 tests/codex-integration/codex-quota-parser-parity.test.ts +371 tests/codex-integration/codex-quota-prime.test.ts +177 tests/codex-integration/codex-quota-query-backoff.test.ts +242 tests/codex-integration/codex-quota-rejection.test.ts +154 tests/codex-integration/codex-refresh.test.ts +994 tests/codex-integration/codex-reset-credit-auto-redeem.test.ts +597 tests/codex-integration/codex-reset-credit-operation-ledger.test.ts +535 tests/codex-integration/codex-reset-credit-recovery.test.ts +129 tests/codex-integration/codex-restart-contract-parity.test.ts +128 tests/codex-integration/codex-restart-route.test.ts +7005 tests/codex-integration/codex-restore-app-rewrite.test.ts +9747 tests/codex-integration/codex-retained-root-serialization.test.ts +154 tests/codex-integration/codex-routing-cache-affinity-detour.test.ts +3534 tests/codex-integration/codex-routing.test.ts +174 tests/codex-integration/codex-runtime-wsl-desktop.test.ts +243 tests/codex-integration/codex-runtime.test.ts +542 tests/codex-integration/codex-service-manager-probe-hardening.test.ts +249 tests/codex-integration/codex-service-manager-probe.test.ts +20788 tests/codex-integration/codex-shim-autorestore.test.ts +631 tests/codex-integration/codex-shim-destroyed-probe.test.ts +94 tests/codex-integration/codex-shim-ensure-failure.test.ts +139 tests/codex-integration/codex-shim-fnm.test.ts +744 tests/codex-integration/codex-shim-migration.test.ts +120 tests/codex-integration/codex-shim-overlay-candidates.test.ts +802 tests/codex-integration/codex-shim-overlay-readiness.test.ts +2882 tests/codex-integration/codex-shim-overlay-transaction.test.ts +655 tests/codex-integration/codex-shim-overlay.test.ts +3802 tests/codex-integration/codex-shim-path-readiness.test.ts +28325 tests/codex-integration/codex-shim-readiness.test.ts +110 tests/codex-integration/codex-shim-runtime-preflight.test.ts +1043 tests/codex-integration/codex-shim-standalone.test.ts +26281 tests/codex-integration/codex-shim.test.ts +1882 tests/codex-integration/codex-sidecar-credit-policy.test.ts +193 tests/codex-integration/codex-signin-lockout.test.ts +224 tests/codex-integration/codex-spark-visibility.test.ts +183 tests/codex-integration/codex-sqlite-home.test.ts +1782 tests/codex-integration/codex-sync-api.test.ts +492 tests/codex-integration/codex-sync-new-model-policy.test.ts +162 tests/codex-integration/codex-sync-response.test.ts +216 tests/codex-integration/codex-systemd-env-probe.test.ts +185 tests/codex-integration/codex-tool-mode.test.ts +982 tests/codex-integration/codex-transition-state-adoption.test.ts +164 tests/codex-integration/codex-transition-state-first-use-regression.test.ts +2110 tests/codex-integration/codex-transition-state-race.test.ts +290 tests/codex-integration/codex-transition-state.test.ts +154 tests/codex-integration/codex-user-identity.test.ts +1314 tests/codex-integration/codex-v2-gate.test.ts +568 tests/codex-integration/codex-warmup.test.ts +4026 tests/codex-integration/codex-web-search-switch.test.ts +72 tests/codex-integration/codex-websocket-registry.test.ts +4006 tests/codex-integration/codex-write-lock.test.ts +83 tests/codex-integration/combo-authoritative-reset.test.ts +164 tests/codex-integration/combo-codex-exhaustion-cooldown.test.ts +85 tests/codex-integration/combo-last-resort.test.ts +81 tests/codex-integration/combo-permanent-failure-cooldown.test.ts +160 tests/codex-integration/combo-reset-window-model-scope.test.ts +356 tests/codex-integration/combos.test.ts +981 tests/codex-integration/compatibility-manifest.test.ts +232 tests/codex-integration/configured-native-models.test.ts +139 tests/codex-integration/context-compat.test.ts +122 tests/codex-integration/custom-model-catalog-migration.test.ts +169 tests/codex-integration/discovered-native-models.test.ts +553 tests/codex-integration/doctor.test.ts +604 tests/codex-integration/effort-policy.test.ts +86 tests/codex-integration/fast-row-listing.test.ts +369 tests/codex-integration/fast-row.test.ts +209 tests/codex-integration/gather-routed-models-single-flight.test.ts +132 tests/codex-integration/gpt6-native-rows.test.ts +145 tests/codex-integration/gpt61-sol-rows.test.ts +95 tests/codex-integration/history-migration-guardian.test.ts +217 tests/codex-integration/history-ocx-compaction-recovery.test.ts +110 tests/codex-integration/history-paginated-openai-compat.test.ts +3591 tests/codex-integration/history-paginated-transition-destinations.test.ts +156 tests/codex-integration/injection-link-websocket.test.ts +294 tests/codex-integration/injection-model-api.test.ts +176 tests/codex-integration/injection-model-suggest-routes.test.ts +82 tests/codex-integration/injection-routing-drift.test.ts +5187 tests/codex-integration/injection-routing-heal-apply.test.ts +653 tests/codex-integration/injection-routing-healer.test.ts +5852 tests/codex-integration/issue-452-empty-503.test.ts +3756 tests/codex-integration/issue-702-expired-replay-state.test.ts +668 tests/codex-integration/issue-914-transport-attribution.test.ts +1248 tests/codex-integration/low-quota-protection.test.ts +573 tests/codex-integration/main-account-credit-renewal.test.ts +100 tests/codex-integration/main-account-external-usage.test.ts +17569 tests/codex-integration/main-account-hard-lock-auth.test.ts +347 tests/codex-integration/main-account-hard-lock-default.test.ts +103 tests/codex-integration/main-account-hard-lock-policy.test.ts +487 tests/codex-integration/main-account-hard-lock-recovery.test.ts +134 tests/codex-integration/main-account-hard-lock-retirement.test.ts +105 tests/codex-integration/main-account-hard-lock-thresholds.test.ts +354 tests/codex-integration/main-account-policy-binding-wait.test.ts +156 tests/codex-integration/main-device-reauth-api.test.ts +299 tests/codex-integration/main-device-reauth.test.ts +251 tests/codex-integration/main-quota-evidence-validation.test.ts +1007 tests/codex-integration/main-quota-provenance.test.ts +1492 tests/codex-integration/main-quota-window-observation.test.ts +3303 tests/codex-integration/messaging-local-cli.test.ts +173 tests/codex-integration/messaging-local-discovery.test.ts +214 tests/codex-integration/messaging-local-envelope.test.ts +767 tests/codex-integration/messaging-local-lifecycle.test.ts +569 tests/codex-integration/messaging-local-send.test.ts +154 tests/codex-integration/messaging-local-socket.test.ts +92 tests/codex-integration/model-cache-generation-tombstone.test.ts +200 tests/codex-integration/model-cache.test.ts +210 tests/codex-integration/model-display-names-management-api.test.ts +223 tests/codex-integration/model-metadata-sync.test.ts +830 tests/codex-integration/model-pinned-effort.test.ts +881 tests/codex-integration/model-visibility-management-api.test.ts +1626 tests/codex-integration/multi-agent-compat.test.ts +324 tests/codex-integration/multi-agent-keep-native-v1.test.ts +78 tests/codex-integration/multi-agent-origin.test.ts +148 tests/codex-integration/native-alias-maintainer-regressions.test.ts +149 tests/codex-integration/native-claude-code-toggle.test.ts +438 tests/codex-integration/native-claude-desktop-toggle.test.ts +4695 tests/codex-integration/native-codex-toggle.test.ts +565 tests/codex-integration/native-grok-toggle.test.ts +116 tests/codex-integration/native-main-auth-temp.test.ts +157 tests/codex-integration/native-main-claim-cache.test.ts +232 tests/codex-integration/native-main-claim.test.ts +9377 tests/codex-integration/native-main-owner-lifetime.test.ts +545 tests/codex-integration/native-model-toggle.test.ts +111 tests/codex-integration/native-profile-api.test.ts +10737 tests/codex-integration/native-profile-crash-boundaries.test.ts +2355 tests/codex-integration/native-profile-drain-server.test.ts +3903 tests/codex-integration/native-profile-manager.test.ts +408 tests/codex-integration/native-profile-processes.test.ts 106 tests/codex-integration/native-profile-recovery.test.ts -342 tests/codex-integration/native-profile-route-security.test.ts -303 tests/codex-integration/native-profile-stage-lifecycle.test.ts -14260 tests/codex-integration/native-profile-startup.test.ts -564 tests/codex-integration/native-profile-store.test.ts -328 tests/codex-integration/orca-import.test.ts -148 tests/codex-integration/parallel-tool-calls-optin.test.ts -382 tests/codex-integration/project-config-warnings.test.ts -244 tests/codex-integration/reasoning-effort.test.ts -127 tests/codex-integration/reasoning-metadata.test.ts -185 tests/codex-integration/reserve-auth-context.test.ts -122 tests/codex-integration/reserve-availability.test.ts -5880 tests/codex-integration/reserve-catalog-lifecycle.test.ts -153 tests/codex-integration/reserve-catalog.test.ts -1751 tests/codex-integration/reserve-dispatch.test.ts -171 tests/codex-integration/reserve-helper-boundary.test.ts -119 tests/codex-integration/reserve-passive-revocation.test.ts -143 tests/codex-integration/reserve-quota-scope.test.ts -129 tests/codex-integration/selected-models.test.ts -146 tests/codex-integration/slug-codec.test.ts -192 tests/codex-integration/token-guardian.test.ts -109 tests/codex-integration/ultrafast-tier-honesty.test.ts -128 tests/codex-integration/upstream-reachability.test.ts -91 tests/codex-integration/warmup.test.ts -182 tests/config/client-config-export-new-clients.test.ts -166 tests/config/client-config-export.test.ts -130 tests/config/client-config-new-clients.test.ts -95 tests/config/config-account-thresholds.test.ts -92 tests/config/config-catalog-auto-refresh.test.ts -86 tests/config/config-commandcode-claude-pin.test.ts -140 tests/config/config-load-degrade.test.ts -1085 tests/config/config-mutation-lock.test.ts -98 tests/config/config-non-object-backup.test.ts -100 tests/config/config-ownership-uninstall.test.ts -91 tests/config/config-rebase-provenance-writers.test.ts -93 tests/config/config-save-boundary.test.ts -95 tests/config/config-spend-ceilings.test.ts -513 tests/config/config-user-edits.test.ts -82 tests/config/expand-user-path.test.ts -249 tests/config/model-pinned-effort-config.test.ts -141 tests/config/settings-desktop-switch-apply.test.ts -186 tests/config/settings-fast-rows.test.ts -259 tests/config/settings-main-account-hard-lock.test.ts -173 tests/config/settings-oauth-open-browser.test.ts -172 tests/config/settings-startup-health-seam.test.ts -2628 tests/config/settings-stream-mode.test.ts -78 tests/config/types-barrel-identity.test.ts -94 tests/config/url-normalization.test.ts -99 tests/config/yaml-fragment-source.test.ts -133 tests/e2e-style/phase100-native-parity.test.ts -104 tests/gui/alibaba-intl-token-plan.test.ts -106 tests/gui/claude-manual-env.test.ts -92 tests/gui/codex-account-mode-state.test.ts -90 tests/gui/codex-auth-modal-status.test.ts -105 tests/gui/combo-workspace-data.test.ts -98 tests/gui/dashboard-uptime.test.ts -188 tests/gui/gui-api-error.test.ts -229 tests/gui/gui-codex-usage-score-parity.test.ts -188 tests/gui/gui-desktop-sidecar-script.test.ts -262 tests/gui/gui-management-session.test.ts -89 tests/gui/gui-pair-capability.test.ts -91 tests/gui/gui-pair-client.test.ts -105 tests/gui/gui-static.test.ts -87 tests/gui/gui-tray-vibrancy-surface.test.ts -375 tests/gui/integrations-invariants.test.ts -89 tests/gui/logs-model-tier-confirmation.test.ts -108 tests/gui/main-device-reauth-ui.test.ts -86 tests/gui/models-feedback-callback.test.ts -94 tests/gui/models-free-filter.test.ts -84 tests/gui/models-page-groups.test.ts -193 tests/gui/models-workspace-tabs.test.ts -86 tests/gui/oauth-first-add-hint.test.ts -96 tests/gui/oauth-tos-warning.test.ts -402 tests/gui/platform-dialog-guard.test.ts -130 tests/gui/provider-payload.test.ts -106 tests/gui/provider-workspace-auth.test.ts -174 tests/gui/provider-workspace-data.test.ts -164 tests/gui/provider-workspace-rail.test.ts -94 tests/gui/provider-workspace-state.test.ts -148 tests/gui/quota-bars-rows.test.ts -96 tests/gui/qwen-cloud-endpoints.test.ts -102 tests/gui/rate-limit-reset-credits.test.ts -98 tests/gui/routing-intelligence-ui.test.ts -89 tests/gui/routing-profile-editor-data.test.ts -99 tests/gui/standalone-build-script.test.ts -97 tests/gui/startup-health-ui.test.ts -200 tests/gui/tencent-siliconflow-providers.test.ts -263 tests/gui/vision-sidecar-timeout-bounds.test.ts -261 tests/gui/volcengine-providers.test.ts -228 tests/images/artifacts-prune.test.ts -109 tests/images/artifacts-ssrf.test.ts -90 tests/images/download-cap-default.test.ts -123 tests/images/gemini-inline.test.ts -156 tests/images/loop-reasoning-replay.test.ts -3410 tests/images/loop.test.ts -108 tests/images/pinned-https-get.test.ts -123 tests/images/plan.test.ts -88 tests/images/synthetic-tool.test.ts -148 tests/images/xai-client.test.ts -87 tests/images/z-fulfill.test.ts -235 tests/images/z-handler-activation.test.ts -504 tests/lab/core-lab-boundary.test.ts -332 tests/lab/lab-activation.test.ts -372 tests/lab/lab-automation-coderabbit-regressions.test.ts -294 tests/lab/lab-automation-final-coderabbit-regressions.test.ts -264 tests/lab/lab-automation-ingwannu-regressions.test.ts -228 tests/lab/lab-automation-management-http.test.ts -78 tests/lab/lab-automation-persisted-cap-regression.test.ts -673 tests/lab/lab-automation-review-regressions.test.ts -495 tests/lab/lab-automation.test.ts -271 tests/lab/lab-community-evidence.test.ts -214 tests/lab/lab-community-filename-contract.test.ts -103 tests/lab/lab-community-mutation-lock.test.ts -190 tests/lab/lab-community-publisher-continuity.test.ts -267 tests/lab/lab-conformance-harness.test.ts -174 tests/lab/lab-conformance-runner-failures.test.ts -476 tests/lab/lab-evidence-ledger.test.ts -362 tests/lab/lab-evidence-sanitization.test.ts -152 tests/lab/lab-fabric-outcome-validation.test.ts -93 tests/lab/lab-fabric-persistence-boundary.test.ts -173 tests/lab/lab-fabric-producer-deadline.test.ts -68756 tests/lab/lab-fabric-task.test.ts -110 tests/lab/lab-installation-salt-cache.test.ts -762 tests/lab/lab-ledger-mutation-lock.test.ts -360 tests/lab/lab-live-pinned-timeouts.test.ts +532 tests/codex-integration/native-profile-route-security.test.ts +325 tests/codex-integration/native-profile-stage-lifecycle.test.ts +364 tests/codex-integration/native-profile-startup-release.test.ts +19453 tests/codex-integration/native-profile-startup.test.ts +837 tests/codex-integration/native-profile-store.test.ts +331 tests/codex-integration/orca-import.test.ts +114 tests/codex-integration/parallel-tool-calls-optin.test.ts +128 tests/codex-integration/project-config-warning-snapshot.test.ts +514 tests/codex-integration/project-config-warnings.test.ts +200 tests/codex-integration/reasoning-effort.test.ts +125 tests/codex-integration/reasoning-metadata.test.ts +483 tests/codex-integration/reserve-auth-context.test.ts +297 tests/codex-integration/reserve-availability.test.ts +6428 tests/codex-integration/reserve-catalog-lifecycle.test.ts +148 tests/codex-integration/reserve-catalog.test.ts +1759 tests/codex-integration/reserve-dispatch.test.ts +182 tests/codex-integration/reserve-helper-boundary.test.ts +129 tests/codex-integration/reserve-passive-revocation.test.ts +292 tests/codex-integration/reserve-quota-scope.test.ts +102 tests/codex-integration/selected-models.test.ts +166 tests/codex-integration/slug-codec.test.ts +108 tests/codex-integration/token-guardian.test.ts +1667 tests/codex-integration/ultrafast-tier-honesty.test.ts +161 tests/codex-integration/upstream-reachability.test.ts +167 tests/codex-integration/warmup.test.ts +299 tests/config/client-config-export-new-clients.test.ts +120 tests/config/client-config-export-output-limit.test.ts +260 tests/config/client-config-export.test.ts +248 tests/config/client-config-new-clients.test.ts +118 tests/config/compaction-recovery-settings.test.ts +173 tests/config/config-account-thresholds.test.ts +524 tests/config/config-anthropic-instance-pool.test.ts +131 tests/config/config-blocked-model-redirects.test.ts +133 tests/config/config-catalog-auto-refresh.test.ts +130 tests/config/config-commandcode-claude-pin.test.ts +197 tests/config/config-load-degrade.test.ts +1667 tests/config/config-mutation-lock.test.ts +182 tests/config/config-non-object-backup.test.ts +161 tests/config/config-ownership-uninstall.test.ts +162 tests/config/config-rebase-provenance-writers.test.ts +101 tests/config/config-save-boundary.test.ts +122 tests/config/config-skills-catalog-refresh.test.ts +179 tests/config/config-spend-ceilings.test.ts +100 tests/config/config-spend-pool-collision.test.ts +536 tests/config/config-user-edits.test.ts +110 tests/config/expand-user-path.test.ts +472 tests/config/hub-disable-live-save.test.ts +337 tests/config/model-pinned-effort-config.test.ts +99 tests/config/owner-registry-acl.test.ts +135 tests/config/protocol-settings.test.ts +1410 tests/config/serving-runtimes.test.ts +4102 tests/config/settings-desktop-switch-apply.test.ts +299 tests/config/settings-fast-rows.test.ts +248 tests/config/settings-main-account-hard-lock.test.ts +190 tests/config/settings-memory-models.test.ts +162 tests/config/settings-oauth-open-browser.test.ts +167 tests/config/settings-startup-health-seam.test.ts +14209 tests/config/settings-stream-mode.test.ts +99 tests/config/types-barrel-identity.test.ts +160 tests/config/url-normalization.test.ts +101 tests/config/yaml-fragment-source.test.ts +224 tests/e2e-style/phase100-native-parity.test.ts +102 tests/gui/alibaba-intl-token-plan.test.ts +91 tests/gui/claude-manual-env.test.ts +79 tests/gui/codex-account-mode-state.test.ts +99 tests/gui/codex-auth-modal-status.test.ts +83 tests/gui/codex-set-prompt.test.ts +104 tests/gui/combo-workspace-data.test.ts +105 tests/gui/combo-workspace-jev-decision.test.ts +158 tests/gui/dashboard-sidecar-codex-write.test.ts +162 tests/gui/dashboard-uptime.test.ts +93 tests/gui/gui-api-error.test.ts +127 tests/gui/gui-codex-usage-score-parity.test.ts +97 tests/gui/gui-desktop-sidecar-script.test.ts +82 tests/gui/gui-desktop-sidecar-signing.test.ts +145 tests/gui/gui-management-session.test.ts +101 tests/gui/gui-pair-capability.test.ts +143 tests/gui/gui-pair-client.test.ts +333 tests/gui/gui-pair-delivery.test.ts +1998 tests/gui/gui-pair-http.test.ts +138 tests/gui/gui-static.test.ts +68 tests/gui/gui-tray-vibrancy-surface.test.ts +110 tests/gui/html-lang-boot-script.test.ts +309 tests/gui/integrations-invariants.test.ts +124 tests/gui/logs-model-tier-confirmation.test.ts +122 tests/gui/main-device-reauth-ui.test.ts +97 tests/gui/models-feedback-callback.test.ts +108 tests/gui/models-free-filter.test.ts +79 tests/gui/models-page-groups.test.ts +84 tests/gui/models-workspace-tabs.test.ts +104 tests/gui/oauth-first-add-hint.test.ts +104 tests/gui/oauth-tos-warning.test.ts +482 tests/gui/platform-dialog-guard.test.ts +120 tests/gui/provider-payload.test.ts +143 tests/gui/provider-subscription-cli.test.ts +95 tests/gui/provider-workspace-auth.test.ts +108 tests/gui/provider-workspace-data.test.ts +191 tests/gui/provider-workspace-rail.test.ts +79 tests/gui/provider-workspace-state.test.ts +152 tests/gui/quota-bars-rows.test.ts +93 tests/gui/qwen-cloud-endpoints.test.ts +93 tests/gui/rate-limit-reset-credits.test.ts +102 tests/gui/routing-intelligence-ui.test.ts +77 tests/gui/routing-profile-editor-data.test.ts +69 tests/gui/standalone-build-script.test.ts +157 tests/gui/startup-health-ui.test.ts +171 tests/gui/tencent-siliconflow-providers.test.ts +103 tests/gui/vision-sidecar-timeout-bounds.test.ts +205 tests/gui/volcengine-providers.test.ts +211 tests/images/artifacts-prune.test.ts +132 tests/images/artifacts-ssrf.test.ts +91 tests/images/download-cap-default.test.ts +110 tests/images/download-connect-deadline-default.test.ts +105 tests/images/gemini-inline.test.ts +133 tests/images/loop-reasoning-replay.test.ts +3539 tests/images/loop.test.ts +94 tests/images/pinned-https-get.test.ts +138 tests/images/plan.test.ts +84 tests/images/synthetic-tool.test.ts +180 tests/images/xai-client.test.ts +184 tests/images/z-fulfill.test.ts +261 tests/images/z-handler-activation.test.ts +621 tests/lab/core-lab-boundary.test.ts +379 tests/lab/core-link-boundary.test.ts +248 tests/lab/lab-activation.test.ts +202 tests/lab/lab-automation-coderabbit-regressions.test.ts +251 tests/lab/lab-automation-final-coderabbit-regressions.test.ts +214 tests/lab/lab-automation-ingwannu-regressions.test.ts +201 tests/lab/lab-automation-management-http.test.ts +106 tests/lab/lab-automation-persisted-cap-regression.test.ts +645 tests/lab/lab-automation-review-regressions.test.ts +624 tests/lab/lab-automation.test.ts +323 tests/lab/lab-community-evidence.test.ts +217 tests/lab/lab-community-filename-contract.test.ts +117 tests/lab/lab-community-mutation-lock.test.ts +163 tests/lab/lab-community-publisher-continuity.test.ts +182 tests/lab/lab-conformance-harness.test.ts +134 tests/lab/lab-conformance-runner-failures.test.ts +563 tests/lab/lab-evidence-ledger.test.ts +486 tests/lab/lab-evidence-sanitization.test.ts +235 tests/lab/lab-fabric-outcome-validation.test.ts +141 tests/lab/lab-fabric-persistence-boundary.test.ts +189 tests/lab/lab-fabric-producer-deadline.test.ts +96027 tests/lab/lab-fabric-task.test.ts +187 tests/lab/lab-installation-salt-cache.test.ts +839 tests/lab/lab-ledger-mutation-lock.test.ts +352 tests/lab/lab-live-pinned-timeouts.test.ts 214 tests/lab/lab-live-probe.test.ts -228 tests/lab/lab-live-receipt-integrity.test.ts -149 tests/lab/lab-live-review-regressions.test.ts -523 tests/lab/lab-live-sandbox.test.ts -172 tests/lab/lab-passive-production-evidence.test.ts -159 tests/lab/lab-passive-production-surfaces.test.ts -89 tests/lab/lab-paths-security.test.ts -160 tests/lab/lab-post-merge-hardening.test.ts -138 tests/lab/lab-post-merge-projection.test.ts -108 tests/lab/lab-private-file-consumer-recovery.test.ts -100 tests/lab/lab-private-file-durability.test.ts -210 tests/lab/lab-public-api-json.test.ts -175 tests/lab/lab-public-artifact-policy.test.ts -253 tests/lab/lab-public-coderabbit-regressions.test.ts -96 tests/lab/lab-public-core-contract.test.ts -170 tests/lab/lab-public-deep-review-regressions.test.ts -161 tests/lab/lab-public-evidence.test.ts -178 tests/lab/lab-public-export-transaction.test.ts -103 tests/lab/lab-public-file-safety.test.ts -230 tests/lab/lab-public-final-review-regressions.test.ts -222 tests/lab/lab-public-lifecycle-hardening.test.ts -83 tests/lab/lab-public-privacy-ipv6.test.ts -419 tests/lab/lab-public-provenance-recovery.test.ts -303 tests/lab/lab-public-review-fixes.test.ts -77 tests/lab/lab-public-route-registry.test.ts -108 tests/lab/lab-public-security-regressions.test.ts -510 tests/lab/lab-public-surfaces.test.ts -119 tests/lab/lab-public-wire-contract.test.ts -212 tests/lab/lab-read-filter-validation.test.ts -618 tests/lab/lab-read-surfaces.test.ts -758 tests/lib/abort-idle-deadline.test.ts -88 tests/lib/acl-error-classification.test.ts -313 tests/lib/ambiguous-resend-composition.test.ts -186 tests/lib/ambiguous-resend-gate.test.ts -188 tests/lib/bun-stream-caps.test.ts -120 tests/lib/clearable-deadline.test.ts -317 tests/lib/credential-redirect-guard.test.ts -100 tests/lib/debug.test.ts -95 tests/lib/execution-budget-permits.test.ts -119 tests/lib/failure-attribution.test.ts -84 tests/lib/failure-stage-model.test.ts -191 tests/lib/local-destinations.test.ts -298 tests/lib/optional-shutdown-hooks.test.ts -265 tests/lib/pinned-http-content-coding.test.ts -361 tests/lib/pinned-http.test.ts -95 tests/lib/privacy-mask-account.test.ts -198 tests/lib/process-control-graceful.test.ts -93 tests/lib/process-control.test.ts -184 tests/lib/provider-egress.test.ts -88 tests/lib/reasoning-replay-scope-source.test.ts -142 tests/lib/redact.test.ts -88 tests/lib/remove-tree-helper.test.ts -92 tests/lib/self-launch-argv.test.ts -1512 tests/lib/socks5-fetch.test.ts +128 tests/lab/lab-live-receipt-integrity.test.ts +255 tests/lab/lab-live-review-regressions.test.ts +525 tests/lab/lab-live-sandbox.test.ts +234 tests/lab/lab-passive-production-evidence.test.ts +210 tests/lab/lab-passive-production-surfaces.test.ts +90 tests/lab/lab-paths-security.test.ts +234 tests/lab/lab-post-merge-hardening.test.ts +140 tests/lab/lab-post-merge-projection.test.ts +95 tests/lab/lab-private-file-consumer-recovery.test.ts +103 tests/lab/lab-private-file-durability.test.ts +260 tests/lab/lab-public-api-json.test.ts +125 tests/lab/lab-public-artifact-policy.test.ts +293 tests/lab/lab-public-coderabbit-regressions.test.ts +98 tests/lab/lab-public-core-contract.test.ts +185 tests/lab/lab-public-deep-review-regressions.test.ts +141 tests/lab/lab-public-evidence.test.ts +362 tests/lab/lab-public-export-transaction.test.ts +78 tests/lab/lab-public-file-safety.test.ts +342 tests/lab/lab-public-final-review-regressions.test.ts +308 tests/lab/lab-public-lifecycle-hardening.test.ts +122 tests/lab/lab-public-privacy-ipv6.test.ts +677 tests/lab/lab-public-provenance-recovery.test.ts +531 tests/lab/lab-public-review-fixes.test.ts +127 tests/lab/lab-public-route-registry.test.ts +101 tests/lab/lab-public-security-regressions.test.ts +327 tests/lab/lab-public-surfaces.test.ts +98 tests/lab/lab-public-wire-contract.test.ts +202 tests/lab/lab-read-filter-validation.test.ts +572 tests/lab/lab-read-surfaces.test.ts +750 tests/lib/abort-idle-deadline.test.ts +95 tests/lib/acl-error-classification.test.ts +378 tests/lib/ambiguous-resend-composition.test.ts +79 tests/lib/ambiguous-resend-gate.test.ts +174 tests/lib/bun-runtime-preflight.test.ts +85 tests/lib/bun-stream-caps.test.ts +115 tests/lib/clearable-deadline.test.ts +304 tests/lib/credential-redirect-guard.test.ts +115 tests/lib/debug.test.ts +147 tests/lib/execution-budget-permits.test.ts +113 tests/lib/failure-attribution.test.ts +92 tests/lib/failure-stage-model.test.ts +152 tests/lib/local-chat-completion.test.ts +75 tests/lib/local-destinations.test.ts +85 tests/lib/local-upstream.test.ts +257 tests/lib/optional-shutdown-hooks.test.ts +202 tests/lib/pinned-http-content-coding.test.ts +348 tests/lib/pinned-http.test.ts +451 tests/lib/plugin-loader.test.ts +93 tests/lib/plugin-upstream-hooks.test.ts +64 tests/lib/privacy-mask-account.test.ts +102 tests/lib/process-control-graceful.test.ts +99 tests/lib/process-control.test.ts +94 tests/lib/provider-egress.test.ts +89 tests/lib/reasoning-replay-scope-source.test.ts +1128 tests/lib/redact.test.ts +78 tests/lib/remove-tree-helper.test.ts +111 tests/lib/self-launch-argv.test.ts +1497 tests/lib/socks5-fetch.test.ts 134 tests/lib/socks5-upload-lifecycle.test.ts -186 tests/lib/spend-ceiling-enforcement.test.ts -101 tests/lib/spend-ledger-file-journal.test.ts -718 tests/lib/spend-ledger-owner.test.ts -107 tests/lib/spend-reservation-ledger.test.ts -100 tests/lib/stall-subprocess-exit.test.ts -82 tests/lib/stall-timeout.test.ts +120 tests/lib/spend-ceiling-enforcement.test.ts +82 tests/lib/spend-contract-c.test.ts +151 tests/lib/spend-corruption-compat.test.ts +89 tests/lib/spend-experimental-history.test.ts +141 tests/lib/spend-ledger-file-journal.test.ts +1232 tests/lib/spend-ledger-owner.test.ts +273 tests/lib/spend-pool-continuity.test.ts +101 tests/lib/spend-reservation-ledger.test.ts +1020 tests/lib/spend-seed-overflow.test.ts +100 tests/lib/spend-unbound-expiry.test.ts +164 tests/lib/spend-zero-config-compat.test.ts +82 tests/lib/stall-subprocess-exit.test.ts +94 tests/lib/stall-timeout.test.ts 88 tests/lib/standalone.test.ts 83 tests/lib/strict-semver.test.ts -76 tests/lib/system-restart-contract-security.test.ts -103 tests/lib/token-estimate.test.ts -89 tests/lib/transient-budget-scope-source.test.ts -132 tests/lib/transport-null-body.test.ts -8163 tests/lib/upstream-retry.test.ts -159 tests/lib/workflow-budget.test.ts -331 tests/oauth/adapter-event-oauth-failover.test.ts -4173 tests/oauth/chatgpt-device-auth.test.ts -98 tests/oauth/chatgpt-oauth.test.ts -94 tests/oauth/chatgpt-token-expiry.test.ts -225 tests/oauth/generic-oauth-failover.test.ts -384 tests/oauth/key-login-live-update.test.ts -118 tests/oauth/key-login-preserves-model-costs.test.ts -99 tests/oauth/local-token-detect.test.ts -367 tests/oauth/oauth-account-attribution.test.ts -94 tests/oauth/oauth-account-id-collision.test.ts -151 tests/oauth/oauth-account-quota-rank.test.ts -1023 tests/oauth/oauth-accounts-api.test.ts -72 tests/oauth/oauth-callback-binds.test.ts -113 tests/oauth/oauth-callback-server.test.ts -95 tests/oauth/oauth-device-code-contract.test.ts -141 tests/oauth/oauth-health.test.ts -95 tests/oauth/oauth-log.test.ts -147 tests/oauth/oauth-login-cli-browser-launch.test.ts -453 tests/oauth/oauth-login-cli-live-update.test.ts -115 tests/oauth/oauth-login-open-browser.test.ts -132 tests/oauth/oauth-login-summary.test.ts -534 tests/oauth/oauth-manual-code.test.ts -164 tests/oauth/oauth-open-browser-choice.test.ts -118 tests/oauth/oauth-open-url-result.test.ts -199 tests/oauth/oauth-provider-reconcile.test.ts -310 tests/oauth/oauth-public-surface.test.ts -178 tests/oauth/oauth-reauth-bind.test.ts -141 tests/oauth/oauth-refresh-generic-lock.test.ts -3207 tests/oauth/oauth-refresh-lock-multiprocess.test.ts -496 tests/oauth/oauth-refresh.test.ts -195 tests/oauth/oauth-status-privacy.test.ts -256 tests/oauth/oauth-store-multi.test.ts -244 tests/oauth/oauth-upsert-preserves-api-key.test.ts -81 tests/oauth/pool-kernel-generic-sweep.test.ts -159 tests/oauth/state-store-sweeper.test.ts -184 tests/providers/alibaba-region-backup.test.ts -162 tests/providers/alibaba-region-migration.test.ts -95 tests/providers/alibaba-region-startup.test.ts -340 tests/providers/alibaba-token-plan-responses-optin.test.ts -150 tests/providers/api-key-catalog-authority.test.ts -256 tests/providers/api-key-selection-capture.test.ts -139 tests/providers/aside-client.test.ts -87 tests/providers/auto-compact-budget.test.ts -140 tests/providers/azure-adapter.test.ts -102 tests/providers/azure-model-router-tool-schema.test.ts -296 tests/providers/baseten-provider.test.ts -169 tests/providers/chutes-provider.test.ts -145 tests/providers/cline-pass-deepseek-v4-tool-replay.test.ts -139 tests/providers/cline-pass-provider.test.ts -133 tests/providers/cline-pass-reasoning-efforts.test.ts -188 tests/providers/cline-provider.test.ts -3109 tests/providers/codebuddy-adapter.test.ts -106 tests/providers/codebuddy-live-acceptance.test.ts -2686 tests/providers/codebuddy-mcp-server.test.ts -98 tests/providers/codebuddy-protocol.test.ts -283 tests/providers/codebuddy-tool-bridge-turn.test.ts -129 tests/providers/codebuddy-tool-bridge.test.ts -100 tests/providers/command-code-error-finish.test.ts -278 tests/providers/command-code-fakeip-discovery.test.ts -676 tests/providers/command-code-provider.test.ts -146 tests/providers/command-code-quota.test.ts -432 tests/providers/command-code-tool-text.test.ts -101 tests/providers/command-code-workspace-cache.test.ts -233 tests/providers/commandcode-provider.test.ts -183 tests/providers/context-cap-unknown-window.test.ts -93 tests/providers/context-window-seed-repair.test.ts -183 tests/providers/crusoe-provider.test.ts -357 tests/providers/cursor/cursor-adapter.test.ts -76 tests/providers/cursor/cursor-arg-normalize.test.ts -85 tests/providers/cursor/cursor-blob-integrity.test.ts -506 tests/providers/cursor/cursor-blob.test.ts -94 tests/providers/cursor/cursor-call-id.test.ts -126 tests/providers/cursor/cursor-cancel-provenance.test.ts -118 tests/providers/cursor/cursor-catalog.test.ts +80 tests/lib/synced-state-location.test.ts +91 tests/lib/system-restart-contract-security.test.ts +110 tests/lib/token-estimate.test.ts +105 tests/lib/transient-budget-scope-source.test.ts +122 tests/lib/transport-null-body.test.ts +171 tests/lib/upstream-error-message-fallback.test.ts +78 tests/lib/upstream-retry-zero-output.test.ts +8285 tests/lib/upstream-retry.test.ts +265 tests/lib/worker-embed-coverage.test.ts +83 tests/lib/worker-embed.test.ts +148 tests/lib/workflow-budget.test.ts +590 tests/oauth/adapter-event-oauth-failover.test.ts +580 tests/oauth/anthropic2-discovery-ownership.test.ts +4200 tests/oauth/chatgpt-device-auth.test.ts +95 tests/oauth/chatgpt-oauth.test.ts +219 tests/oauth/chatgpt-token-boundaries.test.ts +156 tests/oauth/chatgpt-token-expiry.test.ts +320 tests/oauth/generic-oauth-failover.test.ts +334 tests/oauth/generic-oauth-refusal-composition.test.ts +370 tests/oauth/key-login-live-update.test.ts +102 tests/oauth/key-login-preserves-model-costs.test.ts +80 tests/oauth/local-token-detect.test.ts +294 tests/oauth/oauth-account-attribution.test.ts +203 tests/oauth/oauth-account-id-collision.test.ts +172 tests/oauth/oauth-account-plans.test.ts +149 tests/oauth/oauth-account-quota-rank.test.ts +1528 tests/oauth/oauth-accounts-api.test.ts +439 tests/oauth/oauth-anthropic-identity.test.ts +406 tests/oauth/oauth-anthropic-instance-health.test.ts +989 tests/oauth/oauth-anthropic-instance-multiprocess.test.ts +389 tests/oauth/oauth-anthropic-instance-refresh.test.ts +498 tests/oauth/oauth-anthropic-instance-registration.test.ts +81 tests/oauth/oauth-callback-binds.test.ts +107 tests/oauth/oauth-callback-server.test.ts +91 tests/oauth/oauth-device-code-contract.test.ts +138 tests/oauth/oauth-health.test.ts +609 tests/oauth/oauth-kiro-refresh-attention.test.ts +86 tests/oauth/oauth-log.test.ts +164 tests/oauth/oauth-login-cli-browser-launch.test.ts +1723 tests/oauth/oauth-login-cli-live-update.test.ts +123 tests/oauth/oauth-login-open-browser.test.ts +92 tests/oauth/oauth-login-summary.test.ts +619 tests/oauth/oauth-manual-code.test.ts +124 tests/oauth/oauth-open-browser-choice.test.ts +112 tests/oauth/oauth-open-url-result.test.ts +209 tests/oauth/oauth-provider-reconcile.test.ts +724 tests/oauth/oauth-public-surface.test.ts +204 tests/oauth/oauth-reauth-bind.test.ts +400 tests/oauth/oauth-refresh-generic-lock.test.ts +3294 tests/oauth/oauth-refresh-lock-multiprocess.test.ts +1616 tests/oauth/oauth-refresh.test.ts +303 tests/oauth/oauth-status-privacy.test.ts +410 tests/oauth/oauth-store-multi.test.ts +308 tests/oauth/oauth-upsert-preserves-api-key.test.ts +170 tests/oauth/pool-kernel-generic-sweep.test.ts +232 tests/oauth/state-store-sweeper.test.ts +107 tests/providers/alibaba-region-backup.test.ts +133 tests/providers/alibaba-region-migration.test.ts +87 tests/providers/alibaba-region-startup.test.ts +244 tests/providers/alibaba-token-plan-responses-optin.test.ts +247 tests/providers/alibaba-token-plan-wire-defaults.test.ts +198 tests/providers/api-key-catalog-authority.test.ts +99 tests/providers/api-key-selection-capture.test.ts +153 tests/providers/aside-client.test.ts +90 tests/providers/auto-compact-budget.test.ts +145 tests/providers/azure-adapter.test.ts +83 tests/providers/azure-model-router-tool-schema.test.ts +225 tests/providers/azure-vendor-metadata.test.ts +139 tests/providers/baseten-provider.test.ts +137 tests/providers/chutes-provider.test.ts +230 tests/providers/claude-cli-adapter.test.ts +156 tests/providers/cline-pass-deepseek-v4-tool-replay.test.ts +124 tests/providers/cline-pass-provider.test.ts +172 tests/providers/cline-pass-reasoning-efforts.test.ts +93 tests/providers/cline-provider.test.ts +3031 tests/providers/codebuddy-adapter.test.ts +102 tests/providers/codebuddy-live-acceptance.test.ts +154 tests/providers/codebuddy-live-models.test.ts +2872 tests/providers/codebuddy-mcp-server.test.ts +95 tests/providers/codebuddy-protocol.test.ts +405 tests/providers/codebuddy-tool-bridge-turn.test.ts +120 tests/providers/codebuddy-tool-bridge.test.ts +111 tests/providers/coding-agent-json-lines-scaling.test.ts +680 tests/providers/command-code-efforts.test.ts +104 tests/providers/command-code-error-finish.test.ts +181 tests/providers/command-code-fakeip-discovery.test.ts +2466 tests/providers/command-code-project-context.test.ts +1085 tests/providers/command-code-provider.test.ts +166 tests/providers/command-code-quota.test.ts +506 tests/providers/command-code-tool-text-prose-split.test.ts +1151 tests/providers/command-code-tool-text.test.ts +97 tests/providers/command-code-workspace-cache.test.ts +173 tests/providers/commandcode-provider.test.ts +89 tests/providers/context-cap-unknown-window.test.ts +112 tests/providers/context-window-seed-repair.test.ts +161 tests/providers/crusoe-provider.test.ts +351 tests/providers/cursor/cursor-adapter.test.ts +87 tests/providers/cursor/cursor-arg-normalize.test.ts +110 tests/providers/cursor/cursor-blob-claude-client-tools.test.ts +78 tests/providers/cursor/cursor-blob-integrity.test.ts +646 tests/providers/cursor/cursor-blob.test.ts +93 tests/providers/cursor/cursor-call-id.test.ts +147 tests/providers/cursor/cursor-cancel-provenance.test.ts +112 tests/providers/cursor/cursor-catalog.test.ts 91 tests/providers/cursor/cursor-claude-id.test.ts -223 tests/providers/cursor/cursor-continuation-invariants.test.ts -250 tests/providers/cursor/cursor-continuity-retention.test.ts -92 tests/providers/cursor/cursor-default-catalog-suppression.test.ts -109 tests/providers/cursor/cursor-desktop-exec.test.ts -95 tests/providers/cursor/cursor-discovery.test.ts -126 tests/providers/cursor/cursor-display-names.test.ts -632 tests/providers/cursor/cursor-effort-rows.test.ts -106 tests/providers/cursor/cursor-effort-suffix.test.ts -103 tests/providers/cursor/cursor-effort-table.test.ts -190 tests/providers/cursor/cursor-envelope-echo-retry.test.ts -172 tests/providers/cursor/cursor-eof-terminal.test.ts -108 tests/providers/cursor/cursor-errors.test.ts -105 tests/providers/cursor/cursor-exec-empty-result.test.ts -119 tests/providers/cursor/cursor-fast-listing.test.ts -96 tests/providers/cursor/cursor-fast-tier.test.ts -196 tests/providers/cursor/cursor-framing.test.ts -264 tests/providers/cursor/cursor-h2-pool-shutdown.test.ts -3267 tests/providers/cursor/cursor-hardening.test.ts -290 tests/providers/cursor/cursor-http1-transport.test.ts -3705 tests/providers/cursor/cursor-images.test.ts -594 tests/providers/cursor/cursor-integration-status.test.ts -111 tests/providers/cursor/cursor-interaction-query.test.ts -77 tests/providers/cursor/cursor-kv-store.test.ts +248 tests/providers/cursor/cursor-continuation-invariants.test.ts +265 tests/providers/cursor/cursor-continuity-retention.test.ts +86 tests/providers/cursor/cursor-default-catalog-suppression.test.ts +125 tests/providers/cursor/cursor-desktop-exec.test.ts +88 tests/providers/cursor/cursor-discovery.test.ts +111 tests/providers/cursor/cursor-display-names.test.ts +726 tests/providers/cursor/cursor-effort-rows.test.ts +84 tests/providers/cursor/cursor-effort-suffix.test.ts +102 tests/providers/cursor/cursor-effort-table.test.ts +163 tests/providers/cursor/cursor-envelope-echo-retry.test.ts +181 tests/providers/cursor/cursor-eof-terminal.test.ts +104 tests/providers/cursor/cursor-errors.test.ts +95 tests/providers/cursor/cursor-exec-empty-result.test.ts +106 tests/providers/cursor/cursor-fast-listing.test.ts +101 tests/providers/cursor/cursor-fast-tier.test.ts +247 tests/providers/cursor/cursor-foreground-shell.test.ts +246 tests/providers/cursor/cursor-framing.test.ts +248 tests/providers/cursor/cursor-h2-pool-shutdown.test.ts +3106 tests/providers/cursor/cursor-hardening.test.ts +138 tests/providers/cursor/cursor-http1-transport.test.ts +3084 tests/providers/cursor/cursor-images.test.ts +738 tests/providers/cursor/cursor-integration-status.test.ts +117 tests/providers/cursor/cursor-interaction-query.test.ts +93 tests/providers/cursor/cursor-kv-store.test.ts 83 tests/providers/cursor/cursor-live-smoke-gate.test.ts -296 tests/providers/cursor/cursor-live-transport.test.ts -359 tests/providers/cursor/cursor-local-models-schema.test.ts -226 tests/providers/cursor/cursor-mcp-manager.test.ts -536 tests/providers/cursor/cursor-mcp-stdio.test.ts -79 tests/providers/cursor/cursor-message-mapper.test.ts -76 tests/providers/cursor/cursor-native-exec-common.test.ts -130 tests/providers/cursor/cursor-native-exec-policy.test.ts -210 tests/providers/cursor/cursor-native-exec-shell.test.ts -593 tests/providers/cursor/cursor-native-exec.test.ts -169 tests/providers/cursor/cursor-oauth-shell.test.ts -731 tests/providers/cursor/cursor-oauth.test.ts -81 tests/providers/cursor/cursor-pool.test.ts -113 tests/providers/cursor/cursor-protobuf-events.test.ts -187 tests/providers/cursor/cursor-repetition-breaker.test.ts -208 tests/providers/cursor/cursor-request-builder.test.ts -91 tests/providers/cursor/cursor-request-compat.test.ts -126 tests/providers/cursor/cursor-roster-account-scope.test.ts -102 tests/providers/cursor/cursor-silent-redirect.test.ts -163 tests/providers/cursor/cursor-static-catalog.test.ts -3572 tests/providers/cursor/cursor-stream-health.test.ts -141 tests/providers/cursor/cursor-structured-edit.test.ts -96 tests/providers/cursor/cursor-tool-arg-decoding.test.ts -105 tests/providers/cursor/cursor-tool-choice.test.ts -133 tests/providers/cursor/cursor-tool-continuation.test.ts -102 tests/providers/cursor/cursor-tool-definitions.test.ts -4012 tests/providers/cursor/cursor-tool-finalize-race.test.ts -181 tests/providers/cursor/cursor-tool-result-image.test.ts -353 tests/providers/cursor/cursor-tool-result-invocation.test.ts -139 tests/providers/cursor/cursor-tool-suspended-checkpoint.test.ts -132 tests/providers/cursor/cursor-toolresult-normalize.test.ts -1868 tests/providers/cursor/cursor-transport-retry.test.ts -99 tests/providers/cursor/cursor-ultra-mode.test.ts -96 tests/providers/cursor/cursor-umbrella-rows.test.ts -90 tests/providers/cursor/cursor-uncallable-quarantine.test.ts -107 tests/providers/cursor/cursor-vision-wire-harness.test.ts -314 tests/providers/cyber-policy-error-fidelity.test.ts -160 tests/providers/deepinfra-provider.test.ts -512 tests/providers/deepseek-inbound-wire.test.ts -156 tests/providers/deepseek-reasoning-replay-gaps.test.ts -256 tests/providers/deepseek-reasoning-replay.test.ts -274 tests/providers/deepseek-responses-item-id-repair.test.ts -136 tests/providers/devin-adapter-reset-wait.test.ts -121 tests/providers/devin-adapter.test.ts -86 tests/providers/devin-cli-authmode-migration.test.ts -95 tests/providers/devin-effort-ladder.test.ts -138 tests/providers/devin-hardening.test.ts -132 tests/providers/devin-image-passthrough.test.ts -123 tests/providers/devin-live-models.test.ts -139 tests/providers/devin-login.test.ts +241 tests/providers/cursor/cursor-live-transport.test.ts +90 tests/providers/cursor/cursor-local-installer.test.ts +446 tests/providers/cursor/cursor-local-models-schema.test.ts +181 tests/providers/cursor/cursor-mcp-manager.test.ts +501 tests/providers/cursor/cursor-mcp-stdio.test.ts +179 tests/providers/cursor/cursor-message-mapper.test.ts +84 tests/providers/cursor/cursor-native-exec-common.test.ts +202 tests/providers/cursor/cursor-native-exec-policy.test.ts +205 tests/providers/cursor/cursor-native-exec-shell.test.ts +322 tests/providers/cursor/cursor-native-exec.test.ts +166 tests/providers/cursor/cursor-oauth-shell.test.ts +812 tests/providers/cursor/cursor-oauth.test.ts +100 tests/providers/cursor/cursor-pool.test.ts +107 tests/providers/cursor/cursor-protobuf-events.test.ts +171 tests/providers/cursor/cursor-repetition-breaker.test.ts +214 tests/providers/cursor/cursor-request-builder.test.ts +122 tests/providers/cursor/cursor-request-compat.test.ts +121 tests/providers/cursor/cursor-roster-account-scope.test.ts +107 tests/providers/cursor/cursor-silent-redirect.test.ts +134 tests/providers/cursor/cursor-static-catalog.test.ts +3594 tests/providers/cursor/cursor-stream-health.test.ts +148 tests/providers/cursor/cursor-structured-edit.test.ts +90 tests/providers/cursor/cursor-tool-arg-decoding.test.ts +89 tests/providers/cursor/cursor-tool-choice.test.ts +204 tests/providers/cursor/cursor-tool-continuation.test.ts +152 tests/providers/cursor/cursor-tool-definitions.test.ts +4025 tests/providers/cursor/cursor-tool-finalize-race.test.ts +135 tests/providers/cursor/cursor-tool-result-image.test.ts +352 tests/providers/cursor/cursor-tool-result-invocation.test.ts +115 tests/providers/cursor/cursor-tool-suspended-checkpoint.test.ts +187 tests/providers/cursor/cursor-tool-wording.test.ts +150 tests/providers/cursor/cursor-toolresult-normalize.test.ts +1877 tests/providers/cursor/cursor-transport-retry.test.ts +175 tests/providers/cursor/cursor-ultra-mode.test.ts +97 tests/providers/cursor/cursor-umbrella-rows.test.ts +85 tests/providers/cursor/cursor-uncallable-quarantine.test.ts +83 tests/providers/cursor/cursor-vision-wire-harness.test.ts +340 tests/providers/cyber-policy-error-fidelity.test.ts +247 tests/providers/deepinfra-provider.test.ts +147 tests/providers/deepseek-artifact-tool-schema.test.ts +651 tests/providers/deepseek-inbound-wire.test.ts +110 tests/providers/deepseek-quota-currency.test.ts +387 tests/providers/deepseek-reasoning-replay-gaps.test.ts +132 tests/providers/deepseek-reasoning-replay.test.ts +301 tests/providers/deepseek-responses-item-id-repair.test.ts +2146 tests/providers/devin-adapter-reset-wait.test.ts +264 tests/providers/devin-adapter.test.ts +240 tests/providers/devin-anthropic-signature-fallback.test.ts +261 tests/providers/devin-chat-wire-fixes.test.ts +122 tests/providers/devin-cli-authmode-migration.test.ts +85 tests/providers/devin-effort-ladder.test.ts +185 tests/providers/devin-family-resolution.test.ts +161 tests/providers/devin-hardening.test.ts +105 tests/providers/devin-image-passthrough.test.ts +126 tests/providers/devin-live-models.test.ts +134 tests/providers/devin-login.test.ts 127 tests/providers/devin-output-budget.test.ts -149 tests/providers/devin-prompt-cache.test.ts -100 tests/providers/devin-provider-merge-migration.test.ts -88 tests/providers/devin-stated-reset-hardening.test.ts -103 tests/providers/devin-stated-reset-retry.test.ts -121 tests/providers/devin-stream-deadline.test.ts -198 tests/providers/digitalocean-scaleway-provider.test.ts -280 tests/providers/exa-web-search.test.ts -237 tests/providers/fast-row-ingress.test.ts -296 tests/providers/featherless-provider.test.ts -149 tests/providers/flash-route-image-modalities.test.ts -399 tests/providers/forward-admission-separation.test.ts -379 tests/providers/github-copilot/github-copilot-account-origin.test.ts -16117 tests/providers/github-copilot/github-copilot-oauth.test.ts -98 tests/providers/github-copilot/github-copilot-sse-rewrite.test.ts -174 tests/providers/github-copilot/github-copilot-stream-contract.test.ts -289 tests/providers/github-copilot/github-copilot-wire-defaults.test.ts -155 tests/providers/hyperbolic-provider.test.ts -546 tests/providers/initial-model-selection.test.ts -298 tests/providers/initial-selection-write-fence.test.ts -101 tests/providers/kimi-oauth-identity.test.ts -118 tests/providers/kimi-responses-adjacency.test.ts -128 tests/providers/kiro/kiro-account-quota.test.ts -255 tests/providers/kiro/kiro-adapter.test.ts -291 tests/providers/kiro/kiro-auth-context-continuation.test.ts -142 tests/providers/kiro/kiro-builder-id-profile.test.ts -80 tests/providers/kiro/kiro-calibration.test.ts -103 tests/providers/kiro/kiro-fallback-error-body.test.ts -1745 tests/providers/kiro/kiro-images.test.ts -293 tests/providers/kiro/kiro-oauth.test.ts -136 tests/providers/kiro/kiro-pool-rank.test.ts -131 tests/providers/kiro/kiro-reasoning-roundtrip.test.ts -82 tests/providers/kiro/kiro-remote-image.test.ts -600 tests/providers/kiro/kiro-retry.test.ts -189 tests/providers/kiro/kiro-review-regressions.test.ts -478 tests/providers/kiro/kiro-stream.test.ts -120 tests/providers/kiro/kiro-usage-quota.test.ts -77 tests/providers/kiro/kiro-windows-cli-db-path.test.ts -91 tests/providers/kiro/kiro-windows-cli-executable-path.test.ts -80 tests/providers/kiro/kiro-wire-estimate.test.ts -135 tests/providers/meta-model-api-provider.test.ts -139 tests/providers/meta-muse-device.test.ts -122 tests/providers/meta-muse-login-order.test.ts -135 tests/providers/meta-muse-oauth.test.ts -85 tests/providers/mimo-effort.test.ts -213 tests/providers/mimo-free-provider.test.ts -118 tests/providers/mimo-token-plan-provider.test.ts -1531 tests/providers/minimax-clients.test.ts -286 tests/providers/minimax-reasoning-split.test.ts -95 tests/providers/model-presets.test.ts -199 tests/providers/model-rename-migration.test.ts -78 tests/providers/moonshot-endpoints.test.ts -518 tests/providers/moonshot-tool-schema.test.ts +249 tests/providers/devin-prompt-cache.test.ts +113 tests/providers/devin-provider-merge-migration.test.ts +128 tests/providers/devin-quota.test.ts +119 tests/providers/devin-reasoning-continuation.test.ts +146 tests/providers/devin-stated-reset-hardening.test.ts +124 tests/providers/devin-stated-reset-retry.test.ts +169 tests/providers/devin-stream-deadline.test.ts +231 tests/providers/devin-trajectory.test.ts +169 tests/providers/digitalocean-scaleway-provider.test.ts +147 tests/providers/exa-web-search.test.ts +111 tests/providers/fast-row-ingress.test.ts +170 tests/providers/featherless-provider.test.ts +161 tests/providers/flash-route-image-modalities.test.ts +506 tests/providers/forward-admission-separation.test.ts +819 tests/providers/github-copilot/github-copilot-account-origin.test.ts +1004 tests/providers/github-copilot/github-copilot-auto-runtime.test.ts +121 tests/providers/github-copilot/github-copilot-auto.test.ts +122 tests/providers/github-copilot/github-copilot-catalog-hints.test.ts +103 tests/providers/github-copilot/github-copilot-context-tier.test.ts +16146 tests/providers/github-copilot/github-copilot-oauth.test.ts +102 tests/providers/github-copilot/github-copilot-sse-rewrite.test.ts +319 tests/providers/github-copilot/github-copilot-stream-contract.test.ts +4105 tests/providers/github-copilot/github-copilot-wire-defaults.test.ts +180 tests/providers/haiku-5-5-catalog.test.ts +112 tests/providers/hyperbolic-provider.test.ts +312 tests/providers/initial-model-selection.test.ts +426 tests/providers/initial-selection-write-fence.test.ts +189 tests/providers/jev-provider.test.ts +98 tests/providers/kimi-oauth-identity.test.ts +132 tests/providers/kimi-responses-adjacency.test.ts +100 tests/providers/kiro/kiro-account-load.test.ts +119 tests/providers/kiro/kiro-account-quota.test.ts +4004 tests/providers/kiro/kiro-account-state-disk.test.ts +1238 tests/providers/kiro/kiro-account-state-imports.test.ts +249 tests/providers/kiro/kiro-adapter.test.ts +225 tests/providers/kiro/kiro-auth-context-continuation.test.ts +91 tests/providers/kiro/kiro-auto-selection.test.ts +126 tests/providers/kiro/kiro-builder-id-profile.test.ts +98 tests/providers/kiro/kiro-calibration.test.ts +128 tests/providers/kiro/kiro-device-builder.test.ts +296 tests/providers/kiro/kiro-device-social.test.ts +88 tests/providers/kiro/kiro-fallback-error-body.test.ts +4097 tests/providers/kiro/kiro-images.test.ts +1285 tests/providers/kiro/kiro-leased-responses.test.ts +81 tests/providers/kiro/kiro-metering-events.test.ts +146 tests/providers/kiro/kiro-metering-usage.test.ts +182 tests/providers/kiro/kiro-model-catalog.test.ts +170 tests/providers/kiro/kiro-model-preference.test.ts +266 tests/providers/kiro/kiro-oauth.test.ts +258 tests/providers/kiro/kiro-org-user-identity.test.ts +468 tests/providers/kiro/kiro-pool-load-settings.test.ts +152 tests/providers/kiro/kiro-pool-rank.test.ts +188 tests/providers/kiro/kiro-quota-metrics.test.ts +162 tests/providers/kiro/kiro-reasoning-roundtrip.test.ts +1568 tests/providers/kiro/kiro-refusal-failover.test.ts +79 tests/providers/kiro/kiro-refusal.test.ts +129 tests/providers/kiro/kiro-remote-image.test.ts +599 tests/providers/kiro/kiro-retry.test.ts +251 tests/providers/kiro/kiro-review-regressions.test.ts +144 tests/providers/kiro/kiro-single-final.test.ts +410 tests/providers/kiro/kiro-stream.test.ts +605 tests/providers/kiro/kiro-transport-parity.test.ts +113 tests/providers/kiro/kiro-usage-quota.test.ts +118 tests/providers/kiro/kiro-usage-restart.test.ts +79 tests/providers/kiro/kiro-windows-cli-db-path.test.ts +99 tests/providers/kiro/kiro-windows-cli-executable-path.test.ts +89 tests/providers/kiro/kiro-wire-estimate.test.ts +201 tests/providers/meta-model-api-provider.test.ts +136 tests/providers/meta-muse-device.test.ts +185 tests/providers/meta-muse-login-order.test.ts +183 tests/providers/meta-muse-oauth.test.ts +89 tests/providers/mimo-effort.test.ts +211 tests/providers/mimo-free-provider.test.ts +183 tests/providers/mimo-token-plan-capacity.test.ts +99 tests/providers/mimo-token-plan-provider.test.ts +198 tests/providers/mimo-v26-catalog.test.ts +1698 tests/providers/minimax-clients.test.ts +3537 tests/providers/minimax-combo-image-input.test.ts +116 tests/providers/minimax-quota.test.ts +153 tests/providers/minimax-reasoning-split.test.ts +83 tests/providers/model-presets.test.ts +251 tests/providers/model-rename-migration.test.ts +102 tests/providers/model-roster-seed-repair.test.ts +81 tests/providers/moonshot-endpoints.test.ts +430 tests/providers/moonshot-tool-schema.test.ts 125 tests/providers/muse-key-quota.test.ts -933 tests/providers/muse-passive-quota-cache.test.ts -283 tests/providers/muse-passive-quota-observation.test.ts -127 tests/providers/muse-spark-web-search-compat.test.ts -94 tests/providers/muse-subscription-usage.test.ts -126 tests/providers/muse-tool-name-alias.test.ts +958 tests/providers/muse-passive-quota-cache.test.ts +172 tests/providers/muse-passive-quota-observation.test.ts +140 tests/providers/muse-spark-web-search-compat.test.ts +91 tests/providers/muse-subscription-usage.test.ts +113 tests/providers/muse-tool-name-alias.test.ts +216 tests/providers/new-model-policy-runtime.test.ts 87 tests/providers/new-model-policy.test.ts -101 tests/providers/nous-oauth-live.test.ts -8242 tests/providers/nous-oauth.test.ts -148 tests/providers/novita-provider.test.ts -150 tests/providers/nscale-vultr-provider.test.ts -220 tests/providers/nvidia-nim-hardening.test.ts -782 tests/providers/ollama/ollama-native-parser.test.ts -115 tests/providers/ollama/ollama-native-reasoning-wire.test.ts -89 tests/providers/ollama/ollama-native-structured-output.test.ts -846 tests/providers/ollama/ollama-native-v4.test.ts -151 tests/providers/ollama/ollama-native.test.ts -491 tests/providers/ollama/ollama-show-enrichment-v7.test.ts -151 tests/providers/ollama/ollama-show-enrichment.test.ts -355 tests/providers/ollama/ollama-show-ignore-abort.test.ts -434 tests/providers/opencode-cli.test.ts -122 tests/providers/opencode-free-provider.test.ts -135 tests/providers/opencode-go-deepseek.test.ts -197 tests/providers/opencode-go-grok46-responses.test.ts -429 tests/providers/opencode-go-luna-wire.test.ts -122 tests/providers/opencode-go-muse-context.test.ts -123 tests/providers/opencode-go-muse-vision.test.ts -123 tests/providers/opencode-go-quota.test.ts -1078 tests/providers/opencode-go-session-header.test.ts -234 tests/providers/opencode-management-transport.test.ts -86 tests/providers/opencode-muse-terminal-repair.test.ts -144 tests/providers/opencode-zen-deepseek-reasoning.test.ts -97 tests/providers/opencode-zen-rate-limit.test.ts -137 tests/providers/openrouter-provider-routing.test.ts -260 tests/providers/openrouter-quota-reset-cooldown-4024.test.ts -165 tests/providers/opper-provider.test.ts -303 tests/providers/orcarouter-provider.test.ts -2088 tests/providers/provider-account-quota-persistence.test.ts -264 tests/providers/provider-account-quota.test.ts -745 tests/providers/provider-api-keys.test.ts -86 tests/providers/provider-capacity.test.ts -280 tests/providers/provider-config-batch-management.test.ts -101 tests/providers/provider-config-validation.test.ts -229 tests/providers/provider-connection-test.test.ts -136 tests/providers/provider-cost-overlay-config.test.ts -179 tests/providers/provider-discovery-log-suppression.test.ts -99 tests/providers/provider-egress-outbound.test.ts -113 tests/providers/provider-id-rewrite.test.ts -402 tests/providers/provider-key-store.test.ts -139 tests/providers/provider-live-models.test.ts -176 tests/providers/provider-model-aliases.test.ts -225 tests/providers/provider-model-discovery-contract.test.ts -89 tests/providers/provider-outbound-private-network.test.ts -1879 tests/providers/provider-outbound.test.ts -135 tests/providers/provider-quota-label-sanitize.test.ts -138 tests/providers/provider-quota-observed-marker.test.ts -477 tests/providers/provider-quota.test.ts -284 tests/providers/provider-registry-parity.test.ts -114 tests/providers/provider-static-model-discovery.test.ts -108 tests/providers/qoder-adapter.test.ts -109 tests/providers/qoder-live-models.test.ts -90 tests/providers/qoder-scaffold-guard.test.ts -89 tests/providers/qwen38-preserve-reasoning.test.ts -298 tests/providers/rate-limit-retry.test.ts -221 tests/providers/resolved-model-policy.test.ts -171 tests/providers/sambanova-nebius-provider.test.ts -89 tests/providers/sponsor-presets.test.ts -307 tests/providers/stepfun-provider.test.ts -140 tests/providers/umans-provider.test.ts -7212 tests/providers/upstream-transient-retry.test.ts -141 tests/providers/vercel-gateway-provider-routing.test.ts -137 tests/providers/vision-classification-seed-repair.test.ts -95 tests/providers/volcengine-ark-assistant-content.test.ts -180 tests/providers/xai/grok-attribution.test.ts -201 tests/providers/xai/grok-config-inject.test.ts -220 tests/providers/xai/grok-effort-inject.test.ts -146 tests/providers/xai/grok-lifecycle.test.ts -531 tests/providers/xai/grok-management-api.test.ts -480 tests/providers/xai/grok-models-effort-list.test.ts -188 tests/providers/xai/grok-orphan-adoption.test.ts -112 tests/providers/xai/grok-reset-coupon-cli.test.ts -84 tests/providers/xai/grok-reset-coupons.test.ts -132 tests/providers/xai/grok-selection.test.ts -121 tests/providers/xai/grok-status.test.ts -144 tests/providers/xai/grok-sync.test.ts -137 tests/providers/xai/grok-writer-boundary.test.ts -106 tests/providers/xai/xai-empty-catalog-tool-choice.test.ts -101 tests/providers/xai/xai-oauth-retry.test.ts -228 tests/providers/xai/xai-refresh-lock.test.ts -115 tests/providers/xai/xai-responses-adjacency.test.ts -120 tests/providers/xai/xai-tool-schema.test.ts -192 tests/providers/xai/xai-transport.test.ts -135 tests/providers/xai/xai-web-search-compat.test.ts -141 tests/providers/xai/xai-web-search.test.ts -250 tests/providers/zai-reasoning-replay.test.ts -156 tests/providers/zcode-client.test.ts -263 tests/providers/zhipu-bigmodel-provider.test.ts -117 tests/providers/zhipu-bigmodel-responses-quota.test.ts -84 tests/responses/apply-patch-envelope.test.ts -152 tests/responses/bare-echo-alias.test.ts -411 tests/responses/chat-completions-deferred-tools.test.ts -11752 tests/responses/chat-completions-endpoint.test.ts -3782 tests/responses/chat-conversation-affinity.test.ts -261 tests/responses/chat-inbound-developer-position.test.ts -87 tests/responses/chat-inbound-reasoning-none.test.ts -160 tests/responses/chat-inbound-reasoning-replay.test.ts -127 tests/responses/chat-inline-document-bytes.test.ts -413 tests/responses/chat-json-sse-fallback.test.ts -347 tests/responses/chat-media-translation.test.ts -485 tests/responses/chat-native-developer-role.test.ts -160 tests/responses/chat-native-image-normalization.test.ts -90 tests/responses/chat-native-lenient-finish.test.ts -716 tests/responses/chat-native-spend.test.ts -85 tests/responses/chat-refusal-scope.test.ts -351 tests/responses/chat-refusal.test.ts -594 tests/responses/chat-responses-control-integration.test.ts -123 tests/responses/chat-responses-control-scope.test.ts -90 tests/responses/chat-tool-choice-allowed-tools.test.ts -121 tests/responses/citation-markers.test.ts -117 tests/responses/compaction-progress.test.ts -105 tests/responses/continuation-dedup.test.ts -98 tests/responses/custom-tool-compat.test.ts -512 tests/responses/empty-completion-core.test.ts -93 tests/responses/empty-completion-guard.test.ts -192 tests/responses/empty-completion-hardening.test.ts -192 tests/responses/eventstream-decoder.test.ts -421 tests/responses/fresh-connection-optout.test.ts -84 tests/responses/legacy-shell-compat.test.ts -106 tests/responses/namespace-tool-compat.test.ts -162 tests/responses/opaque-blob-wrapped-rejection.test.ts -1098 tests/responses/openai-responses-passthrough.test.ts -108 tests/responses/parser-content-audio.test.ts -258 tests/responses/passthrough-abort.test.ts -177 tests/responses/passthrough-headers.test.ts -121 tests/responses/passthrough-override.test.ts -142 tests/responses/plaintext-v2-agent-messages.test.ts -99 tests/responses/provider-egress-fetch.test.ts -152 tests/responses/reasoning-effort-summary-default.test.ts -204 tests/responses/reasoning-envelope.test.ts -118 tests/responses/reserve-dispatch-ws.test.ts -106 tests/responses/responses-4546-incident-regression.test.ts -102 tests/responses/responses-account-change-scrub.test.ts -2729 tests/responses/responses-account-label.test.ts -236 tests/responses/responses-anthropic-fast-downgrade.test.ts -146 tests/responses/responses-bare-echo-helper-fence.test.ts -112 tests/responses/responses-canonical-only-top-level-fields.test.ts -100 tests/responses/responses-code-mode-patch-compile.test.ts -169 tests/responses/responses-code-mode-shell-compile.test.ts -6286 tests/responses/responses-compact-handoff-admission.test.ts -955 tests/responses/responses-compaction-override.test.ts -141 tests/responses/responses-compaction-policy-identity.test.ts -5958 tests/responses/responses-compaction-routing.test.ts -395 tests/responses/responses-compaction.test.ts -5884 tests/responses/responses-console-go-upload-retry.test.ts -2358 tests/responses/responses-context-overflow.test.ts -142 tests/responses/responses-continuation-boundaries.test.ts -128 tests/responses/responses-core-modules.test.ts -182 tests/responses/responses-custom-tool-guidance.test.ts -175 tests/responses/responses-custom-tool-historical-replay.test.ts -281 tests/responses/responses-custom-tool-repair-dispatch.test.ts -110 tests/responses/responses-custom-tool-repair.test.ts -106 tests/responses/responses-custom-tool-stream-consistency.test.ts -87 tests/responses/responses-default-namespace-emit-normalize.test.ts -114 tests/responses/responses-fetch-helpers-boundary.test.ts -98 tests/responses/responses-field-backfill.test.ts -114 tests/responses/responses-forward-dangling-call.test.ts -259 tests/responses/responses-forward-incomplete-quota.test.ts -123 tests/responses/responses-forward-posit-continuation.test.ts -133 tests/responses/responses-forward-prompt-envelope.test.ts -150 tests/responses/responses-freeform-wrapper-keys.test.ts -210 tests/responses/responses-function-tool-repair.test.ts -118 tests/responses/responses-hosted-tool-declaration.test.ts -125 tests/responses/responses-hosted-tool-min-spread.test.ts -181 tests/responses/responses-image-gen-repair.test.ts -945 tests/responses/responses-inbound-store-default.test.ts -103 tests/responses/responses-item-id-repair.test.ts +156 tests/providers/nous-oauth-live.test.ts +8229 tests/providers/nous-oauth.test.ts +132 tests/providers/novita-provider.test.ts +206 tests/providers/nscale-vultr-provider.test.ts +172 tests/providers/nvidia-nim-hardening.test.ts +809 tests/providers/ollama/ollama-native-parser.test.ts +108 tests/providers/ollama/ollama-native-reasoning-wire.test.ts +78 tests/providers/ollama/ollama-native-structured-output.test.ts +236 tests/providers/ollama/ollama-native-tool-continuations.test.ts +1628 tests/providers/ollama/ollama-native-v4.test.ts +135 tests/providers/ollama/ollama-native.test.ts +498 tests/providers/ollama/ollama-show-enrichment-v7.test.ts +168 tests/providers/ollama/ollama-show-enrichment.test.ts +389 tests/providers/ollama/ollama-show-ignore-abort.test.ts +260 tests/providers/opencode-cli.test.ts +106 tests/providers/opencode-free-provider.test.ts +202 tests/providers/opencode-go-deepseek.test.ts +150 tests/providers/opencode-go-grok46-responses.test.ts +565 tests/providers/opencode-go-luna-wire.test.ts +115 tests/providers/opencode-go-muse-context.test.ts +121 tests/providers/opencode-go-muse-vision.test.ts +134 tests/providers/opencode-go-quota.test.ts +1431 tests/providers/opencode-go-session-header.test.ts +554 tests/providers/opencode-management-transport.test.ts +134 tests/providers/opencode-muse-terminal-repair.test.ts +129 tests/providers/opencode-zen-deepseek-reasoning.test.ts +98 tests/providers/opencode-zen-rate-limit.test.ts +108 tests/providers/opengateway-key-validation.test.ts +105 tests/providers/opengateway-provider.test.ts +166 tests/providers/openrouter-provider-routing.test.ts +86 tests/providers/openrouter-quota-reset-cooldown-4024.test.ts +254 tests/providers/opper-provider.test.ts +206 tests/providers/orcarouter-provider.test.ts +2999 tests/providers/provider-account-quota-persistence.test.ts +380 tests/providers/provider-account-quota.test.ts +107 tests/providers/provider-anthropic-instance-quota.test.ts +251 tests/providers/provider-anthropic-instance.test.ts +170 tests/providers/provider-antigravity-effort-families.test.ts +211 tests/providers/provider-antigravity-family-catalog.test.ts +118 tests/providers/provider-antigravity-quota-retry.test.ts +3106 tests/providers/provider-antigravity-wire-snapshot.test.ts +717 tests/providers/provider-api-keys.test.ts +93 tests/providers/provider-capacity.test.ts +279 tests/providers/provider-config-batch-management.test.ts +146 tests/providers/provider-config-validation.test.ts +385 tests/providers/provider-connection-test.test.ts +133 tests/providers/provider-cost-overlay-config.test.ts +99 tests/providers/provider-discovery-log-suppression.test.ts +86 tests/providers/provider-egress-outbound.test.ts +111 tests/providers/provider-id-rewrite.test.ts +490 tests/providers/provider-key-store.test.ts +160 tests/providers/provider-live-models.test.ts +142 tests/providers/provider-model-aliases.test.ts +227 tests/providers/provider-model-discovery-contract.test.ts +80 tests/providers/provider-outbound-private-network.test.ts +878 tests/providers/provider-outbound.test.ts +211 tests/providers/provider-quota-label-sanitize.test.ts +133 tests/providers/provider-quota-observed-marker.test.ts +159 tests/providers/provider-quota-ollama-cloud.test.ts +709 tests/providers/provider-quota.test.ts +355 tests/providers/provider-registry-parity.test.ts +78 tests/providers/provider-runtime-fetch.test.ts +132 tests/providers/provider-static-model-discovery.test.ts +152 tests/providers/provider-tls-profile.test.ts +231 tests/providers/qoder-adapter.test.ts +212 tests/providers/qoder-live-models.test.ts +244 tests/providers/qoder-mcp-server.test.ts +103 tests/providers/qoder-scaffold-guard.test.ts +84 tests/providers/qwen38-preserve-reasoning.test.ts +381 tests/providers/rate-limit-retry.test.ts +285 tests/providers/resolved-model-policy.test.ts +127 tests/providers/sambanova-nebius-provider.test.ts +218 tests/providers/sponsor-presets.test.ts +163 tests/providers/stepfun-provider.test.ts +233 tests/providers/tokenlab-protocols.test.ts +102 tests/providers/tokenlab-provider.test.ts +161 tests/providers/umans-provider.test.ts +6988 tests/providers/upstream-transient-retry.test.ts +99 tests/providers/vercel-gateway-provider-routing.test.ts +123 tests/providers/vision-classification-seed-repair.test.ts +102 tests/providers/volcengine-ark-assistant-content.test.ts +106 tests/providers/xai/grok-47-build-fast-metadata.test.ts +4962 tests/providers/xai/grok-47-fast-model-wire.test.ts +144 tests/providers/xai/grok-47-fast-model.test.ts +201 tests/providers/xai/grok-attribution.test.ts +354 tests/providers/xai/grok-config-inject.test.ts +125 tests/providers/xai/grok-config-write-lock.test.ts +583 tests/providers/xai/grok-coupon-claim.test.ts +160 tests/providers/xai/grok-coupon-reconcile.test.ts +218 tests/providers/xai/grok-effort-inject.test.ts +140 tests/providers/xai/grok-lifecycle.test.ts +695 tests/providers/xai/grok-management-api.test.ts +618 tests/providers/xai/grok-models-effort-list.test.ts +417 tests/providers/xai/grok-orphan-adoption.test.ts +107 tests/providers/xai/grok-reset-coupon-cli.test.ts +109 tests/providers/xai/grok-reset-coupons.test.ts +146 tests/providers/xai/grok-selection.test.ts +165 tests/providers/xai/grok-session-identity.test.ts +146 tests/providers/xai/grok-status.test.ts +177 tests/providers/xai/grok-sync.test.ts +356 tests/providers/xai/grok-writer-boundary.test.ts +152 tests/providers/xai/xai-empty-catalog-tool-choice.test.ts +117 tests/providers/xai/xai-no-stop.test.ts +127 tests/providers/xai/xai-oauth-retry.test.ts +230 tests/providers/xai/xai-refresh-lock.test.ts +175 tests/providers/xai/xai-responses-adjacency.test.ts +162 tests/providers/xai/xai-tool-schema.test.ts +157 tests/providers/xai/xai-transport.test.ts +157 tests/providers/xai/xai-web-search-compat.test.ts +117 tests/providers/xai/xai-web-search.test.ts +155 tests/providers/zai-reasoning-replay.test.ts +249 tests/providers/zcode-client.test.ts +326 tests/providers/zed-hosted-provider.test.ts +303 tests/providers/zed-retained-budget.test.ts +184 tests/providers/zhipu-bigmodel-provider.test.ts +201 tests/providers/zhipu-bigmodel-responses-quota.test.ts +852 tests/responses/anthropic2-responses-parity.test.ts +113 tests/responses/apply-patch-envelope.test.ts +131 tests/responses/bare-echo-alias.test.ts +405 tests/responses/chat-completions-deferred-tools.test.ts +16807 tests/responses/chat-completions-endpoint.test.ts +563 tests/responses/chat-completions-pool-mode.test.ts +8919 tests/responses/chat-conversation-affinity.test.ts +328 tests/responses/chat-inbound-developer-position.test.ts +86 tests/responses/chat-inbound-reasoning-none.test.ts +97 tests/responses/chat-inbound-reasoning-replay.test.ts +125 tests/responses/chat-inline-document-bytes.test.ts +441 tests/responses/chat-json-sse-fallback.test.ts +1342 tests/responses/chat-legacy-functions-combo.test.ts +260 tests/responses/chat-media-translation.test.ts +2837 tests/responses/chat-native-combo.test.ts +146 tests/responses/chat-native-decline-reason.test.ts +670 tests/responses/chat-native-developer-role.test.ts +305 tests/responses/chat-native-image-normalization.test.ts +111 tests/responses/chat-native-lenient-finish.test.ts +18635 tests/responses/chat-native-spend.test.ts +341 tests/responses/chat-native-sse-stall.test.ts +110 tests/responses/chat-refusal-scope.test.ts +265 tests/responses/chat-refusal.test.ts +423 tests/responses/chat-responses-control-integration.test.ts +113 tests/responses/chat-responses-control-scope.test.ts +341 tests/responses/chat-sse-framing-guard.test.ts +103 tests/responses/chat-tool-choice-allowed-tools.test.ts +120 tests/responses/citation-markers.test.ts +94 tests/responses/codex-ws-pool-contention.test.ts +123 tests/responses/compaction-event-ownership.test.ts +116 tests/responses/compaction-progress.test.ts +110 tests/responses/continuation-dedup.test.ts +87 tests/responses/custom-tool-compat.test.ts +1102 tests/responses/droid-reasoning-defaults.test.ts +505 tests/responses/empty-completion-core.test.ts +100 tests/responses/empty-completion-guard.test.ts +123 tests/responses/empty-completion-hardening.test.ts +79 tests/responses/eventstream-decoder.test.ts +233 tests/responses/external-task-input-repair.test.ts +463 tests/responses/fresh-connection-optout.test.ts +904 tests/responses/jev-initial-effort-wire.test.ts +70 tests/responses/legacy-shell-compat.test.ts +248 tests/responses/messages-native-bridge-policy.test.ts +162 tests/responses/messages-native-eligibility.test.ts +138 tests/responses/messages-native-oauth-eligibility.test.ts +145 tests/responses/namespace-tool-compat.test.ts +221 tests/responses/opaque-blob-wrapped-rejection.test.ts +397 tests/responses/openai-responses-passthrough.test.ts +140 tests/responses/openai-responses-summary-none.test.ts +136 tests/responses/parser-content-audio.test.ts +391 tests/responses/passthrough-abort.test.ts +2636 tests/responses/passthrough-deferred-tools.test.ts +97 tests/responses/passthrough-grok-upstream-envelope-echo.test.ts +221 tests/responses/passthrough-headers.test.ts +141 tests/responses/passthrough-override.test.ts +622 tests/responses/plaintext-v2-agent-messages.test.ts +116 tests/responses/protocol-baseline.test.ts +94 tests/responses/protocol-contract.test.ts +178 tests/responses/protocol-direct-encoders-chat.test.ts +100 tests/responses/protocol-direct-encoders-local-upstream.test.ts +187 tests/responses/protocol-direct-encoders-messages.test.ts +101 tests/responses/protocol-direct-encoders-stall-disabled.test.ts +104 tests/responses/protocol-dto.test.ts +65 tests/responses/protocol-envelope.test.ts +100 tests/responses/protocol-features.test.ts +102 tests/responses/protocol-guard.test.ts +214 tests/responses/protocol-ingress-guard.test.ts +83 tests/responses/protocol-opaque-state.test.ts +88 tests/responses/protocol-path.test.ts +139 tests/responses/protocol-plan-snapshot.test.ts +103 tests/responses/protocol-plan.test.ts +161 tests/responses/protocol-shadow-plan.test.ts +106 tests/responses/protocol-trace.test.ts +77 tests/responses/provider-egress-fetch.test.ts +145 tests/responses/reasoning-effort-summary-default.test.ts +210 tests/responses/reasoning-envelope.test.ts +124 tests/responses/reserve-dispatch-ws.test.ts +114 tests/responses/responses-4546-incident-regression.test.ts +97 tests/responses/responses-account-change-scrub.test.ts +3188 tests/responses/responses-account-label.test.ts +1693 tests/responses/responses-alternate-main-cancellation.test.ts +402 tests/responses/responses-anthropic-fast-downgrade.test.ts +327 tests/responses/responses-azure-opaque-blob-recovery.test.ts +152 tests/responses/responses-bare-echo-helper-fence.test.ts +603 tests/responses/responses-canonical-nonstream.test.ts +134 tests/responses/responses-canonical-only-top-level-fields.test.ts +200 tests/responses/responses-chat-tool-call-content.test.ts +149 tests/responses/responses-chatgpt-web-replay-metadata.test.ts +86 tests/responses/responses-code-mode-goal-helpers.test.ts +141 tests/responses/responses-code-mode-mcp-direct.test.ts +104 tests/responses/responses-code-mode-patch-compile.test.ts +197 tests/responses/responses-code-mode-shell-compile.test.ts +418 tests/responses/responses-combo-exhausted-error.test.ts +6181 tests/responses/responses-compact-handoff-admission.test.ts +5233 tests/responses/responses-compaction-override.test.ts +149 tests/responses/responses-compaction-policy-identity.test.ts +127 tests/responses/responses-compaction-recovery-policy.test.ts +6682 tests/responses/responses-compaction-recovery.test.ts +6330 tests/responses/responses-compaction-routing.test.ts +312 tests/responses/responses-compaction.test.ts +5980 tests/responses/responses-console-go-upload-retry.test.ts +2719 tests/responses/responses-context-overflow.test.ts +102 tests/responses/responses-continuation-boundaries.test.ts +135 tests/responses/responses-core-modules.test.ts +100 tests/responses/responses-custom-tool-guidance.test.ts +347 tests/responses/responses-custom-tool-historical-replay.test.ts +264 tests/responses/responses-custom-tool-repair-dispatch.test.ts +118 tests/responses/responses-custom-tool-repair.test.ts +97 tests/responses/responses-custom-tool-stream-consistency.test.ts +115 tests/responses/responses-default-namespace-emit-normalize.test.ts +923 tests/responses/responses-devin-401-replay.test.ts +93 tests/responses/responses-dispatch-receipt.test.ts +129 tests/responses/responses-fetch-helpers-boundary.test.ts +213 tests/responses/responses-field-backfill.test.ts +9707 tests/responses/responses-first-send-reservation.test.ts +251 tests/responses/responses-first-send-spend-policy.test.ts +126 tests/responses/responses-forward-client-headers.test.ts +142 tests/responses/responses-forward-dangling-call.test.ts +293 tests/responses/responses-forward-incomplete-quota.test.ts +191 tests/responses/responses-forward-output-cap.test.ts +120 tests/responses/responses-forward-posit-continuation.test.ts +144 tests/responses/responses-forward-prompt-envelope.test.ts +229 tests/responses/responses-freeform-wrapper-keys.test.ts +248 tests/responses/responses-function-tool-repair.test.ts +4562 tests/responses/responses-grok-devin-preflight.test.ts +207 tests/responses/responses-hide-raw-reasoning.test.ts +302 tests/responses/responses-hosted-image-delivery.test.ts +116 tests/responses/responses-hosted-image-display.test.ts +383 tests/responses/responses-hosted-image-limits.test.ts +342 tests/responses/responses-hosted-send-hop.test.ts +337 tests/responses/responses-hosted-send-producer.test.ts +376 tests/responses/responses-hosted-send-reservation.test.ts +167 tests/responses/responses-hosted-send-round-parity.test.ts +257 tests/responses/responses-hosted-send-unsent-hop.test.ts +171 tests/responses/responses-hosted-send-ws.test.ts +113 tests/responses/responses-hosted-tool-declaration.test.ts +113 tests/responses/responses-hosted-tool-min-spread.test.ts +153 tests/responses/responses-image-gen-repair.test.ts +826 tests/responses/responses-inbound-store-default.test.ts +131 tests/responses/responses-item-id-repair.test.ts 106 tests/responses/responses-json-events.test.ts -82 tests/responses/responses-legacy-dotted-tool-name-repair.test.ts -228 tests/responses/responses-muse-tool-name-alias.test.ts -234 tests/responses/responses-native-main-refresh.test.ts -4869 tests/responses/responses-opaque-blob-recovery.test.ts -84 tests/responses/responses-parser-agent-message.test.ts -123 tests/responses/responses-parser-malformed-content.test.ts -207 tests/responses/responses-parser.test.ts -120 tests/responses/responses-passthrough-transient-policy.test.ts -1249 tests/responses/responses-pool-401-refresh.test.ts -151 tests/responses/responses-pool-refresh-attribution.test.ts -553 tests/responses/responses-preview-main-read-fence.test.ts -216 tests/responses/responses-reasoning-effort-downgrade.test.ts -179 tests/responses/responses-reasoning-summary-passthrough.test.ts -119 tests/responses/responses-reset-replay.test.ts -133 tests/responses/responses-routed-web-search-fields.test.ts -2175 tests/responses/responses-self-named-namespace-scrub.test.ts -5621 tests/responses/responses-send-budget-counts.test.ts -92 tests/responses/responses-send-budget-errors.test.ts -304 tests/responses/responses-shadow-intercept.test.ts -229 tests/responses/responses-show-thinking-summary.test.ts -558 tests/responses/responses-snapshot-repair-server.test.ts -176 tests/responses/responses-snapshot-repair.test.ts -122 tests/responses/responses-sparse-terminal-tool-scope.test.ts -117 tests/responses/responses-spend-ledger-wiring.test.ts -309 tests/responses/responses-spill-shutdown-clock.test.ts -293 tests/responses/responses-state-write-amplification.test.ts -7918 tests/responses/responses-state.test.ts -858 tests/responses/responses-stateless-dangling-call-repair.test.ts -96 tests/responses/responses-stream-tool-events.test.ts -123 tests/responses/responses-terminal-repair.test.ts -101 tests/responses/responses-tool-conformance.test.ts -86 tests/responses/responses-tool-groups.test.ts -239 tests/responses/responses-tool-search-repair.test.ts -810 tests/responses/responses-undeclared-tool-guard.test.ts -141 tests/responses/responses-usage-passthrough.test.ts -141 tests/responses/responses-xai-request-compat.test.ts -218 tests/responses/sse-client-frame-bounds.test.ts -516 tests/responses/sse-decoder.test.ts -278 tests/responses/sse-failed-tail.test.ts -501 tests/responses/sse-inspector-bounds.test.ts -126 tests/responses/sse-null-data-frame.test.ts -138 tests/responses/sse-payload-rewrite.test.ts -108 tests/responses/sse-unspaced-data-fields.test.ts -96 tests/responses/thought-signature-credential-scope.test.ts -157 tests/responses/ws-endpoint.test.ts -118 tests/responses/ws-failure-stage.test.ts -415 tests/responses/ws-native-injection.test.ts -266 tests/responses/ws-native-result-continuations.test.ts -3315 tests/responses/ws-native-steering.test.ts -282 tests/responses/ws-steering-completion.test.ts -202 tests/responses/ws-steering-smoke.test.ts -261 tests/responses/ws-steering-stability.test.ts -152 tests/responses/ws-upstream-reuse.test.ts -111 tests/responses/ws-upstream-socks5.test.ts -760 tests/responses/ws-upstream.test.ts -146 tests/routing/always-on-429-failover.test.ts -131 tests/routing/anthropic-quorum-cache.test.ts -91 tests/routing/cl01-claude-outbound-review-regressions.test.ts -190 tests/routing/cl01-openai-chat-review-regressions.test.ts -159 tests/routing/cl01-review-regressions.test.ts -361 tests/routing/combo-child-headers.test.ts -588 tests/routing/combo-management-api.test.ts -214 tests/routing/combo-stream-preflight.test.ts -249 tests/routing/compatibility-provider-equivalence.test.ts -102 tests/routing/destination-policy-resolved.test.ts -142 tests/routing/fastwire-characterization-routing.test.ts -403 tests/routing/fastwire-characterization-wire.test.ts -248 tests/routing/fastwire-observability.test.ts -151 tests/routing/fastwire-policy.test.ts -169 tests/routing/policy-execution.test.ts -462 tests/routing/probe-lease-dispatch-wiring.test.ts -318 tests/routing/probe-lease.test.ts -144 tests/routing/resolved-model-policy-consumers.test.ts -111 tests/routing/router-combo-failover-classification.test.ts -154 tests/routing/router-discarded-baseurl-warning.test.ts -129 tests/routing/router-template-baseurl.test.ts -157 tests/routing/router.test.ts -639 tests/routing/routing-analytics.test.ts -95 tests/routing/routing-capability-catalog.test.ts -429 tests/routing/routing-capability-model-matching.test.ts -100 tests/routing/routing-compatibility-auth-identity.test.ts +65 tests/responses/responses-legacy-dotted-tool-name-repair.test.ts +434 tests/responses/responses-main-quota-observation.test.ts +250 tests/responses/responses-memory-models.test.ts +183 tests/responses/responses-muse-tool-choice.test.ts +226 tests/responses/responses-muse-tool-name-alias.test.ts +733 tests/responses/responses-native-main-refresh.test.ts +5140 tests/responses/responses-opaque-blob-recovery.test.ts +93 tests/responses/responses-parser-agent-message.test.ts +129 tests/responses/responses-parser-malformed-content.test.ts +159 tests/responses/responses-parser.test.ts +122 tests/responses/responses-passthrough-transient-policy.test.ts +5232 tests/responses/responses-pool-401-refresh.test.ts +153 tests/responses/responses-pool-refresh-attribution.test.ts +309 tests/responses/responses-preview-main-read-fence.test.ts +223 tests/responses/responses-reasoning-effort-downgrade.test.ts +498 tests/responses/responses-reasoning-summary-passthrough.test.ts +112 tests/responses/responses-reset-replay.test.ts +194 tests/responses/responses-routed-web-search-fields.test.ts +635 tests/responses/responses-run-turn-web-search.test.ts +2256 tests/responses/responses-self-named-namespace-scrub.test.ts +6566 tests/responses/responses-send-budget-counts.test.ts +381 tests/responses/responses-send-budget-errors.test.ts +295 tests/responses/responses-shadow-intercept.test.ts +272 tests/responses/responses-show-thinking-summary.test.ts +226 tests/responses/responses-skills-snapshot.test.ts +871 tests/responses/responses-snapshot-repair-server.test.ts +183 tests/responses/responses-snapshot-repair.test.ts +125 tests/responses/responses-sparse-terminal-tool-scope.test.ts +600 tests/responses/responses-spend-anchor.test.ts +134 tests/responses/responses-spend-capacity-guard.test.ts +169 tests/responses/responses-spend-contract-c.test.ts +148 tests/responses/responses-spend-ledger-wiring.test.ts +113 tests/responses/responses-spill-acl-recovery.test.ts +325 tests/responses/responses-spill-admission-headroom.test.ts +123 tests/responses/responses-spill-inspection.test.ts +125 tests/responses/responses-spill-orphan-sweep.test.ts +316 tests/responses/responses-spill-shutdown-clock.test.ts +90 tests/responses/responses-state-snapshot-select.test.ts +110 tests/responses/responses-state-snapshot-stubs.test.ts +393 tests/responses/responses-state-store-false.test.ts +254 tests/responses/responses-state-write-amplification.test.ts +8178 tests/responses/responses-state.test.ts +728 tests/responses/responses-stateless-dangling-call-repair.test.ts +95 tests/responses/responses-stream-tool-events.test.ts +130 tests/responses/responses-terminal-repair.test.ts +157 tests/responses/responses-tool-conformance.test.ts +90 tests/responses/responses-tool-groups.test.ts +259 tests/responses/responses-tool-search-repair.test.ts +18040 tests/responses/responses-translated-reset.test.ts +583 tests/responses/responses-undeclared-tool-guard.test.ts +138 tests/responses/responses-usage-passthrough.test.ts +174 tests/responses/responses-xai-request-compat.test.ts +256 tests/responses/shadow-intercept-target-lifecycle.test.ts +129 tests/responses/sse-carriage-return.test.ts +584 tests/responses/sse-client-frame-bounds.test.ts +353 tests/responses/sse-decoder.test.ts +412 tests/responses/sse-failed-tail.test.ts +470 tests/responses/sse-inspector-bounds.test.ts +153 tests/responses/sse-null-data-frame.test.ts +158 tests/responses/sse-payload-rewrite.test.ts +207 tests/responses/sse-rewrite-line-endings.test.ts +95 tests/responses/sse-unspaced-data-fields.test.ts +109 tests/responses/thought-signature-credential-scope.test.ts +86 tests/responses/visualization-directives.test.ts +669 tests/responses/ws-ambiguous-resend.test.ts +138 tests/responses/ws-endpoint.test.ts +122 tests/responses/ws-failure-stage.test.ts +498 tests/responses/ws-native-injection.test.ts +425 tests/responses/ws-native-result-continuations.test.ts +2663 tests/responses/ws-native-steering.test.ts +273 tests/responses/ws-steering-completion.test.ts +186 tests/responses/ws-steering-smoke.test.ts +280 tests/responses/ws-steering-stability.test.ts +158 tests/responses/ws-upstream-reuse.test.ts +98 tests/responses/ws-upstream-socks5.test.ts +1048 tests/responses/ws-upstream.test.ts +177 tests/routing/always-on-429-failover.test.ts +148 tests/routing/anthropic-quorum-cache.test.ts +89 tests/routing/cl01-claude-outbound-review-regressions.test.ts +103 tests/routing/cl01-openai-chat-review-regressions.test.ts +117 tests/routing/cl01-review-regressions.test.ts +182 tests/routing/codex-agent-role-models.test.ts +116 tests/routing/codex-role-auto-assign.test.ts +141 tests/routing/codex-role-sizing.test.ts +152 tests/routing/combo-child-headers.test.ts +889 tests/routing/combo-management-api.test.ts +162 tests/routing/combo-stream-preflight.test.ts +259 tests/routing/compatibility-provider-equivalence.test.ts +167 tests/routing/destination-policy-resolved.test.ts +134 tests/routing/fastwire-characterization-routing.test.ts +383 tests/routing/fastwire-characterization-wire.test.ts +255 tests/routing/fastwire-observability.test.ts +175 tests/routing/fastwire-policy.test.ts +812 tests/routing/fastwire-response-authority.test.ts +107 tests/routing/jev-decision-destination.test.ts +291 tests/routing/jev-decision-model-config.test.ts +370 tests/routing/jev-decision-provider-combo.test.ts +153 tests/routing/jev-decision.test.ts +1098 tests/routing/jev-model-backend.test.ts +112 tests/routing/jev-service-exchange.test.ts +84 tests/routing/jev-typesafe-golden.test.ts +176 tests/routing/policy-execution.test.ts +296 tests/routing/probe-lease-dispatch-wiring.test.ts +236 tests/routing/probe-lease.test.ts +138 tests/routing/resolved-model-policy-consumers.test.ts +132 tests/routing/router-blocked-cross-provider.test.ts +81 tests/routing/router-combo-failover-classification.test.ts +148 tests/routing/router-discarded-baseurl-warning.test.ts +229 tests/routing/router-template-baseurl.test.ts +154 tests/routing/router.test.ts +366 tests/routing/routing-analytics.test.ts +103 tests/routing/routing-capability-catalog.test.ts +257 tests/routing/routing-capability-model-matching.test.ts +78 tests/routing/routing-compatibility-auth-identity.test.ts 259 tests/routing/routing-compatibility-boundaries.test.ts -84 tests/routing/routing-compatibility-model-matching.test.ts -220 tests/routing/routing-compatibility.test.ts -181 tests/routing/routing-identity-domains.test.ts -361 tests/routing/routing-policy-fallback.test.ts -116 tests/routing/routing-policy-pool-quota.test.ts -344 tests/routing/routing-policy-surface-parity.test.ts -473 tests/routing/routing-profile-management-editor.test.ts -346 tests/routing/routing-profile.test.ts -130 tests/routing/subagent-context-staleness.test.ts -99 tests/routing/subagent-defaults.test.ts -3828 tests/routing/subagent-fallback-handle-responses.test.ts -194 tests/routing/subagent-fallback-preview-sites.test.ts -209 tests/routing/subagent-model-fallback-api.test.ts -465 tests/routing/subagent-model-fallback.test.ts -182 tests/routing/subagent-roster-retention.test.ts -128 tests/server/account-import.test.ts -1168 tests/server/account-pool-management-api.test.ts -147 tests/server/adapter-resolve.test.ts -177 tests/server/agent-task-recovery-cache.test.ts -394 tests/server/agent-task-recovery-combo.test.ts -202 tests/server/agent-task-recovery-fallback.test.ts -242 tests/server/agent-task-recovery-security.test.ts -1379 tests/server/agent-task-recovery.test.ts -170 tests/server/alias-management-api.test.ts -120 tests/server/api-access-endpoints.test.ts -701 tests/server/api-catalog-route.test.ts -338 tests/server/api-codex-log-guard-compact.test.ts -269 tests/server/api-codex-log-guard-protection.test.ts -170 tests/server/api-codex-log-guard.test.ts -689 tests/server/api-debug.test.ts -2391 tests/server/api-key-attribution.test.ts -130 tests/server/api-key-model-scope.test.ts -169 tests/server/api-key-scope-alpha-search.test.ts -158 tests/server/api-key-scope-audio.test.ts -169 tests/server/api-key-scope-images.test.ts -157 tests/server/api-key-scope-live.test.ts -1559 tests/server/api-keys-routes.test.ts -510 tests/server/aside-profiles-routes.test.ts -112 tests/server/audio-client.test.ts -693 tests/server/audio-dictation.test.ts -823 tests/server/audio-transcriptions.test.ts -371 tests/server/bounded-body.test.ts -145 tests/server/bridge-live-delivery.test.ts -2359 tests/server/cancel-body-on-abort.test.ts -651 tests/server/claude-intercept-integration.test.ts -144 tests/server/companion-settings.test.ts -427 tests/server/config.test.ts -138 tests/server/consume-for-inspection-cancel.test.ts -255 tests/server/context-history-ownership.test.ts -121 tests/server/context-history.test.ts -536 tests/server/data-plane-admission-identity.test.ts -87 tests/server/debug-settings.test.ts -173 tests/server/error-fidelity.test.ts -227 tests/server/errors-adapter-failure.test.ts -336 tests/server/fetch-header-timeout.test.ts -414 tests/server/health-scoring.test.ts -364 tests/server/hub-usage.test.ts -126 tests/server/input-admission.test.ts -141 tests/server/live-call-bindings.test.ts -343 tests/server/local-aside-sync-capability.test.ts -79 tests/server/local-management-attestation.test.ts -84 tests/server/local-management-capability.test.ts -700 tests/server/local-management-direct-transport.test.ts -81 tests/server/local-provider-reload-client.test.ts -580 tests/server/logs-timezone.test.ts -134 tests/server/loopback-companion-client-targets.test.ts -138 tests/server/loopback-listener-admission.test.ts -2199 tests/server/loopback-listener-integration.test.ts -147 tests/server/management-anthropic-reset-grants.test.ts -185 tests/server/management-api-logs-metrics.test.ts -437 tests/server/management-client-config-route.test.ts -180 tests/server/management-google-tool-schema-policy.test.ts -264 tests/server/management-integration-journal-delete.test.ts -641 tests/server/management-integration-routes.test.ts -1379 tests/server/management-metrics-export.test.ts -150 tests/server/management-model-roster-gather-race.test.ts -136 tests/server/management-model-roster.test.ts -103 tests/server/management-origin-tls.test.ts -256 tests/server/management-provider-pinsless-validation.test.ts -245 tests/server/management-provider-proto-override.test.ts -91 tests/server/management-provider-reset-replay.test.ts -370 tests/server/management-provider-synthetic-max.test.ts -6724 tests/server/management-provider-validation.test.ts -322 tests/server/management-provider-verbosity.test.ts -205 tests/server/management-route-registry.test.ts -141 tests/server/management-workflow-budget-routes.test.ts -538 tests/server/memory-watchdog.test.ts -200 tests/server/model-costs-management-api.test.ts -346 tests/server/model-discovery-management-api.test.ts -124 tests/server/outbound-body-guard.test.ts -88 tests/server/owned-service-home.test.ts -94 tests/server/passive-route-linker.test.ts -5380 tests/server/plaintext-v2-agent-messages-server.test.ts -922 tests/server/port-reclaim.test.ts -543 tests/server/ports.test.ts +120 tests/routing/routing-compatibility-model-matching.test.ts +198 tests/routing/routing-compatibility.test.ts +99 tests/routing/routing-identity-domains.test.ts +276 tests/routing/routing-policy-dispatch-scope.test.ts +422 tests/routing/routing-policy-fallback.test.ts +112 tests/routing/routing-policy-pool-quota.test.ts +204 tests/routing/routing-policy-surface-parity.test.ts +728 tests/routing/routing-profile-management-editor.test.ts +314 tests/routing/routing-profile.test.ts +131 tests/routing/subagent-context-staleness.test.ts +137 tests/routing/subagent-defaults.test.ts +5035 tests/routing/subagent-fallback-handle-responses.test.ts +159 tests/routing/subagent-fallback-preview-sites.test.ts +230 tests/routing/subagent-model-fallback-api.test.ts +451 tests/routing/subagent-model-fallback.test.ts +149 tests/routing/subagent-roster-migration.test.ts +454 tests/routing/subagent-roster-retention.test.ts +275 tests/server/account-import.test.ts +2359 tests/server/account-pool-management-api.test.ts +255 tests/server/adapter-resolve.test.ts +287 tests/server/agent-task-recovery-cache.test.ts +318 tests/server/agent-task-recovery-combo.test.ts +201 tests/server/agent-task-recovery-fallback.test.ts +341 tests/server/agent-task-recovery-security.test.ts +11413 tests/server/agent-task-recovery.test.ts +250 tests/server/alias-management-api.test.ts +234 tests/server/anthropic2-management.test.ts +104 tests/server/api-access-endpoints.test.ts +660 tests/server/api-catalog-route.test.ts +378 tests/server/api-codex-log-guard-compact.test.ts +160 tests/server/api-codex-log-guard-protection.test.ts +193 tests/server/api-codex-log-guard.test.ts +843 tests/server/api-debug.test.ts +2984 tests/server/api-key-attribution.test.ts +145 tests/server/api-key-model-scope.test.ts +671 tests/server/api-key-scope-alpha-search.test.ts +315 tests/server/api-key-scope-audio.test.ts +161 tests/server/api-key-scope-images.test.ts +165 tests/server/api-key-scope-live.test.ts +1957 tests/server/api-keys-routes.test.ts +2078 tests/server/api-usage.test.ts +769 tests/server/aside-profiles-routes.test.ts +113 tests/server/audio-client.test.ts +839 tests/server/audio-dictation.test.ts +1077 tests/server/audio-transcriptions.test.ts +740 tests/server/bounded-body.test.ts +171 tests/server/bridge-live-delivery.test.ts +163 tests/server/caller-session-identity.test.ts +474 tests/server/caller-session-ingress.test.ts +2590 tests/server/cancel-body-on-abort.test.ts +1307 tests/server/claude-intercept-integration.test.ts +224 tests/server/codex-agent-role-routes.test.ts +183 tests/server/codex-role-auto-assign-routes.test.ts +71 tests/server/combo-requested-effort.test.ts +147 tests/server/companion-settings.test.ts +431 tests/server/config.test.ts +110 tests/server/consume-for-inspection-cancel.test.ts +585 tests/server/context-history-ownership.test.ts +255 tests/server/context-history.test.ts +252 tests/server/data-plane-admission-identity.test.ts +86 tests/server/debug-settings.test.ts +114 tests/server/decision-discovery.test.ts +339 tests/server/decision-routes.test.ts +148 tests/server/error-fidelity.test.ts +331 tests/server/errors-adapter-failure.test.ts +331 tests/server/fetch-header-timeout.test.ts +348 tests/server/health-scoring.test.ts +479 tests/server/hub-usage.test.ts +122 tests/server/inference-attempt.test.ts +152 tests/server/inference-client-encoder-delivery.test.ts +94 tests/server/inference-client-wire.test.ts +109 tests/server/inference-final-log.test.ts +1426 tests/server/inference-send-budget.test.ts +148 tests/server/input-admission.test.ts +163 tests/server/jev-decision-scope.test.ts +215 tests/server/link-enrollment.test.ts +2565 tests/server/link-join-route.test.ts +753 tests/server/link-listener-admission.test.ts +171 tests/server/link-listener-lifecycle.test.ts +116 tests/server/link-listener-mutation.test.ts +660 tests/server/link-management-routes.test.ts +131 tests/server/live-call-bindings.test.ts +94 tests/server/live-native-calls.test.ts +178 tests/server/local-account-switch-capability.test.ts +272 tests/server/local-account-switch-ingress.test.ts +475 tests/server/local-aside-sync-capability.test.ts +101 tests/server/local-management-attestation.test.ts +121 tests/server/local-management-capability.test.ts +813 tests/server/local-management-direct-transport.test.ts +92 tests/server/local-provider-reload-client.test.ts +481 tests/server/local-read-response-proof.test.ts +161 tests/server/logs-timezone.test.ts +181 tests/server/loopback-companion-client-targets.test.ts +131 tests/server/loopback-listener-admission.test.ts +3018 tests/server/loopback-listener-integration.test.ts +125 tests/server/management-anthropic-reset-grants.test.ts +187 tests/server/management-api-logs-metrics.test.ts +257 tests/server/management-api-logs-timing-basis.test.ts +632 tests/server/management-client-config-route.test.ts +329 tests/server/management-droid-reasoning-defaults.test.ts +200 tests/server/management-google-tool-schema-policy.test.ts +191 tests/server/management-inference-port.test.ts +375 tests/server/management-integration-journal-delete.test.ts +1917 tests/server/management-integration-routes.test.ts +1729 tests/server/management-metrics-export.test.ts +166 tests/server/management-model-roster-gather-race.test.ts +233 tests/server/management-model-roster.test.ts +123 tests/server/management-origin-tls.test.ts +216 tests/server/management-provider-atomicity.test.ts +1332 tests/server/management-provider-compat-carry.test.ts +665 tests/server/management-provider-hide-raw-reasoning.test.ts +267 tests/server/management-provider-pinsless-validation.test.ts +303 tests/server/management-provider-proto-override.test.ts +695 tests/server/management-provider-reasoning-lists.test.ts +200 tests/server/management-provider-request-pacing.test.ts +101 tests/server/management-provider-reset-replay.test.ts +137 tests/server/management-provider-synthetic-max.test.ts +205 tests/server/management-provider-upstream-websocket.test.ts +9513 tests/server/management-provider-validation.test.ts +369 tests/server/management-provider-verbosity.test.ts +437 tests/server/management-route-registry.test.ts +405 tests/server/management-sidecar-anthropic-instance.test.ts +246 tests/server/management-workflow-budget-routes.test.ts +638 tests/server/memory-watchdog.test.ts +373 tests/server/model-costs-management-api.test.ts +177 tests/server/model-discovery-management-api.test.ts +419 tests/server/model-export-new-model-policy.test.ts +183 tests/server/model-settings-management-api.test.ts +120 tests/server/outbound-body-guard.test.ts +95 tests/server/owned-service-home.test.ts +149 tests/server/passive-route-linker.test.ts +15148 tests/server/plaintext-v2-agent-messages-server.test.ts +1226 tests/server/port-reclaim.test.ts +536 tests/server/ports.test.ts +2698 tests/server/probe-timeout-env.test.ts +133 tests/server/protocol-provider-summary.test.ts +151 tests/server/protocol-routes.test.ts +140 tests/server/protocol-settings-route.test.ts 298 tests/server/provider-account-quota-routes.test.ts -97 tests/server/provider-egress-management-validation.test.ts -358 tests/server/provider-send-path-import.test.ts -129 tests/server/proxy-env.test.ts -121 tests/server/proxy-liveness.test.ts -711 tests/server/relay-eager.test.ts -707 tests/server/replay-refusal-parity.test.ts -412 tests/server/reserve-claude-policy.test.ts -4966 tests/server/reserve-ingress.test.ts -787 tests/server/response-log-inspection.test.ts -208 tests/server/response-model-identity.test.ts -161 tests/server/retry-after-429.test.ts -99 tests/server/retry-delay-hardening.test.ts -245 tests/server/route-decision-trace.test.ts -642 tests/server/server-403-permission-e2e.test.ts -269 tests/server/server-agent-task-recovery-replay.test.ts -234 tests/server/server-auth-localhost-bind.test.ts -1652 tests/server/server-auth-scoped-quota.test.ts -17291 tests/server/server-auth.test.ts -629 tests/server/server-background-lifecycle.test.ts -81 tests/server/server-clickjacking-headers.test.ts -8206 tests/server/server-combo-failover-e2e.test.ts -93 tests/server/server-combo-held-response.test.ts -3804 tests/server/server-combo-reasoning-replay-eligibility.test.ts -219 tests/server/server-combo-zero-output-failover.test.ts -1175 tests/server/server-google-antigravity-oauth-401-replay.test.ts -108 tests/server/server-gui-bundle-freshness.test.ts -144 tests/server/server-images-bodyless-content-length.test.ts -3645 tests/server/server-images.test.ts -3733 tests/server/server-key-failover-e2e.test.ts -894 tests/server/server-kiro-completion-e2e.test.ts -512 tests/server/server-kiro-oauth-401-replay.test.ts -164 tests/server/server-live-frame-log.test.ts -826 tests/server/server-live-realtime-fixtures.test.ts -4860 tests/server/server-live.test.ts -190 tests/server/server-loopback-host-gate.test.ts -1442 tests/server/server-management-auth.test.ts -374 tests/server/server-opencode-go-goal-streaming.test.ts -1232 tests/server/server-rate-limit-retry-e2e.test.ts -1183 tests/server/server-request-body-size.test.ts -1263 tests/server/server-search.test.ts -688 tests/server/server-startup-reconcile-resilience.test.ts -376 tests/server/server-stop-config-hardening.test.ts -298 tests/server/server-xai-chat-reasoning-streaming.test.ts -651 tests/server/server-xai-header-parity.test.ts -603 tests/server/server-xai-oauth-401-replay.test.ts -747 tests/server/server-xai-responses-streaming.test.ts -157 tests/server/session-affinity.test.ts -296 tests/server/session-lane-recall-harness.test.ts -178 tests/server/sidebar-routes.test.ts -81 tests/server/sidebar-star-state.test.ts -108 tests/server/spend-instrumentation-log.test.ts -91 tests/server/spend-ledger-lifecycle.test.ts -291 tests/server/spend-ledger-owner-startup.test.ts -115 tests/server/startup-action-control-elevation.test.ts -142 tests/server/startup-action-control.test.ts -100 tests/server/startup-prompt.test.ts -122 tests/server/stream-aborted-marker.test.ts -172 tests/server/system-env.test.ts -193 tests/server/system-restart.test.ts -347 tests/server/system-routes.test.ts -4191 tests/server/terminal-guard-server.test.ts -170 tests/server/terminal-guard.test.ts -87 tests/server/upstream-connect-error.test.ts -105 tests/server/upstream-http-version.test.ts -577 tests/server/v1-hub-state.test.ts -277 tests/server/v2-agent-message-failfast.test.ts -145 tests/service/autostart-health.test.ts -3644 tests/service/container-bootstrap.test.ts -184 tests/service/crash-guard.test.ts -134 tests/service/doctor-codex-envkey-readiness.test.ts -134 tests/service/doctor-oauth.test.ts -143 tests/service/doctor-provider-apikey.test.ts -85 tests/service/doctor-service-memory-contract.test.ts -116 tests/service/init-backup-cleanup.test.ts -7056 tests/service/init-eof.test.ts -242 tests/service/launchd-repair.test.ts -117 tests/service/live-service-manager-guard.test.ts -89 tests/service/managing-cli.test.ts -89 tests/service/process-state.test.ts -151 tests/service/service-auth-qualified-localhost.test.ts -104 tests/service/service-claim.test.ts -82 tests/service/service-ownership-compatibility.test.ts -189 tests/service/service-ownership-handover.test.ts -351 tests/service/service-ownership-state.test.ts -87 tests/service/service-probe-docker.test.ts -99 tests/service/service-secrets.test.ts -262 tests/service/service-sqlite-home.test.ts -191 tests/service/service-start-environment.test.ts -115 tests/service/service-stop-verification.test.ts -323 tests/service/service-tier-capability.test.ts -124 tests/service/service-wsl-home-ownership.test.ts -496 tests/service/service.test.ts -337 tests/service/shutdown-drain.test.ts -4745 tests/service/shutdown-launcher.test.ts -74 tests/service/stale-state-purge.test.ts -82 tests/service/standalone-service.test.ts -3590 tests/service/stop-deferred-teardown.test.ts -92 tests/service/systemd-install-cleanup-hardening.test.ts -127 tests/service/winsw.test.ts -3107 tests/storage/api-storage-cleanup.test.ts -1221 tests/storage/storage-cleanup.test.ts -6567 tests/storage/storage-mutation-race.test.ts -116 tests/storage/storage-policy-config-race.test.ts -2167 tests/storage/storage-policy-job-responsive.test.ts -160 tests/storage/storage-policy.test.ts -96 tests/storage/storage-restore-job-errors.test.ts -5122 tests/storage/storage-restore-job-responsive.test.ts -175 tests/storage/storage-scanner.test.ts -1048 tests/storage/storage-worker-lifecycle.test.ts -186 tests/storage/storage-worker-os-join-settle.test.ts -4270 tests/storage/storage-worker-teardown-isolate.test.ts -285 tests/test-layout-tooling.test.ts -202 tests/test-layout.test.ts -92 tests/update/update-badge.test.ts -5263 tests/update/update-bun-ownership-lease.test.ts -99 tests/update/update-desktop-owner.test.ts -861 tests/update/update-job.test.ts -106 tests/update/update-notify.test.ts -274 tests/update/update-npm-cache-preflight.test.ts -82 tests/update/update-npm-invocation.test.ts -172 tests/update/update-pnpm.test.ts -1319 tests/update/update-stop-classification.test.ts -3647 tests/update/update-stop-first.test.ts -304 tests/update/update-transactional.test.ts -89 tests/update/update-tray-handoff.test.ts -259 tests/update/update-tree-ownership.test.ts -129 tests/usage/cache-diagnostic.test.ts -199 tests/usage/cost-cap-unknown-evidence.test.ts -122 tests/usage/cost-scoring.test.ts -120 tests/usage/key-attribution.test.ts -399 tests/usage/quota-401-recovery-runtime.test.ts -78 tests/usage/quota-401-recovery.test.ts -114 tests/usage/quota-reset-account-key.test.ts -402 tests/usage/quota-reset-core-boundary.test.ts -92 tests/usage/quota-reset-detector.test.ts -84 tests/usage/quota-reset-notify-config.test.ts -440 tests/usage/quota-reset-notify.test.ts -626 tests/usage/quota-reset-observation.test.ts -1690 tests/usage/quota-reset-seen-store.test.ts -170 tests/usage/quota-scoring.test.ts -1388 tests/usage/request-decompress.test.ts -184 tests/usage/request-evidence.test.ts -1073 tests/usage/request-history-index.test.ts -154 tests/usage/request-log-conversation.test.ts -116 tests/usage/request-log-estimate-cap.test.ts -317 tests/usage/request-log-nonstream.test.ts -123 tests/usage/request-log-served-model.test.ts -322 tests/usage/request-log.test.ts -182 tests/usage/request-outcome-agreement.test.ts -248 tests/usage/request-pacing.test.ts -264 tests/usage/usage-aggregate-cache.test.ts -92 tests/usage/usage-anthropic-fast-pricing.test.ts -85 tests/usage/usage-attempt-delivery.test.ts -115 tests/usage/usage-cost.test.ts -253 tests/usage/usage-debug.test.ts -88 tests/usage/usage-failure-fingerprint.test.ts -147 tests/usage/usage-failure-persistence.test.ts -82 tests/usage/usage-failure-projection.test.ts -138 tests/usage/usage-ledger-retention.test.ts -301 tests/usage/usage-ledger-scanner.test.ts -188 tests/usage/usage-log-ws-stage.test.ts -2374 tests/usage/usage-log.test.ts -97 tests/usage/usage-provider-label.test.ts -332 tests/usage/usage-shape-extraction.test.ts -105 tests/usage/usage-spend-cache-provenance.test.ts -222 tests/usage/usage-summary.test.ts -80 tests/usage/usage-surfaces.test.ts -185 tests/usage/usage-time-range.test.ts -96 tests/usage/usage-timeline.test.ts -143 tests/usage/user-cost-overlay-coderabbit-regressions.test.ts -1767 tests/usage/user-cost-overlay-live-reconcile.test.ts -127 tests/usage/user-cost-overlay-provider-delete.test.ts -108 tests/videos/fulfill-video.test.ts -113 tests/videos/plan-video.test.ts -118 tests/videos/xai-video-client.test.ts -151 tests/vision/sidecar-abort.test.ts -143 tests/vision/sidecar-auth.test.ts -139 tests/vision/sidecar-candidates.test.ts -200 tests/vision/sidecar-settings-vision-controls.test.ts -216 tests/vision/sidecar-settings-vision-filter.test.ts -250 tests/vision/sidecar-settings-web-search-gate.test.ts -221 tests/vision/sidecar-settings-web-search-stream.test.ts -85 tests/vision/sidecar-tracker.test.ts -183 tests/vision/vision-anthropic.test.ts -253 tests/vision/vision-backend-union.test.ts -152 tests/vision/vision-cache.test.ts -124 tests/vision/vision-custom-row-precedence.test.ts -105 tests/vision/vision-eligibility.test.ts -128 tests/vision/vision-fail-closed.test.ts -181 tests/vision/vision-reasoning-contract.test.ts -395 tests/vision/vision-routed.test.ts -873 tests/vision/vision-sidecar-e2e.test.ts -121 tests/vision/vision-text-only-predicate.test.ts -86 tests/web-search/format-result.test.ts -129 tests/web-search/web-search-anthropic.test.ts -309 tests/web-search/web-search-backend-union.test.ts -133 tests/web-search/web-search-bridge-replay.test.ts -132 tests/web-search/web-search-candidates.test.ts -96 tests/web-search/web-search-parse.test.ts -490 tests/web-search/web-search-passthrough-bridge.test.ts -2620 tests/web-search/web-search-progress-stream.test.ts -1644 tests/web-search/web-search-retry-heartbeat.test.ts -7323 tests/web-search/web-search-sidecar-429.test.ts -89 tests/web-search/web-search-sources.test.ts -449 tests/web-search/web-search-timeout-contract.test.ts -133 tests/web-search/web-search-timeout-plan.test.ts -5412 tests/web-search/web-search.test.ts -187 tests/windows/tray-proxy-deadline.test.ts -98 tests/windows/tray-proxy.test.ts -92 tests/windows/win-exec.test.ts -186 tests/windows/win-paths.test.ts -99 tests/windows/windows-acl-start-cost.test.ts -185 tests/windows/windows-atomic-replace.test.ts -95 tests/windows/windows-deploy-close-regressions.test.ts -371 tests/windows/windows-elevation-spawn.test.ts -98 tests/windows/windows-elevation.test.ts -139 tests/windows/windows-popup-fix.test.ts -163 tests/windows/windows-scheduler-install-verification.test.ts -380 tests/windows/windows-secret-acl.test.ts -159 tests/windows/windows-service-mutation-lock.test.ts -87 tests/windows/windows-service-wrappers.test.ts -175 tests/windows/windows-text-decoding.test.ts -88 tests/windows/windows-tray-restart-hardening.test.ts -98 tests/windows/windows-tray-run-limit.test.ts -245 tests/windows/windows-tray.test.ts -84 tests/windows/windows-user-principal-nonascii.test.ts -181 tests/windows/windows-user-principal.test.ts -75 tests/windows/winsw-stop-hardening.test.ts +82 tests/server/provider-egress-management-validation.test.ts +423 tests/server/provider-send-path-import.test.ts +187 tests/server/proxy-env-macos.test.ts +130 tests/server/proxy-env.test.ts +88 tests/server/proxy-liveness-package-tree-fence.test.ts +114 tests/server/proxy-liveness.test.ts +817 tests/server/relay-eager.test.ts +1495 tests/server/replay-refusal-parity.test.ts +325 tests/server/reserve-claude-policy.test.ts +2007 tests/server/reserve-ingress.test.ts +942 tests/server/response-log-inspection.test.ts +524 tests/server/response-model-identity.test.ts +191 tests/server/restart-replacement.test.ts +229 tests/server/retry-after-429.test.ts +95 tests/server/retry-delay-hardening.test.ts +207 tests/server/route-decision-trace.test.ts +443 tests/server/server-403-permission-e2e.test.ts +226 tests/server/server-agent-task-recovery-replay.test.ts +117 tests/server/server-antigravity-verify-replay-scope.test.ts +768 tests/server/server-antigravity-web-search-validation.test.ts +239 tests/server/server-auth-localhost-bind.test.ts +1763 tests/server/server-auth-scoped-quota.test.ts +22034 tests/server/server-auth.test.ts +1028 tests/server/server-background-lifecycle.test.ts +263 tests/server/server-cli-key-probe-admission.test.ts +87 tests/server/server-clickjacking-headers.test.ts +216 tests/server/server-combo-cooldown-fallback.test.ts +967 tests/server/server-combo-cooldown-recording.test.ts +8793 tests/server/server-combo-failover-e2e.test.ts +97 tests/server/server-combo-held-response.test.ts +917 tests/server/server-combo-plan-model-refusal.test.ts +3667 tests/server/server-combo-reasoning-replay-eligibility.test.ts +302 tests/server/server-combo-zero-output-failover.test.ts +3671 tests/server/server-google-antigravity-oauth-401-replay.test.ts +396 tests/server/server-google-antigravity-oauth-429-budget.test.ts +84 tests/server/server-gui-bundle-freshness.test.ts +160 tests/server/server-images-bodyless-content-length.test.ts +290 tests/server/server-images-pool-admission.test.ts +4756 tests/server/server-images.test.ts +354 tests/server/server-jev-combo-e2e.test.ts +330 tests/server/server-jev-model-decision-e2e.test.ts +4296 tests/server/server-key-failover-e2e.test.ts +1282 tests/server/server-kiro-completion-e2e.test.ts +387 tests/server/server-kiro-device-login.test.ts +730 tests/server/server-kiro-oauth-401-replay.test.ts +1608 tests/server/server-kiro-refusal-e2e.test.ts +377 tests/server/server-live-existing-call.test.ts +179 tests/server/server-live-frame-log.test.ts +862 tests/server/server-live-realtime-fixtures.test.ts +6004 tests/server/server-live.test.ts +214 tests/server/server-loopback-host-gate.test.ts +1930 tests/server/server-management-auth.test.ts +462 tests/server/server-management-log-admission.test.ts +830 tests/server/server-new-model-policy-arrival.test.ts +390 tests/server/server-opencode-go-goal-streaming.test.ts +1588 tests/server/server-rate-limit-retry-e2e.test.ts +1202 tests/server/server-request-body-size.test.ts +2449 tests/server/server-search.test.ts +558 tests/server/server-startup-reconcile-resilience.test.ts +507 tests/server/server-stop-config-hardening.test.ts +572 tests/server/server-xai-chat-reasoning-streaming.test.ts +400 tests/server/server-xai-header-parity.test.ts +834 tests/server/server-xai-oauth-401-replay.test.ts +585 tests/server/server-xai-responses-streaming.test.ts +185 tests/server/session-affinity.test.ts +282 tests/server/session-lane-recall-harness.test.ts +138 tests/server/sidebar-routes.test.ts +131 tests/server/sidebar-star-state.test.ts +121 tests/server/spend-instrumentation-log.test.ts +168 tests/server/spend-ledger-lifecycle.test.ts +1047 tests/server/spend-ledger-owner-startup.test.ts +145 tests/server/startup-action-control-elevation.test.ts +286 tests/server/startup-action-control.test.ts +11724 tests/server/startup-health-packaged-probe.test.ts +179 tests/server/startup-prompt.test.ts +130 tests/server/stream-aborted-marker.test.ts +310 tests/server/subagent-model-force-api.test.ts +1038 tests/server/system-env.test.ts +149 tests/server/system-restart-admission.test.ts +184 tests/server/system-restart.test.ts +160 tests/server/system-routes.test.ts +4256 tests/server/terminal-guard-server.test.ts +155 tests/server/terminal-guard.test.ts +258 tests/server/update-async-routes.test.ts +150 tests/server/upstream-connect-error.test.ts +88 tests/server/upstream-http-version.test.ts +756 tests/server/v1-hub-state.test.ts +414 tests/server/v2-agent-message-failfast.test.ts +183 tests/service/autostart-health.test.ts +3486 tests/service/container-bootstrap.test.ts +207 tests/service/crash-guard.test.ts +25302 tests/service/doctor-codex-envkey-readiness.test.ts +185 tests/service/doctor-oauth.test.ts +117 tests/service/doctor-provider-apikey.test.ts +81 tests/service/doctor-service-memory-contract.test.ts +138 tests/service/init-backup-cleanup.test.ts +7427 tests/service/init-eof.test.ts +421 tests/service/launchd-repair.test.ts +205 tests/service/live-service-manager-guard.test.ts +114 tests/service/managing-cli.test.ts +133 tests/service/ownership-mutation-lease-diagnostics.test.ts +109 tests/service/process-state.test.ts +269 tests/service/service-auth-qualified-localhost.test.ts +155 tests/service/service-child-ownership.test.ts +210 tests/service/service-claim.test.ts +96 tests/service/service-desktop-runtime-preflight.test.ts +159 tests/service/service-desktop-startup-health.test.ts +270 tests/service/service-desktop-startup-linux.test.ts +403 tests/service/service-desktop-startup.test.ts +171 tests/service/service-handled-signal-exit.test.ts +124 tests/service/service-health-sentinel-launch.test.ts +122 tests/service/service-health-sentinel-policy.test.ts +8406 tests/service/service-health-sentinel-process.test.ts +484 tests/service/service-health-sentinel-runner.test.ts +195 tests/service/service-multishell-path.test.ts +102 tests/service/service-ownership-compatibility.test.ts +421 tests/service/service-ownership-handover.test.ts +501 tests/service/service-ownership-state.test.ts +210 tests/service/service-probe-docker.test.ts +201 tests/service/service-runtime-preflight.test.ts +262 tests/service/service-secrets.test.ts +332 tests/service/service-sqlite-home.test.ts +102 tests/service/service-start-environment.test.ts +168 tests/service/service-stop-verification.test.ts +446 tests/service/service-tier-capability.test.ts +343 tests/service/service-wsl-home-ownership.test.ts +1109 tests/service/service.test.ts +320 tests/service/shutdown-drain.test.ts +4911 tests/service/shutdown-launcher.test.ts +172 tests/service/stale-state-purge.test.ts +187 tests/service/standalone-service.test.ts +3613 tests/service/stop-deferred-teardown.test.ts +110 tests/service/systemd-install-cleanup-hardening.test.ts +147 tests/service/winsw.test.ts +3612 tests/storage/api-storage-cleanup.test.ts +2220 tests/storage/api-storage-policy-already-running.test.ts +1689 tests/storage/api-storage-policy-mutation-busy.test.ts +955 tests/storage/api-storage-policy-put-race.test.ts +1587 tests/storage/api-storage-policy-run.test.ts +692 tests/storage/api-storage-policy.test.ts +557 tests/storage/api-storage.test.ts +1057 tests/storage/storage-cleanup.test.ts +6703 tests/storage/storage-mutation-race.test.ts +209 tests/storage/storage-policy-config-race.test.ts +2235 tests/storage/storage-policy-job-responsive.test.ts +163 tests/storage/storage-policy.test.ts +82 tests/storage/storage-restore-job-errors.test.ts +5375 tests/storage/storage-restore-job-responsive.test.ts +255 tests/storage/storage-scanner.test.ts +1065 tests/storage/storage-worker-lifecycle.test.ts +80 tests/storage/storage-worker-os-join-settle.test.ts +4492 tests/storage/storage-worker-teardown-isolate.test.ts +430 tests/test-layout-tooling.test.ts +176 tests/test-layout.test.ts +154 tests/update/pnpm-command-isolation.test.ts +104 tests/update/update-badge.test.ts +5833 tests/update/update-bun-ownership-lease.test.ts +77 tests/update/update-desktop-badge.test.ts +91 tests/update/update-desktop-owner.test.ts +679 tests/update/update-job.test.ts +154 tests/update/update-mise-launcher-target.test.ts +86 tests/update/update-mise-node-runtime.test.ts +383 tests/update/update-mise.test.ts +119 tests/update/update-notify.test.ts +562 tests/update/update-npm-cache-preflight.test.ts +97 tests/update/update-npm-invocation.test.ts +188 tests/update/update-pnpm.test.ts +276 tests/update/update-refresh.test.ts +36456 tests/update/update-restart-lease.test.ts +1793 tests/update/update-stop-classification.test.ts +4277 tests/update/update-stop-first.test.ts +611 tests/update/update-transactional-leftovers.test.ts +288 tests/update/update-transactional.test.ts +201 tests/update/update-tray-handoff.test.ts +209 tests/update/update-tree-ownership.test.ts +91 tests/update/update-worker-launch.test.ts +103 tests/usage/anthropic2-usage-attribution.test.ts +126 tests/usage/cache-diagnostic.test.ts +251 tests/usage/cost-cap-unknown-evidence.test.ts +97 tests/usage/cost-scoring.test.ts +103 tests/usage/jev-stats.test.ts +132 tests/usage/key-attribution.test.ts +275 tests/usage/quota-401-recovery-runtime.test.ts +113 tests/usage/quota-401-recovery.test.ts +133 tests/usage/quota-reset-account-key.test.ts +463 tests/usage/quota-reset-core-boundary.test.ts +95 tests/usage/quota-reset-detector.test.ts +96 tests/usage/quota-reset-notify-config.test.ts +381 tests/usage/quota-reset-notify.test.ts +690 tests/usage/quota-reset-observation.test.ts +1805 tests/usage/quota-reset-seen-store.test.ts +164 tests/usage/quota-scoring.test.ts +2236 tests/usage/request-decompress.test.ts +89 tests/usage/request-evidence.test.ts +1005 tests/usage/request-history-index.test.ts +103 tests/usage/request-log-attempt-identity.test.ts +117 tests/usage/request-log-conversation.test.ts +179 tests/usage/request-log-estimate-cap.test.ts +199 tests/usage/request-log-generation-window.test.ts +91 tests/usage/request-log-jev.test.ts +97 tests/usage/request-log-model-filter.test.ts +266 tests/usage/request-log-nonstream.test.ts +139 tests/usage/request-log-protocol-trace.test.ts +100 tests/usage/request-log-served-model.test.ts +383 tests/usage/request-log.test.ts +183 tests/usage/request-outcome-agreement.test.ts +271 tests/usage/request-pacing.test.ts +19093 tests/usage/upstream-failure-diagnostics.test.ts +232 tests/usage/usage-aggregate-cache.test.ts +64 tests/usage/usage-anthropic-fast-pricing.test.ts +71 tests/usage/usage-antigravity-55.test.ts +163 tests/usage/usage-attempt-delivery.test.ts +149 tests/usage/usage-cost.test.ts +91 tests/usage/usage-cursor-fast-pricing.test.ts +243 tests/usage/usage-debug.test.ts +87 tests/usage/usage-failure-fingerprint.test.ts +115 tests/usage/usage-failure-persistence.test.ts +68 tests/usage/usage-failure-projection.test.ts +104 tests/usage/usage-haiku-5-5-pricing.test.ts +178 tests/usage/usage-ledger-retention.test.ts +323 tests/usage/usage-ledger-scanner.test.ts +97 tests/usage/usage-log-ws-stage.test.ts +3085 tests/usage/usage-log.test.ts +76 tests/usage/usage-provider-label.test.ts +336 tests/usage/usage-shape-extraction.test.ts +150 tests/usage/usage-snapshot-digest-reuse.test.ts +114 tests/usage/usage-spend-cache-provenance.test.ts +239 tests/usage/usage-summary.test.ts +102 tests/usage/usage-surfaces.test.ts +91 tests/usage/usage-throughput.test.ts +79 tests/usage/usage-time-range.test.ts +89 tests/usage/usage-timeline.test.ts +124 tests/usage/user-cost-overlay-coderabbit-regressions.test.ts +1663 tests/usage/user-cost-overlay-live-reconcile.test.ts +154 tests/usage/user-cost-overlay-provider-delete.test.ts +128 tests/videos/fulfill-video.test.ts +101 tests/videos/plan-video.test.ts +122 tests/videos/xai-video-client.test.ts +168 tests/vision/sidecar-abort.test.ts +139 tests/vision/sidecar-auth.test.ts +185 tests/vision/sidecar-candidates.test.ts +308 tests/vision/sidecar-settings-vision-controls.test.ts +253 tests/vision/sidecar-settings-vision-filter.test.ts +390 tests/vision/sidecar-settings-web-search-gate.test.ts +208 tests/vision/sidecar-settings-web-search-off.test.ts +229 tests/vision/sidecar-settings-web-search-stream.test.ts +138 tests/vision/sidecar-tracker.test.ts +314 tests/vision/vision-anthropic-instance-sidecar.test.ts +175 tests/vision/vision-anthropic.test.ts +334 tests/vision/vision-backend-union.test.ts +144 tests/vision/vision-cache.test.ts +100 tests/vision/vision-custom-row-precedence.test.ts +99 tests/vision/vision-eligibility.test.ts +214 tests/vision/vision-fail-closed.test.ts +184 tests/vision/vision-reasoning-contract.test.ts +597 tests/vision/vision-routed.test.ts +1321 tests/vision/vision-sidecar-e2e.test.ts +104 tests/vision/vision-text-only-predicate.test.ts +145 tests/web-search/devin-web-search.test.ts +99 tests/web-search/format-result.test.ts +193 tests/web-search/web-search-anthropic-instance.test.ts +166 tests/web-search/web-search-anthropic.test.ts +293 tests/web-search/web-search-backend-union.test.ts +87 tests/web-search/web-search-bridge-replay.test.ts +124 tests/web-search/web-search-candidates.test.ts +123 tests/web-search/web-search-forced-declaration.test.ts +2580 tests/web-search/web-search-pacing-deadline.test.ts +214 tests/web-search/web-search-parse.test.ts +463 tests/web-search/web-search-passthrough-bridge.test.ts +2855 tests/web-search/web-search-progress-stream.test.ts +115 tests/web-search/web-search-recovery-kind.test.ts +1620 tests/web-search/web-search-retry-heartbeat.test.ts +154 tests/web-search/web-search-run-turn-loop.test.ts +7844 tests/web-search/web-search-sidecar-429.test.ts +101 tests/web-search/web-search-sources.test.ts +567 tests/web-search/web-search-timeout-contract.test.ts +114 tests/web-search/web-search-timeout-plan.test.ts +5431 tests/web-search/web-search.test.ts +74 tests/windows/tray-proxy-deadline.test.ts +2388 tests/windows/tray-proxy.test.ts +94 tests/windows/win-exec.test.ts +92 tests/windows/win-paths.test.ts +85 tests/windows/windows-acl-start-cost.test.ts +111 tests/windows/windows-atomic-replace.test.ts +77 tests/windows/windows-deploy-close-regressions.test.ts +367 tests/windows/windows-elevation-spawn.test.ts +88 tests/windows/windows-elevation.test.ts +108 tests/windows/windows-guarded-manager.test.ts +150 tests/windows/windows-popup-fix.test.ts +89 tests/windows/windows-process-priority.test.ts +113 tests/windows/windows-scheduler-install-verification.test.ts +103 tests/windows/windows-secret-acl-removal-flight.test.ts +392 tests/windows/windows-secret-acl.test.ts +234 tests/windows/windows-service-mutation-lock.test.ts +142 tests/windows/windows-service-wrappers.test.ts +88 tests/windows/windows-text-decoding.test.ts +101 tests/windows/windows-tray-restart-hardening.test.ts +145 tests/windows/windows-tray-run-limit.test.ts +233 tests/windows/windows-tray.test.ts +98 tests/windows/windows-user-principal-nonascii.test.ts +81 tests/windows/windows-user-principal.test.ts +206 tests/windows/winsw-stop-hardening.test.ts diff --git a/scripts/ci/test-durations.windows.tsv b/scripts/ci/test-durations.windows.tsv new file mode 100644 index 00000000000..e216dd46cf5 --- /dev/null +++ b/scripts/ci/test-durations.windows.tsv @@ -0,0 +1,2246 @@ +# Per-file Bun test durations in milliseconds, read from hosted CI job logs. +# Consumed by scripts/ci/run-bun-test-batches.sh to balance shards; a file without a row weighs the median. +# Regenerate with scripts/ci/test-durations.ts (usage in its header); do not edit by hand. +# Source: run 38250055508 (dev dc45bb2ca9, lane=all), windows 1/9-9/9 +7665 tests/adapters/abort-race.test.ts +3710 tests/adapters/adapter-buffered-tool-conformance.test.ts +2560 tests/adapters/adapter-error-inline.test.ts +6010 tests/adapters/adapter-inner-send-budget-wiring.test.ts +4077 tests/adapters/adapter-inner-send-budget.test.ts +2297 tests/adapters/adapter-input-media-guard.test.ts +143 tests/adapters/adapter-registry-authority.test.ts +2536 tests/adapters/adapter-tool-conformance.test.ts +127 tests/adapters/adapter-usage.test.ts +15352 tests/adapters/anthropic/anthropic-429-policy.test.ts +94 tests/adapters/anthropic/anthropic-account-metadata.test.ts +6105 tests/adapters/anthropic/anthropic-account-pause-outbound.test.ts +18837 tests/adapters/anthropic/anthropic-account-pause.test.ts +33903 tests/adapters/anthropic/anthropic-account-pool.test.ts +31891 tests/adapters/anthropic/anthropic-account-threshold.test.ts +128 tests/adapters/anthropic/anthropic-agentrouter-language-framing.test.ts +144 tests/adapters/anthropic/anthropic-baseurl-override.test.ts +176 tests/adapters/anthropic/anthropic-beta-allowlist.test.ts +143 tests/adapters/anthropic/anthropic-client-identity.test.ts +2215 tests/adapters/anthropic/anthropic-combo-account-cooldown.test.ts +2667 tests/adapters/anthropic/anthropic-compatible-stream.test.ts +2805 tests/adapters/anthropic/anthropic-cooldown-recovery.test.ts +143 tests/adapters/anthropic/anthropic-empty-content.test.ts +192 tests/adapters/anthropic/anthropic-eof-tolerance.test.ts +2314 tests/adapters/anthropic/anthropic-error-body.test.ts +189 tests/adapters/anthropic/anthropic-error-stop-reason.test.ts +4020 tests/adapters/anthropic/anthropic-fast-opt-in.test.ts +175 tests/adapters/anthropic/anthropic-fast-speed.test.ts +145 tests/adapters/anthropic/anthropic-haiku-5-5-contract.test.ts +144 tests/adapters/anthropic/anthropic-hardening.test.ts +219 tests/adapters/anthropic/anthropic-image-guard.test.ts +3641 tests/adapters/anthropic/anthropic-image-normalize.test.ts +4117 tests/adapters/anthropic/anthropic-image-retry-e2e.test.ts +222 tests/adapters/anthropic/anthropic-image-retry.test.ts +24634 tests/adapters/anthropic/anthropic-instance-isolation.test.ts +39287 tests/adapters/anthropic/anthropic-instance-pool-parity.test.ts +46273 tests/adapters/anthropic/anthropic-instance-quota.test.ts +92152 tests/adapters/anthropic/anthropic-instance-recovery.test.ts +174 tests/adapters/anthropic/anthropic-message-stop-termination.test.ts +129 tests/adapters/anthropic/anthropic-messages-passthrough-oauth.test.ts +142 tests/adapters/anthropic/anthropic-messages-passthrough.test.ts +28681 tests/adapters/anthropic/anthropic-model-routes.test.ts +12323 tests/adapters/anthropic/anthropic-model-weekly-admission.test.ts +159 tests/adapters/anthropic/anthropic-opaque-strip.test.ts +191 tests/adapters/anthropic/anthropic-output-maxima.test.ts +143 tests/adapters/anthropic/anthropic-parallel-tool-disable.test.ts +288 tests/adapters/anthropic/anthropic-pool-toggle-copy.test.ts +19193 tests/adapters/anthropic/anthropic-quota-dispatch.test.ts +8049 tests/adapters/anthropic/anthropic-ratelimit-headers.test.ts +207 tests/adapters/anthropic/anthropic-reasoning.test.ts +1821 tests/adapters/anthropic/anthropic-reset-grants.test.ts +12330 tests/adapters/anthropic/anthropic-revoked-token-boundaries.test.ts +5885 tests/adapters/anthropic/anthropic-revoked-token-continuation.test.ts +11924 tests/adapters/anthropic/anthropic-revoked-token-sidecars.test.ts +48338 tests/adapters/anthropic/anthropic-revoked-token.test.ts +855 tests/adapters/anthropic/anthropic-sidecar-account-failover.test.ts +160 tests/adapters/anthropic/anthropic-sonnet-5-5-contract.test.ts +130 tests/adapters/anthropic/anthropic-stream-hardening.test.ts +111 tests/adapters/anthropic/anthropic-tail-guard.test.ts +204 tests/adapters/anthropic/anthropic-thinking-signature.test.ts +128 tests/adapters/anthropic/anthropic-thinking-usage.test.ts +117 tests/adapters/anthropic/anthropic-tool-call-id.test.ts +176 tests/adapters/anthropic/anthropic-tool-declaration-constraints.test.ts +196 tests/adapters/anthropic/anthropic-tool-schema.test.ts +10713 tests/adapters/anthropic/anthropic2-fast-parity.test.ts +173 tests/adapters/bridge-legacy-shell-normalization.test.ts +3786 tests/adapters/bridge-lifecycle.test.ts +779 tests/adapters/bridge-nonstreaming-terminal.test.ts +369 tests/adapters/bridge-raw-reasoning-hidden.test.ts +343 tests/adapters/bridge-reasoning-replay-batch.test.ts +206 tests/adapters/bridge-stall.test.ts +127 tests/adapters/bridge-terminal-singleness.test.ts +479 tests/adapters/bridge.test.ts +190 tests/adapters/buffered-response-shape-guards.test.ts +309 tests/adapters/coding-agent-tool-result-images.test.ts +3031 tests/adapters/empty-tool-output-annotation.test.ts +237 tests/adapters/exec-tool-result-normalize.test.ts +156 tests/adapters/google/antigravity-baseurl-override.test.ts +445 tests/adapters/google/antigravity-discovered-families.test.ts +143 tests/adapters/google/antigravity-refusal.test.ts +418 tests/adapters/google/antigravity-static-catalog.test.ts +950 tests/adapters/google/gcp-adc.test.ts +176 tests/adapters/google/gemini-37-flash-migration.test.ts +191 tests/adapters/google/gemini-web-search.test.ts +223 tests/adapters/google/google-adapter.test.ts +4626 tests/adapters/google/google-antigravity-oauth.test.ts +3757 tests/adapters/google/google-antigravity-replay.test.ts +184 tests/adapters/google/google-antigravity-wire.test.ts +94 tests/adapters/google/google-buffered-stop-reason.test.ts +95 tests/adapters/google/google-claude-prefill-guard.test.ts +113 tests/adapters/google/google-empty-content.test.ts +332 tests/adapters/google/google-errors.test.ts +161 tests/adapters/google/google-hardening.test.ts +781 tests/adapters/google/google-models-listing.test.ts +130 tests/adapters/google/google-output-clamp.test.ts +143 tests/adapters/google/google-provider-metadata-roundtrip.test.ts +508 tests/adapters/google/google-signature-history-roundtrip.test.ts +112 tests/adapters/google/google-strict-tool-validated-mode.test.ts +111 tests/adapters/google/google-structured-output.test.ts +138 tests/adapters/google/google-tool-result-adjacency.test.ts +156 tests/adapters/google/google-tool-schema-contract.test.ts +191 tests/adapters/google/google-tool-schema.test.ts +3304 tests/adapters/google/google-vertex-http.test.ts +126 tests/adapters/google/google-vertex-stream.test.ts +97 tests/adapters/google/google-vertex-thought-signature.test.ts +127 tests/adapters/google/google-wire-compiler.test.ts +113 tests/adapters/google/google-wire-shape.test.ts +174 tests/adapters/google/vertex-catalog.test.ts +518 tests/adapters/identity-neutralize.test.ts +931 tests/adapters/identity-subagent.test.ts +13111 tests/adapters/key-failover.test.ts +206 tests/adapters/openai/inline-think-boundaries.test.ts +10329 tests/adapters/openai/openai-api-virtual-models.test.ts +255 tests/adapters/openai/openai-chat-bounded-tool-names.test.ts +159 tests/adapters/openai/openai-chat-dangling-toolcalls.test.ts +159 tests/adapters/openai/openai-chat-developer-position.test.ts +174 tests/adapters/openai/openai-chat-eof.test.ts +158 tests/adapters/openai/openai-chat-glm-summary.test.ts +336 tests/adapters/openai/openai-chat-hardening.test.ts +5481 tests/adapters/openai/openai-chat-image-normalization.test.ts +140 tests/adapters/openai/openai-chat-inline-think-tags.test.ts +142 tests/adapters/openai/openai-chat-invalid-tool-call-diagnostics.test.ts +223 tests/adapters/openai/openai-chat-model-suffix.test.ts +1627 tests/adapters/openai/openai-chat-native-policy.test.ts +191 tests/adapters/openai/openai-chat-parallel-stream.test.ts +123 tests/adapters/openai/openai-chat-path-override.test.ts +97 tests/adapters/openai/openai-chat-qwen38-leading-system.test.ts +96 tests/adapters/openai/openai-chat-reasoning-wire-policy.test.ts +208 tests/adapters/openai/openai-chat-sanitization-review-regressions.test.ts +144 tests/adapters/openai/openai-chat-serialized-tool-call-content.test.ts +285 tests/adapters/openai/openai-chat-serialized-tool-call-hold-bound.test.ts +269 tests/adapters/openai/openai-chat-serialized-tool-call-scaling.test.ts +190 tests/adapters/openai/openai-chat-serialized-tool-call-think.test.ts +286 tests/adapters/openai/openai-chat-system-order.test.ts +143 tests/adapters/openai/openai-chat-tool-call-id-remint.test.ts +157 tests/adapters/openai/openai-chat-tool-result-images.test.ts +111 tests/adapters/openai/openai-chat-url.test.ts +143 tests/adapters/openai/openai-chat-video-part.test.ts +192 tests/adapters/openai/openai-empty-assistant-history.test.ts +6501 tests/adapters/openai/openai-provider-option-e2e.test.ts +96 tests/adapters/openai/openai-provider-option-migration.test.ts +806 tests/adapters/openai/openai-provider-option-startup.test.ts +318 tests/adapters/openai/openai-provider-option-tooling.test.ts +125 tests/adapters/openai/openai-provider-option.test.ts +311 tests/adapters/physical-send.test.ts +218 tests/adapters/reasoning-replay-identity.test.ts +2406 tests/adapters/reasoning-replay-robustness.test.ts +448 tests/adapters/routed-agent-messages.test.ts +156 tests/adapters/run-turn-queue.test.ts +1702 tests/adapters/terminal-continuation-owner-rotation.test.ts +98 tests/adapters/tool-argument-integers.test.ts +313 tests/adapters/tool-catalog-nudge.test.ts +335 tests/adapters/tool-choice-performance.test.ts +129 tests/adapters/tool-envelope-echo-whole-line.test.ts +359 tests/adapters/translator-budget-lifetime.test.ts +3584 tests/adapters/translator-budget.test.ts +127 tests/adapters/upstream-http-error.test.ts +127 tests/ci-workflows/assert-mergeable-review.test.ts +368 tests/ci-workflows/brand-favicons.test.ts +364 tests/ci-workflows/build-desktop-icon-set.test.ts +501 tests/ci-workflows/build-release-changelog.test.ts +604 tests/ci-workflows/bump-dev-version.test.ts +790 tests/ci-workflows/bun-runtime.test.ts +144 tests/ci-workflows/ci-bun-crash-classifier.test.ts +127 tests/ci-workflows/ci-concurrency-groups.test.ts +143 tests/ci-workflows/ci-crash-disposition.test.ts +3042 tests/ci-workflows/ci-gui-typecheck-gate.test.ts +111 tests/ci-workflows/ci-privacy-gate.test.ts +567 tests/ci-workflows/ci-review-lanes.test.ts +1411 tests/ci-workflows/ci-scope-gaps.test.ts +171 tests/ci-workflows/ci-scope-reduction.test.ts +159 tests/ci-workflows/ci-shard-balance.test.ts +94 tests/ci-workflows/ci-structure-gate.test.ts +1226 tests/ci-workflows/ci-workflows.test.ts +159 tests/ci-workflows/cleanup-orphaned-workflows.test.ts +142 tests/ci-workflows/closed-pr-branch-cleanup.test.ts +7676 tests/ci-workflows/cold-spawn-warmup.test.ts +435 tests/ci-workflows/compatibility-version.test.ts +147 tests/ci-workflows/docs-429-failover-claims.test.ts +144 tests/ci-workflows/docs-bun-source-requirement.test.ts +161 tests/ci-workflows/docs-developer-role-policy.test.ts +343 tests/ci-workflows/docs-gjc-reasoning-export.test.ts +127 tests/ci-workflows/docs-gui-screenshot-policy.test.ts +380 tests/ci-workflows/docs-link-targets.test.ts +159 tests/ci-workflows/docs-provider-billing-claims.test.ts +186 tests/ci-workflows/docs-provider-discovery-limits.test.ts +163 tests/ci-workflows/docs-provider-preset-counts.test.ts +272 tests/ci-workflows/docs-readme-memory-inventory.test.ts +144 tests/ci-workflows/docs-readme-translation-parity.test.ts +162 tests/ci-workflows/docs-remote-hub-claims.test.ts +304 tests/ci-workflows/dsh-path-contract.test.ts +647 tests/ci-workflows/dsh-rc6-compat-script.test.ts +1103 tests/ci-workflows/dsh-writer-lock.test.ts +300 tests/ci-workflows/exhaustive-deps-suppression.test.ts +410 tests/ci-workflows/file-size-ratchet.test.ts +332 tests/ci-workflows/fixture-dir-uniqueness.test.ts +142 tests/ci-workflows/fixture-teardown-helper.test.ts +175 tests/ci-workflows/install-scripts.test.ts +177 tests/ci-workflows/installed-gate-drivers.test.ts +155 tests/ci-workflows/keyring-smoke.test.ts +137 tests/ci-workflows/linux-desktop-packaged-ci.test.ts +95 tests/ci-workflows/linux-desktop-packaged-e2e.test.ts +162 tests/ci-workflows/macos-serial-lanes.test.ts +3855 tests/ci-workflows/package-tree-fenced-restart.test.ts +4263 tests/ci-workflows/package-tree-integrity.test.ts +334 tests/ci-workflows/package-tree-restart-ownership.test.ts +356 tests/ci-workflows/package-tree-retarget.test.ts +158 tests/ci-workflows/pr-readiness-reattest.test.ts +122 tests/ci-workflows/privacy-scan-asset-names.test.ts +154 tests/ci-workflows/privacy-scan-meta-key.test.ts +202 tests/ci-workflows/privacy-scan-ssh-endpoint.test.ts +442 tests/ci-workflows/release-desktop-scripts.test.ts +40759 tests/ci-workflows/release-helper.test.ts +361 tests/ci-workflows/release-notes.test.ts +144 tests/ci-workflows/release-outcome-report.test.ts +775 tests/ci-workflows/release-pipeline-contract.test.ts +159 tests/ci-workflows/release-preflight.test.ts +146 tests/ci-workflows/release-resume-identity.test.ts +173 tests/ci-workflows/release-version-line.test.ts +606 tests/ci-workflows/release-version-sources.test.ts +617 tests/ci-workflows/repo-hygiene.test.ts +708 tests/ci-workflows/repo-import-resolution.test.ts +1823 tests/ci-workflows/setup-hooks.test.ts +1221 tests/ci-workflows/skill-ocx-generated.test.ts +428 tests/ci-workflows/skill-ocx-workflows.test.ts +155 tests/ci-workflows/skill-ocx.test.ts +2809 tests/ci-workflows/structure-ssot.test.ts +59749 tests/ci-workflows/test-home-guard.test.ts +222 tests/ci-workflows/test-runner-bun.test.ts +22500 tests/ci-workflows/test-runner.test.ts +128 tests/ci-workflows/test-sandbox-cleanup.test.ts +145 tests/ci-workflows/test-temp-quarantine.test.ts +129 tests/ci-workflows/version-line.test.ts +192 tests/ci-workflows/warmup-registration.test.ts +322 tests/ci-workflows/zz-ci-api-usage-isolation.test.ts +111 tests/ci-workflows/zz-ci-storage-policy-isolation.test.ts +207 tests/ci-workflows/zz-pr-coderabbit-readiness-revalidation.test.ts +46703 tests/claude-integration/anthropic2-native-routing.test.ts +13495 tests/claude-integration/claude-529-mapping.test.ts +269 tests/claude-integration/claude-agent-startup-sync.test.ts +205 tests/claude-integration/claude-agents-inject-client.test.ts +554 tests/claude-integration/claude-agents-inject.test.ts +145 tests/claude-integration/claude-alias.test.ts +95 tests/claude-integration/claude-auth-detect.test.ts +439 tests/claude-integration/claude-auth-mode.test.ts +123 tests/claude-integration/claude-authmode-migration.test.ts +2106 tests/claude-integration/claude-cli-picker-surface.test.ts +850 tests/claude-integration/claude-cli-picker.test.ts +224 tests/claude-integration/claude-cli.test.ts +728 tests/claude-integration/claude-code-thought-signature-scope.test.ts +288 tests/claude-integration/claude-compatibility.test.ts +1538 tests/claude-integration/claude-config-home-isolation.test.ts +269 tests/claude-integration/claude-context-accounting.test.ts +145 tests/claude-integration/claude-context-windows.test.ts +251 tests/claude-integration/claude-desktop-1m.test.ts +33007 tests/claude-integration/claude-desktop-cli.test.ts +4768 tests/claude-integration/claude-desktop-config-path.test.ts +5743 tests/claude-integration/claude-desktop-discovery.test.ts +22083 tests/claude-integration/claude-desktop-first-party-guards.test.ts +75866 tests/claude-integration/claude-desktop-first-party.test.ts +539 tests/claude-integration/claude-desktop-mode-explanation.test.ts +172 tests/claude-integration/claude-desktop-native-context.test.ts +5799 tests/claude-integration/claude-desktop-picker-profile.test.ts +25728 tests/claude-integration/claude-desktop-picker-routes.test.ts +3490 tests/claude-integration/claude-desktop-picker.test.ts +127 tests/claude-integration/claude-desktop-policy.test.ts +38272 tests/claude-integration/claude-desktop-remote-hub.test.ts +437 tests/claude-integration/claude-desktop-system-proxy.test.ts +2854 tests/claude-integration/claude-devin-output-order.test.ts +348 tests/claude-integration/claude-dotenv-provenance-transport.test.ts +2826 tests/claude-integration/claude-estimate-projection.test.ts +21085 tests/claude-integration/claude-first-party-agent-sync.test.ts +41676 tests/claude-integration/claude-first-party-union.test.ts +175 tests/claude-integration/claude-gateway-cache.test.ts +1938 tests/claude-integration/claude-inbound-cache-stabilize.test.ts +123 tests/claude-integration/claude-inbound-debug.test.ts +157 tests/claude-integration/claude-inbound-token-footer.test.ts +175 tests/claude-integration/claude-inbound-tool-reference.test.ts +223 tests/claude-integration/claude-inbound.test.ts +159 tests/claude-integration/claude-intercept-cleanup.test.ts +389 tests/claude-integration/claude-intercept-client-class.test.ts +23891 tests/claude-integration/claude-intercept-local-ca-files.test.ts +428 tests/claude-integration/claude-intercept-local-ca-windows.test.ts +18234 tests/claude-integration/claude-intercept-local-ca.test.ts +1286 tests/claude-integration/claude-intercept-model-bindings.test.ts +4732 tests/claude-integration/claude-intercept-no-proxy.test.ts +14616 tests/claude-integration/claude-intercept-on-demand.test.ts +7586 tests/claude-integration/claude-intercept-proxy-auth.test.ts +314 tests/claude-integration/claude-intercept-proxy.test.ts +1295 tests/claude-integration/claude-intercept-settings.test.ts +142 tests/claude-integration/claude-long-context-eligibility.test.ts +99359 tests/claude-integration/claude-management-api.test.ts +43995 tests/claude-integration/claude-messages-endpoint.test.ts +4221 tests/claude-integration/claude-messages-thread.test.ts +747 tests/claude-integration/claude-model-info.test.ts +15392 tests/claude-integration/claude-models-discovery.test.ts +6467 tests/claude-integration/claude-native-affinity.test.ts +67983 tests/claude-integration/claude-native-passthrough.test.ts +347 tests/claude-integration/claude-native-rate-limit-headers.test.ts +746 tests/claude-integration/claude-native-reasoning-continuity.test.ts +984 tests/claude-integration/claude-native-reasoning-provenance.test.ts +16457 tests/claude-integration/claude-outbound.test.ts +208 tests/claude-integration/claude-output-schema-strict.test.ts +352 tests/claude-integration/claude-picker-bootstrap.test.ts +11408 tests/claude-integration/claude-picker-ca-store.test.ts +6570 tests/claude-integration/claude-picker-ca.test.ts +1184 tests/claude-integration/claude-picker-listener.test.ts +148 tests/claude-integration/claude-picker-models.test.ts +17946 tests/claude-integration/claude-picker-recovery.test.ts +27450 tests/claude-integration/claude-picker-runtime.test.ts +26190 tests/claude-integration/claude-picker-startup.test.ts +188 tests/claude-integration/claude-picker-trust.test.ts +1006 tests/claude-integration/claude-picker-upload.test.ts +11622 tests/claude-integration/claude-prompt-too-long.test.ts +143 tests/claude-integration/claude-shell-hook.test.ts +265 tests/claude-integration/claude-sidecar-override.test.ts +143 tests/claude-integration/claude-source-envelope.test.ts +405 tests/claude-integration/claude-subagent-model-force.test.ts +302 tests/claude-integration/claude-system-env-auto.test.ts +1047 tests/claude-integration/messages-native-decline-trace.test.ts +54261 tests/claude-integration/messages-native-oauth.test.ts +3996 tests/claude-integration/messages-native-opaque-state.test.ts +8920 tests/claude-integration/messages-native-scope.test.ts +29289 tests/claude-integration/messages-native.test.ts +37964 tests/claude-integration/messages-request-id-endpoint.test.ts +175 tests/claude-integration/messages-request-id-headers.test.ts +48631 tests/claude-integration/messages-revoked-token.test.ts +335 tests/claude-integration/messages-surface-matrix.test.ts +127 tests/cli/agent-driven.test.ts +174 tests/cli/agent-subagent-force.test.ts +179 tests/cli/claude-config-first-party.test.ts +24387 tests/cli/cli-access-audio-live.test.ts +599 tests/cli/cli-access-audio.test.ts +192 tests/cli/cli-access-data-plane.test.ts +190 tests/cli/cli-access-key-guard.test.ts +991 tests/cli/cli-access-rename.test.ts +448 tests/cli/cli-account-alias-target.test.ts +190 tests/cli/cli-account-cancel-flow.test.ts +236 tests/cli/cli-account-key-quota.test.ts +159 tests/cli/cli-account-kiro-device.test.ts +1457 tests/cli/cli-account-login-options.test.ts +128 tests/cli/cli-account-orca-import.test.ts +405 tests/cli/cli-account-pin-drain.test.ts +7188 tests/cli/cli-account-policy.test.ts +176 tests/cli/cli-account-pool-verbs.test.ts +111 tests/cli/cli-account-threshold.test.ts +459 tests/cli/cli-account-verify-status.test.ts +936 tests/cli/cli-account.test.ts +744 tests/cli/cli-agent-runtime-settings.test.ts +2251 tests/cli/cli-agent-settings.test.ts +207 tests/cli/cli-alias-json.test.ts +172 tests/cli/cli-anthropic-account-threshold.test.ts +174 tests/cli/cli-anthropic2-account.test.ts +484 tests/cli/cli-api-protocols.test.ts +287 tests/cli/cli-aside-sync.test.ts +94 tests/cli/cli-capabilities-arguments.test.ts +213 tests/cli/cli-capabilities.test.ts +143 tests/cli/cli-capability-access-workflows.test.ts +348 tests/cli/cli-capability-account-workflows.test.ts +1065 tests/cli/cli-capability-agent-workflows.test.ts +10007 tests/cli/cli-capability-data.test.ts +635 tests/cli/cli-capability-integration-workflows.test.ts +309 tests/cli/cli-capability-lab-workflows.test.ts +253 tests/cli/cli-capability-operation-workflows.test.ts +253 tests/cli/cli-capability-provider-workflows.test.ts +127 tests/cli/cli-catalog-command-result.test.ts +200 tests/cli/cli-catalog-prewarm.test.ts +2250 tests/cli/cli-claude-desktop-profile.test.ts +126 tests/cli/cli-client-start-fence.test.ts +177 tests/cli/cli-client-start-owner-e2e.test.ts +210 tests/cli/cli-codex-cli-update.test.ts +221 tests/cli/cli-codex-log-guard-compact.test.ts +110 tests/cli/cli-codex-log-guard-protection.test.ts +190 tests/cli/cli-codex-log-guard.test.ts +1507 tests/cli/cli-combo-parity.test.ts +192 tests/cli/cli-combo-partial-update.test.ts +468 tests/cli/cli-combo-stats.test.ts +363 tests/cli/cli-companion-timeline.test.ts +266 tests/cli/cli-companion-usage.test.ts +221 tests/cli/cli-companion.test.ts +6600 tests/cli/cli-config-bom.test.ts +11880 tests/cli/cli-config-command.test.ts +6581 tests/cli/cli-config-default-show.test.ts +4918 tests/cli/cli-config-show-client.test.ts +15137 tests/cli/cli-connect-readiness.test.ts +4679 tests/cli/cli-dispatch.test.ts +160 tests/cli/cli-dto-fidelity.test.ts +713 tests/cli/cli-effort-slug.test.ts +3330 tests/cli/cli-effort.test.ts +450 tests/cli/cli-ensure-client-state.test.ts +327 tests/cli/cli-ensure-readiness.test.ts +4019 tests/cli/cli-export-command.test.ts +143 tests/cli/cli-head.test.ts +3500 tests/cli/cli-headless-parity.test.ts +7346 tests/cli/cli-help-navigation.test.ts +23892 tests/cli/cli-help-paths.test.ts +11408 tests/cli/cli-help-recovery.test.ts +24483 tests/cli/cli-help.test.ts +2301 tests/cli/cli-init-port.test.ts +734 tests/cli/cli-injection-follow.test.ts +112 tests/cli/cli-integer-options.test.ts +222 tests/cli/cli-integration-cursor.test.ts +206 tests/cli/cli-integration-droid.test.ts +2062 tests/cli/cli-integration-journal.test.ts +3286 tests/cli/cli-integration-preview.test.ts +384 tests/cli/cli-json-contract.test.ts +444 tests/cli/cli-json-input.test.ts +385 tests/cli/cli-kiro-auto-selection.test.ts +449 tests/cli/cli-link-enrollment.test.ts +459 tests/cli/cli-link-force.test.ts +379 tests/cli/cli-link.test.ts +535 tests/cli/cli-log-follow.test.ts +523 tests/cli/cli-log-view-filter.test.ts +318 tests/cli/cli-logout-runtime.test.ts +207 tests/cli/cli-logs-request-id.test.ts +460 tests/cli/cli-management-auth.test.ts +2173 tests/cli/cli-models-custom-runtime.test.ts +348 tests/cli/cli-models-display-name.test.ts +159 tests/cli/cli-models-free-only.test.ts +2224 tests/cli/cli-models-local-result.test.ts +204 tests/cli/cli-models-order.test.ts +175 tests/cli/cli-models-price.test.ts +1127 tests/cli/cli-models-reasoning.test.ts +362 tests/cli/cli-models-runtime-dispatch.test.ts +143 tests/cli/cli-models-set.test.ts +30228 tests/cli/cli-models.test.ts +477 tests/cli/cli-native-profile.test.ts +159 tests/cli/cli-observe-logs.test.ts +148 tests/cli/cli-observe-snapshot.test.ts +192 tests/cli/cli-path-diagnostics.test.ts +1221 tests/cli/cli-provider-batch.test.ts +619 tests/cli/cli-provider-lifecycle-runtime.test.ts +174 tests/cli/cli-provider-result.test.ts +302 tests/cli/cli-provider-settings.test.ts +9108 tests/cli/cli-provider-sync-result.test.ts +41308 tests/cli/cli-provider.test.ts +6096 tests/cli/cli-ready-subprocess.test.ts +241 tests/cli/cli-ready.test.ts +419 tests/cli/cli-registry.test.ts +267 tests/cli/cli-remote-workspace-hub.test.ts +6964 tests/cli/cli-resolve-subprocess.test.ts +159 tests/cli/cli-resolve.test.ts +143 tests/cli/cli-restart-handoff.test.ts +7643 tests/cli/cli-restart-health.test.ts +10401 tests/cli/cli-restore-back.test.ts +170 tests/cli/cli-route-policy-not-found.test.ts +3271 tests/cli/cli-route-policy-write.test.ts +23424 tests/cli/cli-start-auxiliary-bind.test.ts +90526 tests/cli/cli-start-journal-order.test.ts +142 tests/cli/cli-start-routing-heal-wiring.test.ts +237 tests/cli/cli-status-codex-routing-drift.test.ts +7319 tests/cli/cli-status-hub-state.test.ts +25442 tests/cli/cli-status-json.test.ts +410 tests/cli/cli-status-oauth-health.test.ts +1931 tests/cli/cli-status-startup-health.test.ts +129 tests/cli/cli-stop-json.test.ts +527 tests/cli/cli-storage-inspect.test.ts +2233 tests/cli/cli-storage-policy-fields.test.ts +286 tests/cli/cli-system-health.test.ts +128 tests/cli/cli-system-settings-parity.test.ts +476 tests/cli/cli-transport-honesty.test.ts +7971 tests/cli/cli-update-badge.test.ts +916 tests/cli/cli-update-restart-admission.test.ts +128 tests/cli/cli-update-restart-child.test.ts +300 tests/cli/cli-update-restart-desktop-service.test.ts +420 tests/cli/cli-update-restart-home.test.ts +206 tests/cli/cli-update-restart-service-record.test.ts +132 tests/cli/cli-update-restart-supervision.test.ts +1485 tests/cli/cli-update-restart-transport.test.ts +254 tests/cli/cli-update-restart.test.ts +436 tests/cli/cli-usage-hub.test.ts +222 tests/cli/cli-usage-model-search.test.ts +186 tests/cli/cli-usage-report.test.ts +253 tests/cli/cli-usage-scope.test.ts +13314 tests/cli/cli-v2-runtime.test.ts +26018 tests/cli/cli-version-skew.test.ts +1413 tests/cli/ensure-desired-integrations-race.test.ts +5631 tests/cli/hub-disable.test.ts +1637 tests/cli/hub-gated-local-clients.test.ts +236 tests/cli/hub-invite.test.ts +354 tests/cli/interactive-confirm.test.ts +142 tests/cli/model-selection-guidance.test.ts +431 tests/cli/ocx-launcher-desktop-handoff.test.ts +65949 tests/cli/ocx-launcher-runtime.test.ts +177 tests/cli/ocx-launcher-source.test.ts +125 tests/cli/ocx-run.test.ts +8797 tests/cli/restore-completes-shared-teardown.test.ts +4101 tests/cli/route-explainability.test.ts +52182 tests/cli/sibling-home-client-sync.test.ts +244 tests/cli/star-deferral.test.ts +119 tests/cli/start-args.test.ts +149 tests/cli/start-ownership-publication.test.ts +143 tests/cli/system-restart-client-package-tree.test.ts +190 tests/cli/system-restart-client.test.ts +8766 tests/cli/uninstall.test.ts +200 tests/clients/aside-profile-identity.test.ts +1230 tests/clients/aside-profile-paths.test.ts +15270 tests/clients/aside-profile-sync-owner.test.ts +2221 tests/clients/aside-profiles-catalog-refresh.test.ts +12191 tests/clients/aside-profiles.test.ts +129 tests/clients/client-catalog-compatibility.test.ts +46676 tests/clients/client-connect.test.ts +159 tests/clients/client-export-effective-metadata.test.ts +554 tests/clients/client-export-live-wire.test.ts +174 tests/clients/client-export-modality-enum.test.ts +190 tests/clients/client-export-reasoning-controls.test.ts +159 tests/clients/client-export-reasoning-metadata.test.ts +286 tests/clients/client-fingerprint.test.ts +204 tests/clients/client-hub-relay.test.ts +815 tests/clients/client-hub-state.test.ts +142 tests/clients/client-hub-usage.test.ts +4879 tests/clients/client-lifecycle-lock.test.ts +9909 tests/clients/client-link-connect.test.ts +1185 tests/clients/client-link-relay.test.ts +13912 tests/clients/client-link-runtime.test.ts +1156 tests/clients/client-link-state.test.ts +127 tests/clients/client-link-status.test.ts +2919 tests/clients/client-link-teardown.test.ts +7267 tests/clients/client-link-tunnel.test.ts +6412 tests/clients/client-machine-listener.test.ts +158 tests/clients/client-runtime.test.ts +127 tests/clients/cline-client.test.ts +6074 tests/clients/cline-writer.test.ts +2296 tests/clients/command-code-client-contract.test.ts +161 tests/clients/command-code-client.test.ts +614 tests/clients/desktop-3p-guard.test.ts +4256 tests/clients/desktop-3p-removal.test.ts +4232 tests/clients/desktop-3p.test.ts +156 tests/clients/desktop-app-restart-posix.test.ts +760 tests/clients/desktop-app-restart.test.ts +1937 tests/clients/desktop-app-server-shim-launcher.test.ts +385 tests/clients/desktop-app-server-shim.test.ts +175 tests/clients/desktop-chatgpt-config.test.ts +143 tests/clients/desktop-cli-command-surface.test.ts +128 tests/clients/desktop-cli-contracts.test.ts +175 tests/clients/desktop-exit-ownership.test.ts +112 tests/clients/desktop-host-visibility.test.ts +129 tests/clients/desktop-install-identity.test.ts +523 tests/clients/desktop-profile.test.ts +144 tests/clients/desktop-proxy-direct-transport.test.ts +24328 tests/clients/desktop-remote-store.test.ts +175 tests/clients/desktop-restart-handoff.test.ts +152 tests/clients/desktop-runtime-identity.test.ts +114 tests/clients/desktop-start-at-login-default.test.ts +161 tests/clients/desktop-startup-surface.test.ts +8909 tests/clients/desktop-supervised-restart.test.ts +159 tests/clients/desktop-tray-availability.test.ts +174 tests/clients/desktop-update-surface.test.ts +95 tests/clients/desktop-widget-entry.test.ts +3056 tests/clients/droid-client.test.ts +3922 tests/clients/droid-managed-reasoning-defaults.test.ts +2489 tests/clients/droid-normalized-reasoning-defaults.test.ts +8328 tests/clients/integrations-current-store.test.ts +9892 tests/clients/integrations-dsh-profile-patch.test.ts +6258 tests/clients/integrations-hermes-affinity.test.ts +3381 tests/clients/integrations-journal.test.ts +1158 tests/clients/integrations-merge.test.ts +159 tests/clients/integrations-serialize.test.ts +1042 tests/clients/integrations-state.test.ts +949 tests/clients/integrations-superseded-store.test.ts +1319 tests/clients/integrations-writer-frozen-config.test.ts +2062 tests/clients/integrations-writer-guard.test.ts +10174 tests/clients/integrations-writer.test.ts +3911 tests/clients/kilo-client.test.ts +94 tests/clients/lazycodex-detection.test.ts +95 tests/clients/link-admission-wait.test.ts +97 tests/clients/link-boundary.test.ts +542 tests/clients/link-compensation.test.ts +143 tests/clients/link-fingerprint.test.ts +1397 tests/clients/link-ports.test.ts +921 tests/clients/link-relay-bound-transport.test.ts +207 tests/clients/link-routes.test.ts +143 tests/clients/link-ssh-argv-discovery.test.ts +1218 tests/clients/link-ssh-argv.test.ts +110 tests/clients/link-ssh-config.test.ts +689 tests/clients/link-ssh-runner.test.ts +458 tests/clients/link-status-projection.test.ts +479 tests/clients/link-store.test.ts +5319 tests/clients/link-supervisor.test.ts +142 tests/clients/link-tunnel-state.test.ts +1794 tests/clients/mutation-plan.test.ts +176 tests/clients/omo-client.test.ts +872 tests/clients/omo-role-models.test.ts +253 tests/clients/omp-path-contract.test.ts +191 tests/clients/omp-yaml-source-inline-comments.test.ts +4066 tests/clients/opencode-kilo-refresh.test.ts +188 tests/clients/pi-path-contract.test.ts +176 tests/clients/prime-client.test.ts +1300 tests/clients/raycast-client.test.ts +128 tests/clients/raycast-detect.test.ts +6932 tests/clients/remote-catalog.test.ts +158 tests/clients/remote-control-prototype.test.ts +220 tests/clients/remote-workspace-activation.test.ts +190 tests/clients/remote-workspace-agent-wire.test.ts +396 tests/clients/remote-workspace-app-server.integration.test.ts +316 tests/clients/remote-workspace-claude.integration.test.ts +222 tests/clients/remote-workspace-cli-runtimes.test.ts +173 tests/clients/remote-workspace-cli.test.ts +349 tests/clients/remote-workspace-codex-runtime.test.ts +364 tests/clients/remote-workspace-command-runner.test.ts +126 tests/clients/remote-workspace-device.test.ts +191 tests/clients/remote-workspace-hub.test.ts +328 tests/clients/remote-workspace-linux-confinement.test.ts +606 tests/clients/remote-workspace-management.test.ts +144 tests/clients/remote-workspace-platform.test.ts +127 tests/clients/remote-workspace-protocol.test.ts +158 tests/clients/remote-workspace-rpc-framing.test.ts +1055 tests/clients/remote-workspace-secret-store.test.ts +8484 tests/clients/remote-workspace-server.test.ts +158 tests/clients/remote-workspace-session-binding.test.ts +130 tests/clients/remote-workspace-sessions.test.ts +156 tests/clients/remote-workspace-tool-bridge.test.ts +401 tests/clients/remote-workspace.test.ts +16041 tests/clients/sync-client-integrations-unattended.test.ts +8408 tests/clients/sync-client-integrations.test.ts +5905 tests/codex-integration/active-registry-admission.test.ts +15323 tests/codex-integration/active-turn-disconnect.test.ts +539 tests/codex-integration/anthropic2-catalog.test.ts +272 tests/codex-integration/app-owned-memory.test.ts +70007 tests/codex-integration/bearer-admission-routed-provider.test.ts +904 tests/codex-integration/catalog-async-lifetime.test.ts +3984 tests/codex-integration/catalog-auto-refresh-client-fanout.test.ts +7420 tests/codex-integration/catalog-auto-refresh-scheduler.test.ts +19901 tests/codex-integration/catalog-auto-review-row.test.ts +301 tests/codex-integration/catalog-cursor-search.test.ts +191 tests/codex-integration/catalog-custom-ordering.test.ts +289 tests/codex-integration/catalog-duplicate-slug-dedup.test.ts +124 tests/codex-integration/catalog-free-pricing-status.test.ts +42654 tests/codex-integration/catalog-full-picker-order.test.ts +128 tests/codex-integration/catalog-gated-native-suppression-reason.test.ts +444 tests/codex-integration/catalog-go-exact-efforts.test.ts +112 tests/codex-integration/catalog-hub-context-window.test.ts +331 tests/codex-integration/catalog-input-modality-enum.test.ts +348 tests/codex-integration/catalog-llamacpp-capabilities.test.ts +4962 tests/codex-integration/catalog-modelalias-unique-sync.test.ts +680 tests/codex-integration/catalog-oauth-observation.test.ts +1003 tests/codex-integration/catalog-opencode-go-context-window.test.ts +156 tests/codex-integration/catalog-opengateway-hints.test.ts +4863 tests/codex-integration/catalog-remote-pull.test.ts +172 tests/codex-integration/catalog-retain-models.test.ts +411 tests/codex-integration/catalog-routed-comp-hash.test.ts +161 tests/codex-integration/catalog-seed-window-fill.test.ts +142 tests/codex-integration/catalog-slug-uniqueness-boundary.test.ts +335 tests/codex-integration/catalog-verbosity-default.test.ts +386 tests/codex-integration/catalog-vision-sidecar-modalities.test.ts +446 tests/codex-integration/catalog-zero-credit-picker.test.ts +23354 tests/codex-integration/client-injection-guard.test.ts +240 tests/codex-integration/code-mode-collab-surface.test.ts +3224 tests/codex-integration/codex-account-clear-paused.test.ts +7264 tests/codex-integration/codex-account-delete-atomicity.test.ts +142 tests/codex-integration/codex-account-label.test.ts +126 tests/codex-integration/codex-account-namespaces.test.ts +9834 tests/codex-integration/codex-account-selection-atomicity.test.ts +1069 tests/codex-integration/codex-account-selection-preferences.test.ts +1182 tests/codex-integration/codex-account-store-refresh-classification.test.ts +5135 tests/codex-integration/codex-account-store.test.ts +1853 tests/codex-integration/codex-account-threshold-api.test.ts +1980 tests/codex-integration/codex-account-threshold-auth.test.ts +1745 tests/codex-integration/codex-account-threshold-routing.test.ts +1918 tests/codex-integration/codex-account-unusable-reason.test.ts +816 tests/codex-integration/codex-admission-primitives.test.ts +1186 tests/codex-integration/codex-admission.test.ts +338 tests/codex-integration/codex-affinity-debug.test.ts +126 tests/codex-integration/codex-app-server-path-spaces.test.ts +13079 tests/codex-integration/codex-app-server-processes.test.ts +219 tests/codex-integration/codex-app-server-restart-service.test.ts +14551 tests/codex-integration/codex-auth-api.test.ts +396 tests/codex-integration/codex-auth-cancel-ownership.test.ts +2183 tests/codex-integration/codex-auth-collision.test.ts +3464 tests/codex-integration/codex-auth-context.test.ts +6065 tests/codex-integration/codex-auth-release.test.ts +1199 tests/codex-integration/codex-catalog-admission.test.ts +147 tests/codex-integration/codex-catalog-exclusions.test.ts +712 tests/codex-integration/codex-catalog-golden.test.ts +6538 tests/codex-integration/codex-catalog-heal-integration.test.ts +444 tests/codex-integration/codex-catalog-identity-neutral.test.ts +12216 tests/codex-integration/codex-catalog-intent.test.ts +423 tests/codex-integration/codex-catalog-ladders.test.ts +966 tests/codex-integration/codex-catalog-model-picker-order.test.ts +128 tests/codex-integration/codex-catalog-refresh-status.test.ts +10754 tests/codex-integration/codex-catalog-restore.test.ts +144 tests/codex-integration/codex-catalog-routed-removal.test.ts +284 tests/codex-integration/codex-catalog-self-heal.test.ts +70037 tests/codex-integration/codex-catalog-sync-hardening.test.ts +4112 tests/codex-integration/codex-catalog-write-audit.test.ts +1000 tests/codex-integration/codex-catalog-write-serialization.test.ts +7239 tests/codex-integration/codex-catalog-writer.test.ts +8476 tests/codex-integration/codex-catalog.test.ts +143 tests/codex-integration/codex-cli-install-provenance.test.ts +192 tests/codex-integration/codex-cli-installation-identity.test.ts +143 tests/codex-integration/codex-cli-installation-targets.test.ts +143 tests/codex-integration/codex-cli-update-launcher-policy.test.ts +3633 tests/codex-integration/codex-cli-update-zero-effect.test.ts +978 tests/codex-integration/codex-cli-windows-installation-files.test.ts +83015 tests/codex-integration/codex-composed-acceptance.test.ts +36774 tests/codex-integration/codex-config-drift-heal.test.ts +4709 tests/codex-integration/codex-config-generation.test.ts +15047 tests/codex-integration/codex-config-preservation.test.ts +25935 tests/codex-integration/codex-config-write-lock.test.ts +301 tests/codex-integration/codex-context-owner.test.ts +40794 tests/codex-integration/codex-convergence-account-selectors.test.ts +12802 tests/codex-integration/codex-convergence-contract.test.ts +8520 tests/codex-integration/codex-cooldown-recovery.test.ts +1048 tests/codex-integration/codex-coordinator-doctor.test.ts +667 tests/codex-integration/codex-credits-after-limit-main.test.ts +3374 tests/codex-integration/codex-credits-after-limit.test.ts +3204 tests/codex-integration/codex-credits-probes.test.ts +1352 tests/codex-integration/codex-credits-settings.test.ts +320 tests/codex-integration/codex-credits.test.ts +3065 tests/codex-integration/codex-desired-state.test.ts +705 tests/codex-integration/codex-entitlement-identity-read-fence.test.ts +5141 tests/codex-integration/codex-envkey-admission-substitution.test.ts +364 tests/codex-integration/codex-exec-invocation.test.ts +1448 tests/codex-integration/codex-failure-window.test.ts +158 tests/codex-integration/codex-features-cache.test.ts +127 tests/codex-integration/codex-features-residual.test.ts +572 tests/codex-integration/codex-filesystem-evidence.test.ts +127 tests/codex-integration/codex-forward-access-programs-writer.test.ts +1235 tests/codex-integration/codex-gather-authority.test.ts +28154 tests/codex-integration/codex-history-job.test.ts +1673 tests/codex-integration/codex-history-lock.test.ts +15525 tests/codex-integration/codex-history-provider.test.ts +570 tests/codex-integration/codex-history-reachability.test.ts +143 tests/codex-integration/codex-history-worker-boundary.test.ts +3871 tests/codex-integration/codex-history-worker.test.ts +1506 tests/codex-integration/codex-history-writer.test.ts +26350 tests/codex-integration/codex-home-owner-lifecycle.test.ts +6832 tests/codex-integration/codex-home-owner.test.ts +147 tests/codex-integration/codex-home-wsl-local-state.test.ts +222 tests/codex-integration/codex-home-wsl.test.ts +13128 tests/codex-integration/codex-idle-window.test.ts +380 tests/codex-integration/codex-inject-history-wording.test.ts +332916 tests/codex-integration/codex-inject-integration.test.ts +10469 tests/codex-integration/codex-inject-missing-config.test.ts +7319 tests/codex-integration/codex-inject-retained-table.test.ts +29555 tests/codex-integration/codex-inject-v1-reconcile.test.ts +42012 tests/codex-integration/codex-inject-write-lock.test.ts +287 tests/codex-integration/codex-inject.test.ts +94 tests/codex-integration/codex-injected-marker.test.ts +965 tests/codex-integration/codex-integration-record.test.ts +8918 tests/codex-integration/codex-journal-recovery.test.ts +72818 tests/codex-integration/codex-journal.test.ts +95 tests/codex-integration/codex-legacy-config-keys.test.ts +1360 tests/codex-integration/codex-lineage-placement.test.ts +11720 tests/codex-integration/codex-linked-account-pause.test.ts +2155 tests/codex-integration/codex-log-guard-coderabbit.test.ts +96 tests/codex-integration/codex-log-guard-doctor-coderabbit.test.ts +153 tests/codex-integration/codex-log-guard-doctor-protection.test.ts +110 tests/codex-integration/codex-log-guard-doctor.test.ts +1870 tests/codex-integration/codex-log-guard-inspect.test.ts +237 tests/codex-integration/codex-log-guard-lock.test.ts +24090 tests/codex-integration/codex-log-guard-maintenance-coderabbit.test.ts +12964 tests/codex-integration/codex-log-guard-maintenance.test.ts +113 tests/codex-integration/codex-log-guard-policy.test.ts +128 tests/codex-integration/codex-log-guard-processes.test.ts +3366 tests/codex-integration/codex-log-guard-protection.test.ts +1265 tests/codex-integration/codex-log-guard-status-zero-write.test.ts +2568 tests/codex-integration/codex-main-account-refresh.test.ts +8720 tests/codex-integration/codex-main-grant-refusal.test.ts +5134 tests/codex-integration/codex-main-rotation.test.ts +601 tests/codex-integration/codex-main-token-invalidated.test.ts +212 tests/codex-integration/codex-management-convergence.test.ts +686 tests/codex-integration/codex-metadata-integrity.test.ts +16486 tests/codex-integration/codex-missing-home.test.ts +141 tests/codex-integration/codex-model-availability-error.test.ts +461 tests/codex-integration/codex-model-denial-evidence.test.ts +301 tests/codex-integration/codex-model-entitlement-admission.test.ts +413 tests/codex-integration/codex-model-entitlements-program-shape.test.ts +3714 tests/codex-integration/codex-model-entitlements.test.ts +3514 tests/codex-integration/codex-models-cache-invalidate.test.ts +6095 tests/codex-integration/codex-native-availability-nux.test.ts +6354 tests/codex-integration/codex-native-residue.test.ts +1183 tests/codex-integration/codex-pin-drain-projection.test.ts +3215 tests/codex-integration/codex-plan.test.ts +6984 tests/codex-integration/codex-plugins-doctor.test.ts +2695 tests/codex-integration/codex-pool-credit-policy.test.ts +824 tests/codex-integration/codex-pool-plan-exclusion.test.ts +110 tests/codex-integration/codex-pool-refresh-backoff.test.ts +6617 tests/codex-integration/codex-pool-request-owned-main.test.ts +43781 tests/codex-integration/codex-pool-rotation.test.ts +18514 tests/codex-integration/codex-priority-failback.test.ts +4314 tests/codex-integration/codex-prompt-adopt.test.ts +2873 tests/codex-integration/codex-prompt-base-variants.test.ts +1092 tests/codex-integration/codex-prompt-journal.test.ts +552 tests/codex-integration/codex-prompt-layers-read.test.ts +4171 tests/codex-integration/codex-prompt-layers-write.test.ts +114 tests/codex-integration/codex-prompt-layers.test.ts +1536 tests/codex-integration/codex-prompt-lock-sharing.test.ts +1943 tests/codex-integration/codex-prompt-lock.test.ts +6144 tests/codex-integration/codex-prompt-route.test.ts +3144 tests/codex-integration/codex-prompt-scalar-scope.test.ts +1918 tests/codex-integration/codex-prompt-text-probe.test.ts +6017 tests/codex-integration/codex-prompt-writes.test.ts +6412 tests/codex-integration/codex-provider-table-retention.test.ts +2080 tests/codex-integration/codex-quota-auto-refresh-generation.test.ts +841 tests/codex-integration/codex-quota-auto-refresh-main-admission.test.ts +10130 tests/codex-integration/codex-quota-auto-refresh.test.ts +174 tests/codex-integration/codex-quota-capacity.test.ts +316 tests/codex-integration/codex-quota-history.test.ts +446 tests/codex-integration/codex-quota-parser-parity.test.ts +3376 tests/codex-integration/codex-quota-prime.test.ts +1897 tests/codex-integration/codex-quota-query-backoff.test.ts +403 tests/codex-integration/codex-quota-rejection.test.ts +2045 tests/codex-integration/codex-refresh.test.ts +7282 tests/codex-integration/codex-reset-credit-auto-redeem.test.ts +9942 tests/codex-integration/codex-reset-credit-operation-ledger.test.ts +578 tests/codex-integration/codex-reset-credit-recovery.test.ts +127 tests/codex-integration/codex-restart-contract-parity.test.ts +443 tests/codex-integration/codex-restart-route.test.ts +29610 tests/codex-integration/codex-restore-app-rewrite.test.ts +39279 tests/codex-integration/codex-retained-root-serialization.test.ts +650 tests/codex-integration/codex-routing-cache-affinity-detour.test.ts +48882 tests/codex-integration/codex-routing.test.ts +460 tests/codex-integration/codex-runtime-wsl-desktop.test.ts +1520 tests/codex-integration/codex-runtime.test.ts +1191 tests/codex-integration/codex-service-manager-probe-hardening.test.ts +2283 tests/codex-integration/codex-service-manager-probe.test.ts +246 tests/codex-integration/codex-shim-autorestore.test.ts +222 tests/codex-integration/codex-shim-destroyed-probe.test.ts +125 tests/codex-integration/codex-shim-ensure-failure.test.ts +185 tests/codex-integration/codex-shim-fnm.test.ts +162 tests/codex-integration/codex-shim-migration.test.ts +161 tests/codex-integration/codex-shim-overlay-candidates.test.ts +190 tests/codex-integration/codex-shim-overlay-readiness.test.ts +182 tests/codex-integration/codex-shim-overlay-transaction.test.ts +79 tests/codex-integration/codex-shim-overlay.test.ts +5044 tests/codex-integration/codex-shim-path-readiness.test.ts +7716 tests/codex-integration/codex-shim-readiness.test.ts +158 tests/codex-integration/codex-shim-runtime-preflight.test.ts +128 tests/codex-integration/codex-shim-standalone.test.ts +16501 tests/codex-integration/codex-shim.test.ts +4323 tests/codex-integration/codex-sidecar-credit-policy.test.ts +236 tests/codex-integration/codex-signin-lockout.test.ts +1220 tests/codex-integration/codex-spark-visibility.test.ts +311 tests/codex-integration/codex-sqlite-home.test.ts +9174 tests/codex-integration/codex-sync-api.test.ts +6140 tests/codex-integration/codex-sync-new-model-policy.test.ts +348 tests/codex-integration/codex-sync-response.test.ts +638 tests/codex-integration/codex-systemd-env-probe.test.ts +380 tests/codex-integration/codex-tool-mode.test.ts +3644 tests/codex-integration/codex-transition-state-adoption.test.ts +1302 tests/codex-integration/codex-transition-state-first-use-regression.test.ts +8470 tests/codex-integration/codex-transition-state-race.test.ts +9081 tests/codex-integration/codex-transition-state.test.ts +1715 tests/codex-integration/codex-user-identity.test.ts +14053 tests/codex-integration/codex-v2-gate.test.ts +682 tests/codex-integration/codex-warmup.test.ts +19820 tests/codex-integration/codex-web-search-switch.test.ts +95 tests/codex-integration/codex-websocket-registry.test.ts +17682 tests/codex-integration/codex-write-lock.test.ts +130 tests/codex-integration/combo-authoritative-reset.test.ts +340 tests/codex-integration/combo-codex-exhaustion-cooldown.test.ts +301 tests/codex-integration/combo-last-resort.test.ts +112 tests/codex-integration/combo-permanent-failure-cooldown.test.ts +110 tests/codex-integration/combo-reset-window-model-scope.test.ts +570 tests/codex-integration/combos.test.ts +1400 tests/codex-integration/compatibility-manifest.test.ts +160 tests/codex-integration/configured-native-models.test.ts +127 tests/codex-integration/context-compat.test.ts +140 tests/codex-integration/custom-model-catalog-migration.test.ts +2156 tests/codex-integration/discovered-native-models.test.ts +5124 tests/codex-integration/doctor.test.ts +1026 tests/codex-integration/effort-policy.test.ts +297 tests/codex-integration/fast-row-listing.test.ts +793 tests/codex-integration/fast-row.test.ts +396 tests/codex-integration/gather-routed-models-single-flight.test.ts +174 tests/codex-integration/gpt6-native-rows.test.ts +209 tests/codex-integration/gpt61-sol-rows.test.ts +143 tests/codex-integration/history-migration-guardian.test.ts +775 tests/codex-integration/history-ocx-compaction-recovery.test.ts +128 tests/codex-integration/history-paginated-openai-compat.test.ts +12027 tests/codex-integration/history-paginated-transition-destinations.test.ts +175 tests/codex-integration/injection-link-websocket.test.ts +3010 tests/codex-integration/injection-model-api.test.ts +348 tests/codex-integration/injection-model-suggest-routes.test.ts +192 tests/codex-integration/injection-routing-drift.test.ts +13888 tests/codex-integration/injection-routing-heal-apply.test.ts +739 tests/codex-integration/injection-routing-healer.test.ts +17176 tests/codex-integration/issue-452-empty-503.test.ts +38712 tests/codex-integration/issue-702-expired-replay-state.test.ts +1549 tests/codex-integration/issue-914-transport-attribution.test.ts +4176 tests/codex-integration/low-quota-protection.test.ts +1530 tests/codex-integration/main-account-credit-renewal.test.ts +144 tests/codex-integration/main-account-external-usage.test.ts +54184 tests/codex-integration/main-account-hard-lock-auth.test.ts +1190 tests/codex-integration/main-account-hard-lock-default.test.ts +250 tests/codex-integration/main-account-hard-lock-policy.test.ts +1269 tests/codex-integration/main-account-hard-lock-recovery.test.ts +206 tests/codex-integration/main-account-hard-lock-retirement.test.ts +179 tests/codex-integration/main-account-hard-lock-thresholds.test.ts +847 tests/codex-integration/main-account-policy-binding-wait.test.ts +356 tests/codex-integration/main-device-reauth-api.test.ts +728 tests/codex-integration/main-device-reauth.test.ts +507 tests/codex-integration/main-quota-evidence-validation.test.ts +5818 tests/codex-integration/main-quota-provenance.test.ts +2554 tests/codex-integration/main-quota-window-observation.test.ts +410 tests/codex-integration/messaging-local-cli.test.ts +238 tests/codex-integration/messaging-local-discovery.test.ts +206 tests/codex-integration/messaging-local-envelope.test.ts +587 tests/codex-integration/messaging-local-lifecycle.test.ts +160 tests/codex-integration/messaging-local-send.test.ts +334 tests/codex-integration/messaging-local-socket.test.ts +141 tests/codex-integration/model-cache-generation-tombstone.test.ts +309 tests/codex-integration/model-cache.test.ts +236 tests/codex-integration/model-display-names-management-api.test.ts +409 tests/codex-integration/model-metadata-sync.test.ts +8140 tests/codex-integration/model-pinned-effort.test.ts +16478 tests/codex-integration/model-visibility-management-api.test.ts +1949 tests/codex-integration/multi-agent-compat.test.ts +3805 tests/codex-integration/multi-agent-keep-native-v1.test.ts +113 tests/codex-integration/multi-agent-origin.test.ts +158 tests/codex-integration/native-alias-maintainer-regressions.test.ts +937 tests/codex-integration/native-claude-code-toggle.test.ts +9624 tests/codex-integration/native-claude-desktop-toggle.test.ts +35950 tests/codex-integration/native-codex-toggle.test.ts +2227 tests/codex-integration/native-grok-toggle.test.ts +394 tests/codex-integration/native-main-auth-temp.test.ts +112 tests/codex-integration/native-main-claim-cache.test.ts +624 tests/codex-integration/native-main-claim.test.ts +35806 tests/codex-integration/native-main-owner-lifetime.test.ts +953 tests/codex-integration/native-model-toggle.test.ts +159 tests/codex-integration/native-profile-api.test.ts +31558 tests/codex-integration/native-profile-crash-boundaries.test.ts +8342 tests/codex-integration/native-profile-drain-server.test.ts +14922 tests/codex-integration/native-profile-manager.test.ts +669 tests/codex-integration/native-profile-processes.test.ts +146 tests/codex-integration/native-profile-recovery.test.ts +3675 tests/codex-integration/native-profile-route-security.test.ts +1360 tests/codex-integration/native-profile-stage-lifecycle.test.ts +1428 tests/codex-integration/native-profile-startup-release.test.ts +64051 tests/codex-integration/native-profile-startup.test.ts +1014 tests/codex-integration/native-profile-store.test.ts +6118 tests/codex-integration/orca-import.test.ts +652 tests/codex-integration/parallel-tool-calls-optin.test.ts +190 tests/codex-integration/project-config-warning-snapshot.test.ts +238 tests/codex-integration/project-config-warnings.test.ts +523 tests/codex-integration/reasoning-effort.test.ts +446 tests/codex-integration/reasoning-metadata.test.ts +900 tests/codex-integration/reserve-auth-context.test.ts +175 tests/codex-integration/reserve-availability.test.ts +20920 tests/codex-integration/reserve-catalog-lifecycle.test.ts +222 tests/codex-integration/reserve-catalog.test.ts +2408 tests/codex-integration/reserve-dispatch.test.ts +395 tests/codex-integration/reserve-helper-boundary.test.ts +478 tests/codex-integration/reserve-passive-revocation.test.ts +2585 tests/codex-integration/reserve-quota-scope.test.ts +174 tests/codex-integration/selected-models.test.ts +749 tests/codex-integration/slug-codec.test.ts +8039 tests/codex-integration/token-guardian.test.ts +8426 tests/codex-integration/ultrafast-tier-honesty.test.ts +191 tests/codex-integration/upstream-reachability.test.ts +176 tests/codex-integration/warmup.test.ts +191 tests/config/client-config-export-new-clients.test.ts +127 tests/config/client-config-export-output-limit.test.ts +254 tests/config/client-config-export.test.ts +570 tests/config/client-config-new-clients.test.ts +190 tests/config/compaction-recovery-settings.test.ts +459 tests/config/config-account-thresholds.test.ts +5145 tests/config/config-anthropic-instance-pool.test.ts +426 tests/config/config-blocked-model-redirects.test.ts +461 tests/config/config-catalog-auto-refresh.test.ts +129 tests/config/config-commandcode-claude-pin.test.ts +4355 tests/config/config-load-degrade.test.ts +7987 tests/config/config-mutation-lock.test.ts +804 tests/config/config-non-object-backup.test.ts +1297 tests/config/config-ownership-uninstall.test.ts +127 tests/config/config-rebase-provenance-writers.test.ts +175 tests/config/config-save-boundary.test.ts +763 tests/config/config-skills-catalog-refresh.test.ts +887 tests/config/config-spend-ceilings.test.ts +447 tests/config/config-spend-pool-collision.test.ts +19125 tests/config/config-user-edits.test.ts +108 tests/config/expand-user-path.test.ts +13849 tests/config/hub-disable-live-save.test.ts +17473 tests/config/model-pinned-effort-config.test.ts +173 tests/config/owner-registry-acl.test.ts +193 tests/config/protocol-settings.test.ts +15051 tests/config/serving-runtimes.test.ts +5904 tests/config/settings-desktop-switch-apply.test.ts +1473 tests/config/settings-fast-rows.test.ts +2203 tests/config/settings-main-account-hard-lock.test.ts +1466 tests/config/settings-memory-models.test.ts +2046 tests/config/settings-oauth-open-browser.test.ts +532 tests/config/settings-startup-health-seam.test.ts +30366 tests/config/settings-stream-mode.test.ts +95 tests/config/types-barrel-identity.test.ts +337 tests/config/url-normalization.test.ts +143 tests/config/yaml-fragment-source.test.ts +572 tests/e2e-style/phase100-native-parity.test.ts +324 tests/gui/alibaba-intl-token-plan.test.ts +333 tests/gui/claude-manual-env.test.ts +160 tests/gui/codex-account-mode-state.test.ts +127 tests/gui/codex-auth-modal-status.test.ts +97 tests/gui/codex-set-prompt.test.ts +515 tests/gui/combo-workspace-data.test.ts +159 tests/gui/combo-workspace-jev-decision.test.ts +174 tests/gui/dashboard-sidecar-codex-write.test.ts +269 tests/gui/dashboard-uptime.test.ts +158 tests/gui/gui-api-error.test.ts +840 tests/gui/gui-codex-usage-score-parity.test.ts +91 tests/gui/gui-desktop-sidecar-script.test.ts +145 tests/gui/gui-desktop-sidecar-signing.test.ts +174 tests/gui/gui-management-session.test.ts +109 tests/gui/gui-pair-capability.test.ts +1511 tests/gui/gui-pair-client.test.ts +21411 tests/gui/gui-pair-delivery.test.ts +7717 tests/gui/gui-pair-http.test.ts +347 tests/gui/gui-static.test.ts +317 tests/gui/gui-tray-vibrancy-surface.test.ts +476 tests/gui/html-lang-boot-script.test.ts +11169 tests/gui/integrations-invariants.test.ts +143 tests/gui/logs-model-tier-confirmation.test.ts +112 tests/gui/main-device-reauth-ui.test.ts +397 tests/gui/models-feedback-callback.test.ts +192 tests/gui/models-free-filter.test.ts +79 tests/gui/models-page-groups.test.ts +126 tests/gui/models-workspace-tabs.test.ts +211 tests/gui/oauth-first-add-hint.test.ts +337 tests/gui/oauth-tos-warning.test.ts +2 tests/gui/platform-dialog-guard.test.ts +176 tests/gui/provider-payload.test.ts +196 tests/gui/provider-subscription-cli.test.ts +120 tests/gui/provider-workspace-auth.test.ts +254 tests/gui/provider-workspace-data.test.ts +142 tests/gui/provider-workspace-rail.test.ts +298 tests/gui/provider-workspace-state.test.ts +280 tests/gui/quota-bars-rows.test.ts +142 tests/gui/qwen-cloud-endpoints.test.ts +347 tests/gui/rate-limit-reset-credits.test.ts +128 tests/gui/routing-intelligence-ui.test.ts +175 tests/gui/routing-profile-editor-data.test.ts +349 tests/gui/standalone-build-script.test.ts +317 tests/gui/startup-health-ui.test.ts +175 tests/gui/tencent-siliconflow-providers.test.ts +253 tests/gui/vision-sidecar-timeout-bounds.test.ts +525 tests/gui/volcengine-providers.test.ts +221 tests/images/artifacts-prune.test.ts +411 tests/images/artifacts-ssrf.test.ts +148 tests/images/download-cap-default.test.ts +174 tests/images/download-connect-deadline-default.test.ts +143 tests/images/gemini-inline.test.ts +521 tests/images/loop-reasoning-replay.test.ts +3658 tests/images/loop.test.ts +320 tests/images/pinned-https-get.test.ts +159 tests/images/plan.test.ts +96 tests/images/synthetic-tool.test.ts +234 tests/images/xai-client.test.ts +427 tests/images/z-fulfill.test.ts +648 tests/images/z-handler-activation.test.ts +838 tests/lab/core-lab-boundary.test.ts +428 tests/lab/core-link-boundary.test.ts +15613 tests/lab/lab-activation.test.ts +1241 tests/lab/lab-automation-coderabbit-regressions.test.ts +2533 tests/lab/lab-automation-final-coderabbit-regressions.test.ts +953 tests/lab/lab-automation-ingwannu-regressions.test.ts +967 tests/lab/lab-automation-management-http.test.ts +126 tests/lab/lab-automation-persisted-cap-regression.test.ts +4764 tests/lab/lab-automation-review-regressions.test.ts +4163 tests/lab/lab-automation.test.ts +1951 tests/lab/lab-community-evidence.test.ts +1855 tests/lab/lab-community-filename-contract.test.ts +239 tests/lab/lab-community-mutation-lock.test.ts +1046 tests/lab/lab-community-publisher-continuity.test.ts +348 tests/lab/lab-conformance-harness.test.ts +255 tests/lab/lab-conformance-runner-failures.test.ts +5627 tests/lab/lab-evidence-ledger.test.ts +2767 tests/lab/lab-evidence-sanitization.test.ts +221 tests/lab/lab-fabric-outcome-validation.test.ts +191 tests/lab/lab-fabric-persistence-boundary.test.ts +462 tests/lab/lab-fabric-producer-deadline.test.ts +152782 tests/lab/lab-fabric-task.test.ts +222 tests/lab/lab-installation-salt-cache.test.ts +1975 tests/lab/lab-ledger-mutation-lock.test.ts +836 tests/lab/lab-live-pinned-timeouts.test.ts +1710 tests/lab/lab-live-probe.test.ts +238 tests/lab/lab-live-receipt-integrity.test.ts +606 tests/lab/lab-live-review-regressions.test.ts +1617 tests/lab/lab-live-sandbox.test.ts +240 tests/lab/lab-passive-production-evidence.test.ts +350 tests/lab/lab-passive-production-surfaces.test.ts +127 tests/lab/lab-paths-security.test.ts +507 tests/lab/lab-post-merge-hardening.test.ts +550 tests/lab/lab-post-merge-projection.test.ts +369 tests/lab/lab-private-file-consumer-recovery.test.ts +124 tests/lab/lab-private-file-durability.test.ts +206 tests/lab/lab-public-api-json.test.ts +141 tests/lab/lab-public-artifact-policy.test.ts +2451 tests/lab/lab-public-coderabbit-regressions.test.ts +509 tests/lab/lab-public-core-contract.test.ts +1417 tests/lab/lab-public-deep-review-regressions.test.ts +787 tests/lab/lab-public-evidence.test.ts +1062 tests/lab/lab-public-export-transaction.test.ts +127 tests/lab/lab-public-file-safety.test.ts +1584 tests/lab/lab-public-final-review-regressions.test.ts +2243 tests/lab/lab-public-lifecycle-hardening.test.ts +158 tests/lab/lab-public-privacy-ipv6.test.ts +3450 tests/lab/lab-public-provenance-recovery.test.ts +3471 tests/lab/lab-public-review-fixes.test.ts +143 tests/lab/lab-public-route-registry.test.ts +1492 tests/lab/lab-public-security-regressions.test.ts +1921 tests/lab/lab-public-surfaces.test.ts +491 tests/lab/lab-public-wire-contract.test.ts +190 tests/lab/lab-read-filter-validation.test.ts +48084 tests/lab/lab-read-surfaces.test.ts +871 tests/lib/abort-idle-deadline.test.ts +237 tests/lib/acl-error-classification.test.ts +445 tests/lib/ambiguous-resend-composition.test.ts +129 tests/lib/ambiguous-resend-gate.test.ts +378 tests/lib/bun-runtime-preflight.test.ts +143 tests/lib/bun-stream-caps.test.ts +173 tests/lib/clearable-deadline.test.ts +716 tests/lib/credential-redirect-guard.test.ts +144 tests/lib/debug.test.ts +189 tests/lib/execution-budget-permits.test.ts +160 tests/lib/failure-attribution.test.ts +126 tests/lib/failure-stage-model.test.ts +382 tests/lib/local-chat-completion.test.ts +113 tests/lib/local-destinations.test.ts +176 tests/lib/local-upstream.test.ts +2678 tests/lib/optional-shutdown-hooks.test.ts +380 tests/lib/pinned-http-content-coding.test.ts +407 tests/lib/pinned-http.test.ts +223 tests/lib/plugin-loader.test.ts +128 tests/lib/plugin-upstream-hooks.test.ts +157 tests/lib/privacy-mask-account.test.ts +402 tests/lib/process-control-graceful.test.ts +109 tests/lib/process-control.test.ts +273 tests/lib/provider-egress.test.ts +141 tests/lib/reasoning-replay-scope-source.test.ts +1290 tests/lib/redact.test.ts +160 tests/lib/remove-tree-helper.test.ts +298 tests/lib/self-launch-argv.test.ts +1628 tests/lib/socks5-fetch.test.ts +204 tests/lib/socks5-upload-lifecycle.test.ts +2014 tests/lib/spend-ceiling-enforcement.test.ts +314 tests/lib/spend-contract-c.test.ts +94 tests/lib/spend-corruption-compat.test.ts +112 tests/lib/spend-experimental-history.test.ts +2729 tests/lib/spend-ledger-file-journal.test.ts +7428 tests/lib/spend-ledger-owner.test.ts +1519 tests/lib/spend-pool-continuity.test.ts +128 tests/lib/spend-reservation-ledger.test.ts +1440 tests/lib/spend-seed-overflow.test.ts +174 tests/lib/spend-unbound-expiry.test.ts +158 tests/lib/spend-zero-config-compat.test.ts +142 tests/lib/stall-subprocess-exit.test.ts +310 tests/lib/stall-timeout.test.ts +478 tests/lib/standalone.test.ts +144 tests/lib/strict-semver.test.ts +142 tests/lib/synced-state-location.test.ts +79 tests/lib/system-restart-contract-security.test.ts +174 tests/lib/token-estimate.test.ts +302 tests/lib/transient-budget-scope-source.test.ts +430 tests/lib/transport-null-body.test.ts +317 tests/lib/upstream-error-message-fallback.test.ts +286 tests/lib/upstream-retry-zero-output.test.ts +8314 tests/lib/upstream-retry.test.ts +339 tests/lib/worker-embed-coverage.test.ts +239 tests/lib/worker-embed.test.ts +332 tests/lib/workflow-budget.test.ts +8646 tests/oauth/adapter-event-oauth-failover.test.ts +10976 tests/oauth/anthropic2-discovery-ownership.test.ts +4240 tests/oauth/chatgpt-device-auth.test.ts +157 tests/oauth/chatgpt-oauth.test.ts +188 tests/oauth/chatgpt-token-boundaries.test.ts +238 tests/oauth/chatgpt-token-expiry.test.ts +20253 tests/oauth/generic-oauth-failover.test.ts +4220 tests/oauth/generic-oauth-refusal-composition.test.ts +2597 tests/oauth/key-login-live-update.test.ts +205 tests/oauth/key-login-preserves-model-costs.test.ts +110 tests/oauth/local-token-detect.test.ts +6423 tests/oauth/oauth-account-attribution.test.ts +348 tests/oauth/oauth-account-id-collision.test.ts +8287 tests/oauth/oauth-account-plans.test.ts +2109 tests/oauth/oauth-account-quota-rank.test.ts +24774 tests/oauth/oauth-accounts-api.test.ts +16711 tests/oauth/oauth-anthropic-identity.test.ts +11232 tests/oauth/oauth-anthropic-instance-health.test.ts +11569 tests/oauth/oauth-anthropic-instance-multiprocess.test.ts +15296 tests/oauth/oauth-anthropic-instance-refresh.test.ts +23356 tests/oauth/oauth-anthropic-instance-registration.test.ts +386 tests/oauth/oauth-callback-binds.test.ts +143 tests/oauth/oauth-callback-server.test.ts +158 tests/oauth/oauth-device-code-contract.test.ts +4276 tests/oauth/oauth-health.test.ts +7108 tests/oauth/oauth-kiro-refresh-attention.test.ts +111 tests/oauth/oauth-log.test.ts +2059 tests/oauth/oauth-login-cli-browser-launch.test.ts +8729 tests/oauth/oauth-login-cli-live-update.test.ts +191 tests/oauth/oauth-login-open-browser.test.ts +570 tests/oauth/oauth-login-summary.test.ts +2789 tests/oauth/oauth-manual-code.test.ts +221 tests/oauth/oauth-open-browser-choice.test.ts +111 tests/oauth/oauth-open-url-result.test.ts +3484 tests/oauth/oauth-provider-reconcile.test.ts +1882 tests/oauth/oauth-public-surface.test.ts +1016 tests/oauth/oauth-reauth-bind.test.ts +4026 tests/oauth/oauth-refresh-generic-lock.test.ts +5209 tests/oauth/oauth-refresh-lock-multiprocess.test.ts +26300 tests/oauth/oauth-refresh.test.ts +791 tests/oauth/oauth-status-privacy.test.ts +2083 tests/oauth/oauth-store-multi.test.ts +1705 tests/oauth/oauth-upsert-preserves-api-key.test.ts +111 tests/oauth/pool-kernel-generic-sweep.test.ts +841 tests/oauth/state-store-sweeper.test.ts +333 tests/providers/alibaba-region-backup.test.ts +1039 tests/providers/alibaba-region-migration.test.ts +143 tests/providers/alibaba-region-startup.test.ts +984 tests/providers/alibaba-token-plan-responses-optin.test.ts +1524 tests/providers/alibaba-token-plan-wire-defaults.test.ts +887 tests/providers/api-key-catalog-authority.test.ts +108 tests/providers/api-key-selection-capture.test.ts +633 tests/providers/aside-client.test.ts +203 tests/providers/auto-compact-budget.test.ts +428 tests/providers/azure-adapter.test.ts +322 tests/providers/azure-model-router-tool-schema.test.ts +1151 tests/providers/azure-vendor-metadata.test.ts +353 tests/providers/baseten-provider.test.ts +179 tests/providers/chutes-provider.test.ts +270 tests/providers/claude-cli-adapter.test.ts +300 tests/providers/cline-pass-deepseek-v4-tool-replay.test.ts +219 tests/providers/cline-pass-provider.test.ts +153 tests/providers/cline-pass-reasoning-efforts.test.ts +159 tests/providers/cline-provider.test.ts +2248 tests/providers/codebuddy-adapter.test.ts +110 tests/providers/codebuddy-live-acceptance.test.ts +195 tests/providers/codebuddy-live-models.test.ts +4919 tests/providers/codebuddy-mcp-server.test.ts +148 tests/providers/codebuddy-protocol.test.ts +560 tests/providers/codebuddy-tool-bridge-turn.test.ts +396 tests/providers/codebuddy-tool-bridge.test.ts +143 tests/providers/coding-agent-json-lines-scaling.test.ts +837 tests/providers/command-code-efforts.test.ts +158 tests/providers/command-code-error-finish.test.ts +204 tests/providers/command-code-fakeip-discovery.test.ts +4647 tests/providers/command-code-project-context.test.ts +1808 tests/providers/command-code-provider.test.ts +684 tests/providers/command-code-quota.test.ts +668 tests/providers/command-code-tool-text-prose-split.test.ts +2379 tests/providers/command-code-tool-text.test.ts +142 tests/providers/command-code-workspace-cache.test.ts +512 tests/providers/commandcode-provider.test.ts +112 tests/providers/context-cap-unknown-window.test.ts +108 tests/providers/context-window-seed-repair.test.ts +206 tests/providers/crusoe-provider.test.ts +315 tests/providers/cursor/cursor-adapter.test.ts +95 tests/providers/cursor/cursor-arg-normalize.test.ts +160 tests/providers/cursor/cursor-blob-claude-client-tools.test.ts +79 tests/providers/cursor/cursor-blob-integrity.test.ts +587 tests/providers/cursor/cursor-blob.test.ts +324 tests/providers/cursor/cursor-call-id.test.ts +241 tests/providers/cursor/cursor-cancel-provenance.test.ts +239 tests/providers/cursor/cursor-catalog.test.ts +160 tests/providers/cursor/cursor-claude-id.test.ts +507 tests/providers/cursor/cursor-continuation-invariants.test.ts +314 tests/providers/cursor/cursor-continuity-retention.test.ts +301 tests/providers/cursor/cursor-default-catalog-suppression.test.ts +238 tests/providers/cursor/cursor-desktop-exec.test.ts +125 tests/providers/cursor/cursor-discovery.test.ts +369 tests/providers/cursor/cursor-display-names.test.ts +6168 tests/providers/cursor/cursor-effort-rows.test.ts +97 tests/providers/cursor/cursor-effort-suffix.test.ts +135 tests/providers/cursor/cursor-effort-table.test.ts +205 tests/providers/cursor/cursor-envelope-echo-retry.test.ts +189 tests/providers/cursor/cursor-eof-terminal.test.ts +145 tests/providers/cursor/cursor-errors.test.ts +128 tests/providers/cursor/cursor-exec-empty-result.test.ts +199 tests/providers/cursor/cursor-fast-listing.test.ts +143 tests/providers/cursor/cursor-fast-tier.test.ts +394 tests/providers/cursor/cursor-foreground-shell.test.ts +223 tests/providers/cursor/cursor-framing.test.ts +313 tests/providers/cursor/cursor-h2-pool-shutdown.test.ts +4032 tests/providers/cursor/cursor-hardening.test.ts +175 tests/providers/cursor/cursor-http1-transport.test.ts +3940 tests/providers/cursor/cursor-images.test.ts +7595 tests/providers/cursor/cursor-integration-status.test.ts +209 tests/providers/cursor/cursor-interaction-query.test.ts +130 tests/providers/cursor/cursor-kv-store.test.ts +104 tests/providers/cursor/cursor-live-smoke-gate.test.ts +298 tests/providers/cursor/cursor-live-transport.test.ts +159 tests/providers/cursor/cursor-local-installer.test.ts +4440 tests/providers/cursor/cursor-local-models-schema.test.ts +237 tests/providers/cursor/cursor-mcp-manager.test.ts +967 tests/providers/cursor/cursor-mcp-stdio.test.ts +175 tests/providers/cursor/cursor-message-mapper.test.ts +96 tests/providers/cursor/cursor-native-exec-common.test.ts +254 tests/providers/cursor/cursor-native-exec-policy.test.ts +342 tests/providers/cursor/cursor-native-exec-shell.test.ts +174 tests/providers/cursor/cursor-native-exec.test.ts +143 tests/providers/cursor/cursor-oauth-shell.test.ts +777 tests/providers/cursor/cursor-oauth.test.ts +129 tests/providers/cursor/cursor-pool.test.ts +157 tests/providers/cursor/cursor-protobuf-events.test.ts +252 tests/providers/cursor/cursor-repetition-breaker.test.ts +287 tests/providers/cursor/cursor-request-builder.test.ts +158 tests/providers/cursor/cursor-request-compat.test.ts +375 tests/providers/cursor/cursor-roster-account-scope.test.ts +123 tests/providers/cursor/cursor-silent-redirect.test.ts +492 tests/providers/cursor/cursor-static-catalog.test.ts +3735 tests/providers/cursor/cursor-stream-health.test.ts +178 tests/providers/cursor/cursor-structured-edit.test.ts +113 tests/providers/cursor/cursor-tool-arg-decoding.test.ts +189 tests/providers/cursor/cursor-tool-choice.test.ts +207 tests/providers/cursor/cursor-tool-continuation.test.ts +130 tests/providers/cursor/cursor-tool-definitions.test.ts +4045 tests/providers/cursor/cursor-tool-finalize-race.test.ts +158 tests/providers/cursor/cursor-tool-result-image.test.ts +397 tests/providers/cursor/cursor-tool-result-invocation.test.ts +174 tests/providers/cursor/cursor-tool-suspended-checkpoint.test.ts +222 tests/providers/cursor/cursor-tool-wording.test.ts +160 tests/providers/cursor/cursor-toolresult-normalize.test.ts +1832 tests/providers/cursor/cursor-transport-retry.test.ts +159 tests/providers/cursor/cursor-ultra-mode.test.ts +139 tests/providers/cursor/cursor-umbrella-rows.test.ts +112 tests/providers/cursor/cursor-uncallable-quarantine.test.ts +350 tests/providers/cursor/cursor-vision-wire-harness.test.ts +774 tests/providers/cyber-policy-error-fidelity.test.ts +281 tests/providers/deepinfra-provider.test.ts +315 tests/providers/deepseek-artifact-tool-schema.test.ts +2082 tests/providers/deepseek-inbound-wire.test.ts +624 tests/providers/deepseek-quota-currency.test.ts +159 tests/providers/deepseek-reasoning-replay-gaps.test.ts +191 tests/providers/deepseek-reasoning-replay.test.ts +2069 tests/providers/deepseek-responses-item-id-repair.test.ts +2335 tests/providers/devin-adapter-reset-wait.test.ts +1031 tests/providers/devin-adapter.test.ts +773 tests/providers/devin-anthropic-signature-fallback.test.ts +393 tests/providers/devin-chat-wire-fixes.test.ts +366 tests/providers/devin-cli-authmode-migration.test.ts +304 tests/providers/devin-effort-ladder.test.ts +221 tests/providers/devin-family-resolution.test.ts +191 tests/providers/devin-hardening.test.ts +397 tests/providers/devin-image-passthrough.test.ts +491 tests/providers/devin-live-models.test.ts +1870 tests/providers/devin-login.test.ts +726 tests/providers/devin-output-budget.test.ts +1203 tests/providers/devin-prompt-cache.test.ts +1317 tests/providers/devin-provider-merge-migration.test.ts +1048 tests/providers/devin-quota.test.ts +175 tests/providers/devin-reasoning-continuation.test.ts +96 tests/providers/devin-stated-reset-hardening.test.ts +143 tests/providers/devin-stated-reset-retry.test.ts +161 tests/providers/devin-stream-deadline.test.ts +175 tests/providers/devin-trajectory.test.ts +177 tests/providers/digitalocean-scaleway-provider.test.ts +220 tests/providers/exa-web-search.test.ts +141 tests/providers/fast-row-ingress.test.ts +221 tests/providers/featherless-provider.test.ts +465 tests/providers/flash-route-image-modalities.test.ts +3919 tests/providers/forward-admission-separation.test.ts +8998 tests/providers/github-copilot/github-copilot-account-origin.test.ts +37772 tests/providers/github-copilot/github-copilot-auto-runtime.test.ts +161 tests/providers/github-copilot/github-copilot-auto.test.ts +379 tests/providers/github-copilot/github-copilot-catalog-hints.test.ts +158 tests/providers/github-copilot/github-copilot-context-tier.test.ts +16134 tests/providers/github-copilot/github-copilot-oauth.test.ts +94 tests/providers/github-copilot/github-copilot-sse-rewrite.test.ts +935 tests/providers/github-copilot/github-copilot-stream-contract.test.ts +6467 tests/providers/github-copilot/github-copilot-wire-defaults.test.ts +125 tests/providers/haiku-5-5-catalog.test.ts +238 tests/providers/hyperbolic-provider.test.ts +6776 tests/providers/initial-model-selection.test.ts +13650 tests/providers/initial-selection-write-fence.test.ts +126 tests/providers/jev-provider.test.ts +1736 tests/providers/kimi-oauth-identity.test.ts +205 tests/providers/kimi-responses-adjacency.test.ts +128 tests/providers/kiro/kiro-account-load.test.ts +2430 tests/providers/kiro/kiro-account-quota.test.ts +7409 tests/providers/kiro/kiro-account-state-disk.test.ts +1935 tests/providers/kiro/kiro-account-state-imports.test.ts +1364 tests/providers/kiro/kiro-adapter.test.ts +2773 tests/providers/kiro/kiro-auth-context-continuation.test.ts +1636 tests/providers/kiro/kiro-auto-selection.test.ts +936 tests/providers/kiro/kiro-builder-id-profile.test.ts +96 tests/providers/kiro/kiro-calibration.test.ts +1364 tests/providers/kiro/kiro-device-builder.test.ts +7898 tests/providers/kiro/kiro-device-social.test.ts +478 tests/providers/kiro/kiro-fallback-error-body.test.ts +3341 tests/providers/kiro/kiro-images.test.ts +12269 tests/providers/kiro/kiro-leased-responses.test.ts +95 tests/providers/kiro/kiro-metering-events.test.ts +143 tests/providers/kiro/kiro-metering-usage.test.ts +2535 tests/providers/kiro/kiro-model-catalog.test.ts +2777 tests/providers/kiro/kiro-model-preference.test.ts +8380 tests/providers/kiro/kiro-oauth.test.ts +23971 tests/providers/kiro/kiro-org-user-identity.test.ts +6161 tests/providers/kiro/kiro-pool-load-settings.test.ts +4098 tests/providers/kiro/kiro-pool-rank.test.ts +4567 tests/providers/kiro/kiro-quota-metrics.test.ts +142 tests/providers/kiro/kiro-reasoning-roundtrip.test.ts +5154 tests/providers/kiro/kiro-refusal-failover.test.ts +127 tests/providers/kiro/kiro-refusal.test.ts +191 tests/providers/kiro/kiro-remote-image.test.ts +671 tests/providers/kiro/kiro-retry.test.ts +3873 tests/providers/kiro/kiro-review-regressions.test.ts +299 tests/providers/kiro/kiro-single-final.test.ts +633 tests/providers/kiro/kiro-stream.test.ts +807 tests/providers/kiro/kiro-transport-parity.test.ts +152 tests/providers/kiro/kiro-usage-quota.test.ts +620 tests/providers/kiro/kiro-usage-restart.test.ts +142 tests/providers/kiro/kiro-windows-cli-db-path.test.ts +96 tests/providers/kiro/kiro-windows-cli-executable-path.test.ts +144 tests/providers/kiro/kiro-wire-estimate.test.ts +273 tests/providers/meta-model-api-provider.test.ts +1079 tests/providers/meta-muse-device.test.ts +392 tests/providers/meta-muse-login-order.test.ts +330 tests/providers/meta-muse-oauth.test.ts +111 tests/providers/mimo-effort.test.ts +348 tests/providers/mimo-free-provider.test.ts +386 tests/providers/mimo-token-plan-capacity.test.ts +174 tests/providers/mimo-token-plan-provider.test.ts +626 tests/providers/mimo-v26-catalog.test.ts +5556 tests/providers/minimax-clients.test.ts +1461 tests/providers/minimax-combo-image-input.test.ts +631 tests/providers/minimax-quota.test.ts +237 tests/providers/minimax-reasoning-split.test.ts +192 tests/providers/model-presets.test.ts +1757 tests/providers/model-rename-migration.test.ts +358 tests/providers/model-roster-seed-repair.test.ts +95 tests/providers/moonshot-endpoints.test.ts +680 tests/providers/moonshot-tool-schema.test.ts +383 tests/providers/muse-key-quota.test.ts +2457 tests/providers/muse-passive-quota-cache.test.ts +951 tests/providers/muse-passive-quota-observation.test.ts +426 tests/providers/muse-spark-web-search-compat.test.ts +112 tests/providers/muse-subscription-usage.test.ts +468 tests/providers/muse-tool-name-alias.test.ts +5603 tests/providers/new-model-policy-runtime.test.ts +112 tests/providers/new-model-policy.test.ts +159 tests/providers/nous-oauth-live.test.ts +13336 tests/providers/nous-oauth.test.ts +222 tests/providers/novita-provider.test.ts +173 tests/providers/nscale-vultr-provider.test.ts +348 tests/providers/nvidia-nim-hardening.test.ts +606 tests/providers/ollama/ollama-native-parser.test.ts +144 tests/providers/ollama/ollama-native-reasoning-wire.test.ts +1 tests/providers/ollama/ollama-native-structured-output.test.ts +505 tests/providers/ollama/ollama-native-tool-continuations.test.ts +827 tests/providers/ollama/ollama-native-v4.test.ts +442 tests/providers/ollama/ollama-native.test.ts +522 tests/providers/ollama/ollama-show-enrichment-v7.test.ts +363 tests/providers/ollama/ollama-show-enrichment.test.ts +361 tests/providers/ollama/ollama-show-ignore-abort.test.ts +996 tests/providers/opencode-cli.test.ts +276 tests/providers/opencode-free-provider.test.ts +126 tests/providers/opencode-go-deepseek.test.ts +448 tests/providers/opencode-go-grok46-responses.test.ts +3139 tests/providers/opencode-go-luna-wire.test.ts +299 tests/providers/opencode-go-muse-context.test.ts +520 tests/providers/opencode-go-muse-vision.test.ts +427 tests/providers/opencode-go-quota.test.ts +11091 tests/providers/opencode-go-session-header.test.ts +1032 tests/providers/opencode-management-transport.test.ts +95 tests/providers/opencode-muse-terminal-repair.test.ts +128 tests/providers/opencode-zen-deepseek-reasoning.test.ts +106 tests/providers/opencode-zen-rate-limit.test.ts +110 tests/providers/opengateway-key-validation.test.ts +159 tests/providers/opengateway-provider.test.ts +619 tests/providers/openrouter-provider-routing.test.ts +174 tests/providers/openrouter-quota-reset-cooldown-4024.test.ts +555 tests/providers/opper-provider.test.ts +599 tests/providers/orcarouter-provider.test.ts +3058 tests/providers/provider-account-quota-persistence.test.ts +10596 tests/providers/provider-account-quota.test.ts +381 tests/providers/provider-anthropic-instance-quota.test.ts +223 tests/providers/provider-anthropic-instance.test.ts +161 tests/providers/provider-antigravity-effort-families.test.ts +647 tests/providers/provider-antigravity-family-catalog.test.ts +173 tests/providers/provider-antigravity-quota-retry.test.ts +7365 tests/providers/provider-antigravity-wire-snapshot.test.ts +6909 tests/providers/provider-api-keys.test.ts +114 tests/providers/provider-capacity.test.ts +4112 tests/providers/provider-config-batch-management.test.ts +95 tests/providers/provider-config-validation.test.ts +14169 tests/providers/provider-connection-test.test.ts +1937 tests/providers/provider-cost-overlay-config.test.ts +412 tests/providers/provider-discovery-log-suppression.test.ts +176 tests/providers/provider-egress-outbound.test.ts +205 tests/providers/provider-id-rewrite.test.ts +4317 tests/providers/provider-key-store.test.ts +529 tests/providers/provider-live-models.test.ts +426 tests/providers/provider-model-aliases.test.ts +877 tests/providers/provider-model-discovery-contract.test.ts +155 tests/providers/provider-outbound-private-network.test.ts +4611 tests/providers/provider-outbound.test.ts +700 tests/providers/provider-quota-label-sanitize.test.ts +776 tests/providers/provider-quota-observed-marker.test.ts +407 tests/providers/provider-quota-ollama-cloud.test.ts +14681 tests/providers/provider-quota.test.ts +729 tests/providers/provider-registry-parity.test.ts +230 tests/providers/provider-runtime-fetch.test.ts +270 tests/providers/provider-static-model-discovery.test.ts +127 tests/providers/provider-tls-profile.test.ts +613 tests/providers/qoder-adapter.test.ts +111 tests/providers/qoder-live-models.test.ts +772 tests/providers/qoder-mcp-server.test.ts +158 tests/providers/qoder-scaffold-guard.test.ts +93 tests/providers/qwen38-preserve-reasoning.test.ts +1227 tests/providers/rate-limit-retry.test.ts +285 tests/providers/resolved-model-policy.test.ts +316 tests/providers/sambanova-nebius-provider.test.ts +222 tests/providers/sponsor-presets.test.ts +239 tests/providers/stepfun-provider.test.ts +1173 tests/providers/tokenlab-protocols.test.ts +191 tests/providers/tokenlab-provider.test.ts +159 tests/providers/umans-provider.test.ts +7974 tests/providers/upstream-transient-retry.test.ts +161 tests/providers/vercel-gateway-provider-routing.test.ts +127 tests/providers/vision-classification-seed-repair.test.ts +143 tests/providers/volcengine-ark-assistant-content.test.ts +142 tests/providers/xai/grok-47-build-fast-metadata.test.ts +80069 tests/providers/xai/grok-47-fast-model-wire.test.ts +366 tests/providers/xai/grok-47-fast-model.test.ts +524 tests/providers/xai/grok-attribution.test.ts +5470 tests/providers/xai/grok-config-inject.test.ts +173 tests/providers/xai/grok-config-write-lock.test.ts +35562 tests/providers/xai/grok-coupon-claim.test.ts +2120 tests/providers/xai/grok-coupon-reconcile.test.ts +980 tests/providers/xai/grok-effort-inject.test.ts +162 tests/providers/xai/grok-lifecycle.test.ts +13266 tests/providers/xai/grok-management-api.test.ts +7999 tests/providers/xai/grok-models-effort-list.test.ts +11820 tests/providers/xai/grok-orphan-adoption.test.ts +194 tests/providers/xai/grok-reset-coupon-cli.test.ts +935 tests/providers/xai/grok-reset-coupons.test.ts +652 tests/providers/xai/grok-selection.test.ts +712 tests/providers/xai/grok-session-identity.test.ts +587 tests/providers/xai/grok-status.test.ts +840 tests/providers/xai/grok-sync.test.ts +285 tests/providers/xai/grok-writer-boundary.test.ts +144 tests/providers/xai/xai-empty-catalog-tool-choice.test.ts +174 tests/providers/xai/xai-no-stop.test.ts +145 tests/providers/xai/xai-oauth-retry.test.ts +2666 tests/providers/xai/xai-refresh-lock.test.ts +156 tests/providers/xai/xai-responses-adjacency.test.ts +209 tests/providers/xai/xai-tool-schema.test.ts +347 tests/providers/xai/xai-transport.test.ts +173 tests/providers/xai/xai-web-search-compat.test.ts +175 tests/providers/xai/xai-web-search.test.ts +386 tests/providers/zai-reasoning-replay.test.ts +191 tests/providers/zcode-client.test.ts +930 tests/providers/zed-hosted-provider.test.ts +454 tests/providers/zed-retained-budget.test.ts +245 tests/providers/zhipu-bigmodel-provider.test.ts +652 tests/providers/zhipu-bigmodel-responses-quota.test.ts +17687 tests/responses/anthropic2-responses-parity.test.ts +310 tests/responses/apply-patch-envelope.test.ts +223 tests/responses/bare-echo-alias.test.ts +3092 tests/responses/chat-completions-deferred-tools.test.ts +74323 tests/responses/chat-completions-endpoint.test.ts +3344 tests/responses/chat-completions-pool-mode.test.ts +5652 tests/responses/chat-conversation-affinity.test.ts +920 tests/responses/chat-inbound-developer-position.test.ts +253 tests/responses/chat-inbound-reasoning-none.test.ts +397 tests/responses/chat-inbound-reasoning-replay.test.ts +157 tests/responses/chat-inline-document-bytes.test.ts +3122 tests/responses/chat-json-sse-fallback.test.ts +1953 tests/responses/chat-legacy-functions-combo.test.ts +1687 tests/responses/chat-media-translation.test.ts +7625 tests/responses/chat-native-combo.test.ts +330 tests/responses/chat-native-decline-reason.test.ts +9570 tests/responses/chat-native-developer-role.test.ts +173 tests/responses/chat-native-image-normalization.test.ts +174 tests/responses/chat-native-lenient-finish.test.ts +31782 tests/responses/chat-native-spend.test.ts +376 tests/responses/chat-native-sse-stall.test.ts +248 tests/responses/chat-refusal-scope.test.ts +2333 tests/responses/chat-refusal.test.ts +2967 tests/responses/chat-responses-control-integration.test.ts +159 tests/responses/chat-responses-control-scope.test.ts +4473 tests/responses/chat-sse-framing-guard.test.ts +139 tests/responses/chat-tool-choice-allowed-tools.test.ts +96 tests/responses/citation-markers.test.ts +141 tests/responses/codex-ws-pool-contention.test.ts +178 tests/responses/compaction-event-ownership.test.ts +173 tests/responses/compaction-progress.test.ts +378 tests/responses/continuation-dedup.test.ts +413 tests/responses/custom-tool-compat.test.ts +8181 tests/responses/droid-reasoning-defaults.test.ts +1919 tests/responses/empty-completion-core.test.ts +114 tests/responses/empty-completion-guard.test.ts +143 tests/responses/empty-completion-hardening.test.ts +246 tests/responses/eventstream-decoder.test.ts +477 tests/responses/external-task-input-repair.test.ts +879 tests/responses/fresh-connection-optout.test.ts +7798 tests/responses/jev-initial-effort-wire.test.ts +130 tests/responses/legacy-shell-compat.test.ts +317 tests/responses/messages-native-bridge-policy.test.ts +488 tests/responses/messages-native-eligibility.test.ts +317 tests/responses/messages-native-oauth-eligibility.test.ts +322 tests/responses/namespace-tool-compat.test.ts +583 tests/responses/opaque-blob-wrapped-rejection.test.ts +1789 tests/responses/openai-responses-passthrough.test.ts +192 tests/responses/openai-responses-summary-none.test.ts +152 tests/responses/parser-content-audio.test.ts +428 tests/responses/passthrough-abort.test.ts +39738 tests/responses/passthrough-deferred-tools.test.ts +182 tests/responses/passthrough-grok-upstream-envelope-echo.test.ts +253 tests/responses/passthrough-headers.test.ts +127 tests/responses/passthrough-override.test.ts +191 tests/responses/plaintext-v2-agent-messages.test.ts +143 tests/responses/protocol-baseline.test.ts +142 tests/responses/protocol-contract.test.ts +275 tests/responses/protocol-direct-encoders-chat.test.ts +97 tests/responses/protocol-direct-encoders-local-upstream.test.ts +175 tests/responses/protocol-direct-encoders-messages.test.ts +142 tests/responses/protocol-direct-encoders-stall-disabled.test.ts +157 tests/responses/protocol-dto.test.ts +404 tests/responses/protocol-envelope.test.ts +127 tests/responses/protocol-features.test.ts +126 tests/responses/protocol-guard.test.ts +1061 tests/responses/protocol-ingress-guard.test.ts +111 tests/responses/protocol-opaque-state.test.ts +125 tests/responses/protocol-path.test.ts +190 tests/responses/protocol-plan-snapshot.test.ts +128 tests/responses/protocol-plan.test.ts +608 tests/responses/protocol-shadow-plan.test.ts +175 tests/responses/protocol-trace.test.ts +152 tests/responses/provider-egress-fetch.test.ts +238 tests/responses/reasoning-effort-summary-default.test.ts +253 tests/responses/reasoning-envelope.test.ts +333 tests/responses/reserve-dispatch-ws.test.ts +144 tests/responses/responses-4546-incident-regression.test.ts +258 tests/responses/responses-account-change-scrub.test.ts +6641 tests/responses/responses-account-label.test.ts +5223 tests/responses/responses-alternate-main-cancellation.test.ts +1907 tests/responses/responses-anthropic-fast-downgrade.test.ts +4253 tests/responses/responses-azure-opaque-blob-recovery.test.ts +476 tests/responses/responses-bare-echo-helper-fence.test.ts +3369 tests/responses/responses-canonical-nonstream.test.ts +174 tests/responses/responses-canonical-only-top-level-fields.test.ts +663 tests/responses/responses-chat-tool-call-content.test.ts +414 tests/responses/responses-chatgpt-web-replay-metadata.test.ts +107 tests/responses/responses-code-mode-goal-helpers.test.ts +222 tests/responses/responses-code-mode-mcp-direct.test.ts +176 tests/responses/responses-code-mode-patch-compile.test.ts +159 tests/responses/responses-code-mode-shell-compile.test.ts +2414 tests/responses/responses-combo-exhausted-error.test.ts +8207 tests/responses/responses-compact-handoff-admission.test.ts +6723 tests/responses/responses-compaction-override.test.ts +176 tests/responses/responses-compaction-policy-identity.test.ts +176 tests/responses/responses-compaction-recovery-policy.test.ts +11896 tests/responses/responses-compaction-recovery.test.ts +9757 tests/responses/responses-compaction-routing.test.ts +334 tests/responses/responses-compaction.test.ts +10025 tests/responses/responses-console-go-upload-retry.test.ts +21274 tests/responses/responses-context-overflow.test.ts +205 tests/responses/responses-continuation-boundaries.test.ts +206 tests/responses/responses-core-modules.test.ts +378 tests/responses/responses-custom-tool-guidance.test.ts +1194 tests/responses/responses-custom-tool-historical-replay.test.ts +1840 tests/responses/responses-custom-tool-repair-dispatch.test.ts +205 tests/responses/responses-custom-tool-repair.test.ts +172 tests/responses/responses-custom-tool-stream-consistency.test.ts +130 tests/responses/responses-default-namespace-emit-normalize.test.ts +16198 tests/responses/responses-devin-401-replay.test.ts +127 tests/responses/responses-dispatch-receipt.test.ts +191 tests/responses/responses-fetch-helpers-boundary.test.ts +117 tests/responses/responses-field-backfill.test.ts +12661 tests/responses/responses-first-send-reservation.test.ts +1598 tests/responses/responses-first-send-spend-policy.test.ts +137 tests/responses/responses-forward-client-headers.test.ts +160 tests/responses/responses-forward-dangling-call.test.ts +1633 tests/responses/responses-forward-incomplete-quota.test.ts +156 tests/responses/responses-forward-output-cap.test.ts +286 tests/responses/responses-forward-posit-continuation.test.ts +328 tests/responses/responses-forward-prompt-envelope.test.ts +253 tests/responses/responses-freeform-wrapper-keys.test.ts +667 tests/responses/responses-function-tool-repair.test.ts +18050 tests/responses/responses-grok-devin-preflight.test.ts +966 tests/responses/responses-hide-raw-reasoning.test.ts +1255 tests/responses/responses-hosted-image-delivery.test.ts +222 tests/responses/responses-hosted-image-display.test.ts +412 tests/responses/responses-hosted-image-limits.test.ts +2913 tests/responses/responses-hosted-send-hop.test.ts +7328 tests/responses/responses-hosted-send-producer.test.ts +5990 tests/responses/responses-hosted-send-reservation.test.ts +651 tests/responses/responses-hosted-send-round-parity.test.ts +4082 tests/responses/responses-hosted-send-unsent-hop.test.ts +1635 tests/responses/responses-hosted-send-ws.test.ts +158 tests/responses/responses-hosted-tool-declaration.test.ts +189 tests/responses/responses-hosted-tool-min-spread.test.ts +1325 tests/responses/responses-image-gen-repair.test.ts +1554 tests/responses/responses-inbound-store-default.test.ts +143 tests/responses/responses-item-id-repair.test.ts +316 tests/responses/responses-json-events.test.ts +314 tests/responses/responses-legacy-dotted-tool-name-repair.test.ts +5858 tests/responses/responses-main-quota-observation.test.ts +2377 tests/responses/responses-memory-models.test.ts +1110 tests/responses/responses-muse-tool-choice.test.ts +1334 tests/responses/responses-muse-tool-name-alias.test.ts +6461 tests/responses/responses-native-main-refresh.test.ts +16948 tests/responses/responses-opaque-blob-recovery.test.ts +143 tests/responses/responses-parser-agent-message.test.ts +190 tests/responses/responses-parser-malformed-content.test.ts +209 tests/responses/responses-parser.test.ts +174 tests/responses/responses-passthrough-transient-policy.test.ts +35267 tests/responses/responses-pool-401-refresh.test.ts +210 tests/responses/responses-pool-refresh-attribution.test.ts +4599 tests/responses/responses-preview-main-read-fence.test.ts +902 tests/responses/responses-reasoning-effort-downgrade.test.ts +1171 tests/responses/responses-reasoning-summary-passthrough.test.ts +160 tests/responses/responses-reset-replay.test.ts +394 tests/responses/responses-routed-web-search-fields.test.ts +6074 tests/responses/responses-run-turn-web-search.test.ts +3192 tests/responses/responses-self-named-namespace-scrub.test.ts +10179 tests/responses/responses-send-budget-counts.test.ts +4639 tests/responses/responses-send-budget-errors.test.ts +1312 tests/responses/responses-shadow-intercept.test.ts +3933 tests/responses/responses-show-thinking-summary.test.ts +1283 tests/responses/responses-skills-snapshot.test.ts +8250 tests/responses/responses-snapshot-repair-server.test.ts +565 tests/responses/responses-snapshot-repair.test.ts +128 tests/responses/responses-sparse-terminal-tool-scope.test.ts +8548 tests/responses/responses-spend-anchor.test.ts +159 tests/responses/responses-spend-capacity-guard.test.ts +921 tests/responses/responses-spend-contract-c.test.ts +635 tests/responses/responses-spend-ledger-wiring.test.ts +236 tests/responses/responses-spill-acl-recovery.test.ts +1441 tests/responses/responses-spill-admission-headroom.test.ts +513 tests/responses/responses-spill-inspection.test.ts +445 tests/responses/responses-spill-orphan-sweep.test.ts +396 tests/responses/responses-spill-shutdown-clock.test.ts +197 tests/responses/responses-state-snapshot-select.test.ts +222 tests/responses/responses-state-snapshot-stubs.test.ts +507 tests/responses/responses-state-store-false.test.ts +1496 tests/responses/responses-state-write-amplification.test.ts +11606 tests/responses/responses-state.test.ts +3254 tests/responses/responses-stateless-dangling-call-repair.test.ts +127 tests/responses/responses-stream-tool-events.test.ts +158 tests/responses/responses-terminal-repair.test.ts +384 tests/responses/responses-tool-conformance.test.ts +143 tests/responses/responses-tool-groups.test.ts +1393 tests/responses/responses-tool-search-repair.test.ts +32714 tests/responses/responses-translated-reset.test.ts +6650 tests/responses/responses-undeclared-tool-guard.test.ts +190 tests/responses/responses-usage-passthrough.test.ts +509 tests/responses/responses-xai-request-compat.test.ts +2430 tests/responses/shadow-intercept-target-lifecycle.test.ts +174 tests/responses/sse-carriage-return.test.ts +508 tests/responses/sse-client-frame-bounds.test.ts +523 tests/responses/sse-decoder.test.ts +364 tests/responses/sse-failed-tail.test.ts +361 tests/responses/sse-inspector-bounds.test.ts +176 tests/responses/sse-null-data-frame.test.ts +159 tests/responses/sse-payload-rewrite.test.ts +238 tests/responses/sse-rewrite-line-endings.test.ts +127 tests/responses/sse-unspaced-data-fields.test.ts +518 tests/responses/thought-signature-credential-scope.test.ts +161 tests/responses/visualization-directives.test.ts +3522 tests/responses/ws-ambiguous-resend.test.ts +205 tests/responses/ws-endpoint.test.ts +127 tests/responses/ws-failure-stage.test.ts +4518 tests/responses/ws-native-injection.test.ts +1605 tests/responses/ws-native-result-continuations.test.ts +4909 tests/responses/ws-native-steering.test.ts +2278 tests/responses/ws-steering-completion.test.ts +1617 tests/responses/ws-steering-smoke.test.ts +889 tests/responses/ws-steering-stability.test.ts +286 tests/responses/ws-upstream-reuse.test.ts +95 tests/responses/ws-upstream-socks5.test.ts +16876 tests/responses/ws-upstream.test.ts +4642 tests/routing/always-on-429-failover.test.ts +2177 tests/routing/anthropic-quorum-cache.test.ts +129 tests/routing/cl01-claude-outbound-review-regressions.test.ts +143 tests/routing/cl01-openai-chat-review-regressions.test.ts +175 tests/routing/cl01-review-regressions.test.ts +822 tests/routing/codex-agent-role-models.test.ts +95 tests/routing/codex-role-auto-assign.test.ts +138 tests/routing/codex-role-sizing.test.ts +242 tests/routing/combo-child-headers.test.ts +17455 tests/routing/combo-management-api.test.ts +221 tests/routing/combo-stream-preflight.test.ts +15346 tests/routing/compatibility-provider-equivalence.test.ts +134 tests/routing/destination-policy-resolved.test.ts +190 tests/routing/fastwire-characterization-routing.test.ts +3009 tests/routing/fastwire-characterization-wire.test.ts +1017 tests/routing/fastwire-observability.test.ts +187 tests/routing/fastwire-policy.test.ts +2952 tests/routing/fastwire-response-authority.test.ts +394 tests/routing/jev-decision-destination.test.ts +3939 tests/routing/jev-decision-model-config.test.ts +3062 tests/routing/jev-decision-provider-combo.test.ts +494 tests/routing/jev-decision.test.ts +1131 tests/routing/jev-model-backend.test.ts +131 tests/routing/jev-service-exchange.test.ts +109 tests/routing/jev-typesafe-golden.test.ts +554 tests/routing/policy-execution.test.ts +2483 tests/routing/probe-lease-dispatch-wiring.test.ts +292 tests/routing/probe-lease.test.ts +412 tests/routing/resolved-model-policy-consumers.test.ts +286 tests/routing/router-blocked-cross-provider.test.ts +248 tests/routing/router-combo-failover-classification.test.ts +417 tests/routing/router-discarded-baseurl-warning.test.ts +175 tests/routing/router-template-baseurl.test.ts +467 tests/routing/router.test.ts +12480 tests/routing/routing-analytics.test.ts +159 tests/routing/routing-capability-catalog.test.ts +1665 tests/routing/routing-capability-model-matching.test.ts +127 tests/routing/routing-compatibility-auth-identity.test.ts +31008 tests/routing/routing-compatibility-boundaries.test.ts +366 tests/routing/routing-compatibility-model-matching.test.ts +15852 tests/routing/routing-compatibility.test.ts +172 tests/routing/routing-identity-domains.test.ts +1157 tests/routing/routing-policy-dispatch-scope.test.ts +397 tests/routing/routing-policy-fallback.test.ts +113 tests/routing/routing-policy-pool-quota.test.ts +16023 tests/routing/routing-policy-surface-parity.test.ts +30646 tests/routing/routing-profile-management-editor.test.ts +2559 tests/routing/routing-profile.test.ts +509 tests/routing/subagent-context-staleness.test.ts +142 tests/routing/subagent-defaults.test.ts +24386 tests/routing/subagent-fallback-handle-responses.test.ts +126 tests/routing/subagent-fallback-preview-sites.test.ts +567 tests/routing/subagent-model-fallback-api.test.ts +26150 tests/routing/subagent-model-fallback.test.ts +3900 tests/routing/subagent-roster-migration.test.ts +2188 tests/routing/subagent-roster-retention.test.ts +1870 tests/server/account-import.test.ts +52368 tests/server/account-pool-management-api.test.ts +302 tests/server/adapter-resolve.test.ts +1887 tests/server/agent-task-recovery-cache.test.ts +3191 tests/server/agent-task-recovery-combo.test.ts +856 tests/server/agent-task-recovery-fallback.test.ts +1439 tests/server/agent-task-recovery-security.test.ts +16316 tests/server/agent-task-recovery.test.ts +238 tests/server/alias-management-api.test.ts +3316 tests/server/anthropic2-management.test.ts +143 tests/server/api-access-endpoints.test.ts +6182 tests/server/api-catalog-route.test.ts +2743 tests/server/api-codex-log-guard-compact.test.ts +1977 tests/server/api-codex-log-guard-protection.test.ts +710 tests/server/api-codex-log-guard.test.ts +17204 tests/server/api-debug.test.ts +31173 tests/server/api-key-attribution.test.ts +810 tests/server/api-key-model-scope.test.ts +1825 tests/server/api-key-scope-alpha-search.test.ts +722 tests/server/api-key-scope-audio.test.ts +625 tests/server/api-key-scope-images.test.ts +238 tests/server/api-key-scope-live.test.ts +58917 tests/server/api-keys-routes.test.ts +39495 tests/server/api-usage.test.ts +8717 tests/server/aside-profiles-routes.test.ts +191 tests/server/audio-client.test.ts +16689 tests/server/audio-dictation.test.ts +22912 tests/server/audio-transcriptions.test.ts +947 tests/server/bounded-body.test.ts +422 tests/server/bridge-live-delivery.test.ts +1109 tests/server/caller-session-identity.test.ts +2201 tests/server/caller-session-ingress.test.ts +3517 tests/server/cancel-body-on-abort.test.ts +15591 tests/server/claude-intercept-integration.test.ts +1937 tests/server/codex-agent-role-routes.test.ts +827 tests/server/codex-role-auto-assign-routes.test.ts +129 tests/server/combo-requested-effort.test.ts +620 tests/server/companion-settings.test.ts +8310 tests/server/config.test.ts +213 tests/server/consume-for-inspection-cancel.test.ts +7443 tests/server/context-history-ownership.test.ts +301 tests/server/context-history.test.ts +2014 tests/server/data-plane-admission-identity.test.ts +143 tests/server/debug-settings.test.ts +142 tests/server/decision-discovery.test.ts +316 tests/server/decision-routes.test.ts +383 tests/server/error-fidelity.test.ts +173 tests/server/errors-adapter-failure.test.ts +353 tests/server/fetch-header-timeout.test.ts +11282 tests/server/health-scoring.test.ts +2521 tests/server/hub-usage.test.ts +176 tests/server/inference-attempt.test.ts +284 tests/server/inference-client-encoder-delivery.test.ts +239 tests/server/inference-client-wire.test.ts +138 tests/server/inference-final-log.test.ts +3553 tests/server/inference-send-budget.test.ts +141 tests/server/input-admission.test.ts +271 tests/server/jev-decision-scope.test.ts +3778 tests/server/link-enrollment.test.ts +2581 tests/server/link-join-route.test.ts +6598 tests/server/link-listener-admission.test.ts +634 tests/server/link-listener-lifecycle.test.ts +410 tests/server/link-listener-mutation.test.ts +4563 tests/server/link-management-routes.test.ts +143 tests/server/live-call-bindings.test.ts +95 tests/server/live-native-calls.test.ts +443 tests/server/local-account-switch-capability.test.ts +2233 tests/server/local-account-switch-ingress.test.ts +3030 tests/server/local-aside-sync-capability.test.ts +240 tests/server/local-management-attestation.test.ts +208 tests/server/local-management-capability.test.ts +2963 tests/server/local-management-direct-transport.test.ts +359 tests/server/local-provider-reload-client.test.ts +2065 tests/server/local-read-response-proof.test.ts +206 tests/server/logs-timezone.test.ts +421 tests/server/loopback-companion-client-targets.test.ts +190 tests/server/loopback-listener-admission.test.ts +73964 tests/server/loopback-listener-integration.test.ts +1805 tests/server/management-anthropic-reset-grants.test.ts +460 tests/server/management-api-logs-metrics.test.ts +583 tests/server/management-api-logs-timing-basis.test.ts +891 tests/server/management-client-config-route.test.ts +714 tests/server/management-droid-reasoning-defaults.test.ts +2017 tests/server/management-google-tool-schema-policy.test.ts +1120 tests/server/management-inference-port.test.ts +3436 tests/server/management-integration-journal-delete.test.ts +12120 tests/server/management-integration-routes.test.ts +23767 tests/server/management-metrics-export.test.ts +465 tests/server/management-model-roster-gather-race.test.ts +206 tests/server/management-model-roster.test.ts +236 tests/server/management-origin-tls.test.ts +5073 tests/server/management-provider-atomicity.test.ts +29922 tests/server/management-provider-compat-carry.test.ts +6540 tests/server/management-provider-hide-raw-reasoning.test.ts +2203 tests/server/management-provider-pinsless-validation.test.ts +3396 tests/server/management-provider-proto-override.test.ts +7459 tests/server/management-provider-reasoning-lists.test.ts +1207 tests/server/management-provider-request-pacing.test.ts +111 tests/server/management-provider-reset-replay.test.ts +1202 tests/server/management-provider-synthetic-max.test.ts +1982 tests/server/management-provider-upstream-websocket.test.ts +105607 tests/server/management-provider-validation.test.ts +2301 tests/server/management-provider-verbosity.test.ts +449 tests/server/management-route-registry.test.ts +14131 tests/server/management-sidecar-anthropic-instance.test.ts +347 tests/server/management-workflow-budget-routes.test.ts +608 tests/server/memory-watchdog.test.ts +1314 tests/server/model-costs-management-api.test.ts +191 tests/server/model-discovery-management-api.test.ts +3615 tests/server/model-export-new-model-policy.test.ts +2745 tests/server/model-settings-management-api.test.ts +128 tests/server/outbound-body-guard.test.ts +2879 tests/server/owned-service-home.test.ts +127 tests/server/passive-route-linker.test.ts +30966 tests/server/plaintext-v2-agent-messages-server.test.ts +1187 tests/server/port-reclaim.test.ts +521 tests/server/ports.test.ts +4874 tests/server/probe-timeout-env.test.ts +588 tests/server/protocol-provider-summary.test.ts +208 tests/server/protocol-routes.test.ts +192 tests/server/protocol-settings-route.test.ts +968 tests/server/provider-account-quota-routes.test.ts +89 tests/server/provider-egress-management-validation.test.ts +1206 tests/server/provider-send-path-import.test.ts +209 tests/server/proxy-env-macos.test.ts +380 tests/server/proxy-env.test.ts +145 tests/server/proxy-liveness-package-tree-fence.test.ts +332 tests/server/proxy-liveness.test.ts +918 tests/server/relay-eager.test.ts +11141 tests/server/replay-refusal-parity.test.ts +1743 tests/server/reserve-claude-policy.test.ts +12235 tests/server/reserve-ingress.test.ts +794 tests/server/response-log-inspection.test.ts +1225 tests/server/response-model-identity.test.ts +237 tests/server/restart-replacement.test.ts +651 tests/server/retry-after-429.test.ts +146 tests/server/retry-delay-hardening.test.ts +314 tests/server/route-decision-trace.test.ts +4273 tests/server/server-403-permission-e2e.test.ts +1616 tests/server/server-agent-task-recovery-replay.test.ts +349 tests/server/server-antigravity-verify-replay-scope.test.ts +21282 tests/server/server-antigravity-web-search-validation.test.ts +1013 tests/server/server-auth-localhost-bind.test.ts +8151 tests/server/server-auth-scoped-quota.test.ts +166990 tests/server/server-auth.test.ts +14933 tests/server/server-background-lifecycle.test.ts +1948 tests/server/server-cli-key-probe-admission.test.ts +117 tests/server/server-clickjacking-headers.test.ts +1131 tests/server/server-combo-cooldown-fallback.test.ts +1469 tests/server/server-combo-cooldown-recording.test.ts +61563 tests/server/server-combo-failover-e2e.test.ts +116 tests/server/server-combo-held-response.test.ts +13621 tests/server/server-combo-plan-model-refusal.test.ts +4923 tests/server/server-combo-reasoning-replay-eligibility.test.ts +3193 tests/server/server-combo-zero-output-failover.test.ts +100094 tests/server/server-google-antigravity-oauth-401-replay.test.ts +14209 tests/server/server-google-antigravity-oauth-429-budget.test.ts +97 tests/server/server-gui-bundle-freshness.test.ts +190 tests/server/server-images-bodyless-content-length.test.ts +4132 tests/server/server-images-pool-admission.test.ts +47342 tests/server/server-images.test.ts +444 tests/server/server-jev-combo-e2e.test.ts +221 tests/server/server-jev-model-decision-e2e.test.ts +51941 tests/server/server-key-failover-e2e.test.ts +24660 tests/server/server-kiro-completion-e2e.test.ts +1978 tests/server/server-kiro-device-login.test.ts +7822 tests/server/server-kiro-oauth-401-replay.test.ts +41320 tests/server/server-kiro-refusal-e2e.test.ts +3677 tests/server/server-live-existing-call.test.ts +413 tests/server/server-live-frame-log.test.ts +8565 tests/server/server-live-realtime-fixtures.test.ts +36487 tests/server/server-live.test.ts +109 tests/server/server-loopback-host-gate.test.ts +37140 tests/server/server-management-auth.test.ts +2837 tests/server/server-management-log-admission.test.ts +8175 tests/server/server-new-model-policy-arrival.test.ts +3688 tests/server/server-opencode-go-goal-streaming.test.ts +9373 tests/server/server-rate-limit-retry-e2e.test.ts +4063 tests/server/server-request-body-size.test.ts +18704 tests/server/server-search.test.ts +6158 tests/server/server-startup-reconcile-resilience.test.ts +5463 tests/server/server-stop-config-hardening.test.ts +2637 tests/server/server-xai-chat-reasoning-streaming.test.ts +4998 tests/server/server-xai-header-parity.test.ts +12122 tests/server/server-xai-oauth-401-replay.test.ts +6748 tests/server/server-xai-responses-streaming.test.ts +2776 tests/server/session-affinity.test.ts +2909 tests/server/session-lane-recall-harness.test.ts +508 tests/server/sidebar-routes.test.ts +210 tests/server/sidebar-star-state.test.ts +127 tests/server/spend-instrumentation-log.test.ts +1255 tests/server/spend-ledger-lifecycle.test.ts +3984 tests/server/spend-ledger-owner-startup.test.ts +191 tests/server/startup-action-control-elevation.test.ts +247 tests/server/startup-action-control.test.ts +33636 tests/server/startup-health-packaged-probe.test.ts +127 tests/server/startup-prompt.test.ts +428 tests/server/stream-aborted-marker.test.ts +1748 tests/server/subagent-model-force-api.test.ts +779 tests/server/system-env.test.ts +285 tests/server/system-restart-admission.test.ts +537 tests/server/system-restart.test.ts +254 tests/server/system-routes.test.ts +5949 tests/server/terminal-guard-server.test.ts +208 tests/server/terminal-guard.test.ts +574 tests/server/update-async-routes.test.ts +156 tests/server/upstream-connect-error.test.ts +127 tests/server/upstream-http-version.test.ts +8946 tests/server/v1-hub-state.test.ts +2523 tests/server/v2-agent-message-failfast.test.ts +160 tests/service/autostart-health.test.ts +12949 tests/service/container-bootstrap.test.ts +302 tests/service/crash-guard.test.ts +304 tests/service/doctor-codex-envkey-readiness.test.ts +1016 tests/service/doctor-oauth.test.ts +174 tests/service/doctor-provider-apikey.test.ts +127 tests/service/doctor-service-memory-contract.test.ts +222 tests/service/init-backup-cleanup.test.ts +10339 tests/service/init-eof.test.ts +8814 tests/service/launchd-repair.test.ts +476 tests/service/live-service-manager-guard.test.ts +128 tests/service/managing-cli.test.ts +186 tests/service/ownership-mutation-lease-diagnostics.test.ts +1112 tests/service/process-state.test.ts +1637 tests/service/service-auth-qualified-localhost.test.ts +492 tests/service/service-child-ownership.test.ts +386 tests/service/service-claim.test.ts +110 tests/service/service-desktop-runtime-preflight.test.ts +395 tests/service/service-desktop-startup-health.test.ts +162 tests/service/service-desktop-startup-linux.test.ts +1475 tests/service/service-desktop-startup.test.ts +140 tests/service/service-handled-signal-exit.test.ts +343 tests/service/service-health-sentinel-launch.test.ts +160 tests/service/service-health-sentinel-policy.test.ts +11109 tests/service/service-health-sentinel-process.test.ts +1268 tests/service/service-health-sentinel-runner.test.ts +301 tests/service/service-multishell-path.test.ts +177 tests/service/service-ownership-compatibility.test.ts +823 tests/service/service-ownership-handover.test.ts +6468 tests/service/service-ownership-state.test.ts +142 tests/service/service-probe-docker.test.ts +761 tests/service/service-runtime-preflight.test.ts +4665 tests/service/service-secrets.test.ts +421 tests/service/service-sqlite-home.test.ts +127 tests/service/service-start-environment.test.ts +174 tests/service/service-stop-verification.test.ts +2354 tests/service/service-tier-capability.test.ts +433 tests/service/service-wsl-home-ownership.test.ts +3149 tests/service/service.test.ts +285 tests/service/shutdown-drain.test.ts +143 tests/service/shutdown-launcher.test.ts +156 tests/service/stale-state-purge.test.ts +324 tests/service/standalone-service.test.ts +12835 tests/service/stop-deferred-teardown.test.ts +145 tests/service/systemd-install-cleanup-hardening.test.ts +508 tests/service/winsw.test.ts +26413 tests/storage/api-storage-cleanup.test.ts +6489 tests/storage/api-storage-policy-already-running.test.ts +6522 tests/storage/api-storage-policy-mutation-busy.test.ts +5939 tests/storage/api-storage-policy-put-race.test.ts +7319 tests/storage/api-storage-policy-run.test.ts +5376 tests/storage/api-storage-policy.test.ts +2934 tests/storage/api-storage.test.ts +31100 tests/storage/storage-cleanup.test.ts +31040 tests/storage/storage-mutation-race.test.ts +2572 tests/storage/storage-policy-config-race.test.ts +6050 tests/storage/storage-policy-job-responsive.test.ts +256472 tests/storage/storage-policy.test.ts +184 tests/storage/storage-restore-job-errors.test.ts +11729 tests/storage/storage-restore-job-responsive.test.ts +2014 tests/storage/storage-scanner.test.ts +5111 tests/storage/storage-worker-lifecycle.test.ts +128 tests/storage/storage-worker-os-join-settle.test.ts +25046 tests/storage/storage-worker-teardown-isolate.test.ts +1657 tests/test-layout-tooling.test.ts +432 tests/test-layout.test.ts +159 tests/update/pnpm-command-isolation.test.ts +110 tests/update/update-badge.test.ts +563 tests/update/update-bun-ownership-lease.test.ts +111 tests/update/update-desktop-badge.test.ts +127 tests/update/update-desktop-owner.test.ts +11455 tests/update/update-job.test.ts +127 tests/update/update-mise-launcher-target.test.ts +165 tests/update/update-mise-node-runtime.test.ts +732 tests/update/update-mise.test.ts +583 tests/update/update-notify.test.ts +493 tests/update/update-npm-cache-preflight.test.ts +110 tests/update/update-npm-invocation.test.ts +302 tests/update/update-pnpm.test.ts +617 tests/update/update-refresh.test.ts +58556 tests/update/update-restart-lease.test.ts +2811 tests/update/update-stop-classification.test.ts +392 tests/update/update-stop-first.test.ts +1761 tests/update/update-transactional-leftovers.test.ts +350 tests/update/update-transactional.test.ts +338 tests/update/update-tray-handoff.test.ts +364 tests/update/update-tree-ownership.test.ts +110 tests/update/update-worker-launch.test.ts +111 tests/usage/anthropic2-usage-attribution.test.ts +239 tests/usage/cache-diagnostic.test.ts +807 tests/usage/cost-cap-unknown-evidence.test.ts +269 tests/usage/cost-scoring.test.ts +251 tests/usage/jev-stats.test.ts +507 tests/usage/key-attribution.test.ts +8450 tests/usage/quota-401-recovery-runtime.test.ts +142 tests/usage/quota-401-recovery.test.ts +490 tests/usage/quota-reset-account-key.test.ts +762 tests/usage/quota-reset-core-boundary.test.ts +98 tests/usage/quota-reset-detector.test.ts +161 tests/usage/quota-reset-notify-config.test.ts +1316 tests/usage/quota-reset-notify.test.ts +2848 tests/usage/quota-reset-observation.test.ts +2571 tests/usage/quota-reset-seen-store.test.ts +1192 tests/usage/quota-scoring.test.ts +968 tests/usage/request-decompress.test.ts +96 tests/usage/request-evidence.test.ts +11246 tests/usage/request-history-index.test.ts +220 tests/usage/request-log-attempt-identity.test.ts +161 tests/usage/request-log-conversation.test.ts +461 tests/usage/request-log-estimate-cap.test.ts +381 tests/usage/request-log-generation-window.test.ts +206 tests/usage/request-log-jev.test.ts +112 tests/usage/request-log-model-filter.test.ts +313 tests/usage/request-log-nonstream.test.ts +157 tests/usage/request-log-protocol-trace.test.ts +348 tests/usage/request-log-served-model.test.ts +1503 tests/usage/request-log.test.ts +206 tests/usage/request-outcome-agreement.test.ts +318 tests/usage/request-pacing.test.ts +25796 tests/usage/upstream-failure-diagnostics.test.ts +416 tests/usage/usage-aggregate-cache.test.ts +99 tests/usage/usage-anthropic-fast-pricing.test.ts +126 tests/usage/usage-antigravity-55.test.ts +142 tests/usage/usage-attempt-delivery.test.ts +126 tests/usage/usage-cost.test.ts +127 tests/usage/usage-cursor-fast-pricing.test.ts +454 tests/usage/usage-debug.test.ts +250 tests/usage/usage-failure-fingerprint.test.ts +170 tests/usage/usage-failure-persistence.test.ts +112 tests/usage/usage-failure-projection.test.ts +122 tests/usage/usage-haiku-5-5-pricing.test.ts +868 tests/usage/usage-ledger-retention.test.ts +428 tests/usage/usage-ledger-scanner.test.ts +254 tests/usage/usage-log-ws-stage.test.ts +3666 tests/usage/usage-log.test.ts +115 tests/usage/usage-provider-label.test.ts +778 tests/usage/usage-shape-extraction.test.ts +143 tests/usage/usage-snapshot-digest-reuse.test.ts +174 tests/usage/usage-spend-cache-provenance.test.ts +300 tests/usage/usage-summary.test.ts +174 tests/usage/usage-surfaces.test.ts +175 tests/usage/usage-throughput.test.ts +94 tests/usage/usage-time-range.test.ts +129 tests/usage/usage-timeline.test.ts +1125 tests/usage/user-cost-overlay-coderabbit-regressions.test.ts +6190 tests/usage/user-cost-overlay-live-reconcile.test.ts +953 tests/usage/user-cost-overlay-provider-delete.test.ts +313 tests/videos/fulfill-video.test.ts +110 tests/videos/plan-video.test.ts +142 tests/videos/xai-video-client.test.ts +443 tests/vision/sidecar-abort.test.ts +224 tests/vision/sidecar-auth.test.ts +289 tests/vision/sidecar-candidates.test.ts +2508 tests/vision/sidecar-settings-vision-controls.test.ts +1790 tests/vision/sidecar-settings-vision-filter.test.ts +2242 tests/vision/sidecar-settings-web-search-gate.test.ts +1972 tests/vision/sidecar-settings-web-search-off.test.ts +1606 tests/vision/sidecar-settings-web-search-stream.test.ts +295 tests/vision/sidecar-tracker.test.ts +8055 tests/vision/vision-anthropic-instance-sidecar.test.ts +1046 tests/vision/vision-anthropic.test.ts +1013 tests/vision/vision-backend-union.test.ts +176 tests/vision/vision-cache.test.ts +158 tests/vision/vision-custom-row-precedence.test.ts +206 tests/vision/vision-eligibility.test.ts +269 tests/vision/vision-fail-closed.test.ts +1796 tests/vision/vision-reasoning-contract.test.ts +3840 tests/vision/vision-routed.test.ts +12699 tests/vision/vision-sidecar-e2e.test.ts +161 tests/vision/vision-text-only-predicate.test.ts +523 tests/web-search/devin-web-search.test.ts +143 tests/web-search/format-result.test.ts +5571 tests/web-search/web-search-anthropic-instance.test.ts +426 tests/web-search/web-search-anthropic.test.ts +1587 tests/web-search/web-search-backend-union.test.ts +222 tests/web-search/web-search-bridge-replay.test.ts +127 tests/web-search/web-search-candidates.test.ts +159 tests/web-search/web-search-forced-declaration.test.ts +2543 tests/web-search/web-search-pacing-deadline.test.ts +178 tests/web-search/web-search-parse.test.ts +3447 tests/web-search/web-search-passthrough-bridge.test.ts +2653 tests/web-search/web-search-progress-stream.test.ts +359 tests/web-search/web-search-recovery-kind.test.ts +1695 tests/web-search/web-search-retry-heartbeat.test.ts +160 tests/web-search/web-search-run-turn-loop.test.ts +7760 tests/web-search/web-search-sidecar-429.test.ts +143 tests/web-search/web-search-sources.test.ts +999 tests/web-search/web-search-timeout-contract.test.ts +143 tests/web-search/web-search-timeout-plan.test.ts +5550 tests/web-search/web-search.test.ts +216 tests/windows/tray-proxy-deadline.test.ts +2586 tests/windows/tray-proxy.test.ts +127 tests/windows/win-exec.test.ts +125 tests/windows/win-paths.test.ts +623 tests/windows/windows-acl-start-cost.test.ts +100 tests/windows/windows-atomic-replace.test.ts +159 tests/windows/windows-deploy-close-regressions.test.ts +4914 tests/windows/windows-elevation-spawn.test.ts +345 tests/windows/windows-elevation.test.ts +196 tests/windows/windows-guarded-manager.test.ts +776 tests/windows/windows-popup-fix.test.ts +158 tests/windows/windows-process-priority.test.ts +240 tests/windows/windows-scheduler-install-verification.test.ts +207 tests/windows/windows-secret-acl-removal-flight.test.ts +22864 tests/windows/windows-secret-acl.test.ts +285 tests/windows/windows-service-mutation-lock.test.ts +1777 tests/windows/windows-service-wrappers.test.ts +81 tests/windows/windows-text-decoding.test.ts +125 tests/windows/windows-tray-restart-hardening.test.ts +296 tests/windows/windows-tray-run-limit.test.ts +11227 tests/windows/windows-tray.test.ts +534 tests/windows/windows-user-principal-nonascii.test.ts +520 tests/windows/windows-user-principal.test.ts +127 tests/windows/winsw-stop-hardening.test.ts diff --git a/structure/ops/cross-platform-ci.md b/structure/ops/cross-platform-ci.md index 5d52b0b89d4..09640f00b86 100644 --- a/structure/ops/cross-platform-ci.md +++ b/structure/ops/cross-platform-ci.md @@ -56,12 +56,16 @@ manager guards remain active because the preload installs them before the lock b Test teardown follows the [sandbox cleanup contract](test-sandbox-cleanup.md). `tests/preload.ts` resolves cleanup dependencies after home/lock admission and before test cases; teardown awaits native-main startup releases and config hardening, then the sandbox's registered ACL child reaps before removing that root. Its synchronous exit fallback leaves an undrained root for ownership-checked stale recovery instead of blocking child cleanup with removal retries. `tests/ci-workflows/test-sandbox-cleanup.test.ts` pins that ordering with a delayed reap. `tests/helpers/test-sandbox-cleanup.ts` exposes case-scoped lifecycle ownership: cancellation starts listener stops while owned asynchronous work settles, and repeated close/stop calls share one promise. After teardown starts, only the lifecycle's own abort reason is absorbed; any other error, including a foreign AbortError, still fails its case. Callers settle that lifecycle before draining producers/reaps and restoring or removing a home. The helper does not replace fixture-specific cleanup or claim OS ACL coverage for synthetic tests. -Shard membership follows recorded duration: `scripts/ci/test-durations.tsv` weighs each file, -the heaviest file goes to the least-loaded shard, and a file without a row weighs the table's +Shard membership follows recorded duration: `scripts/ci/test-durations.tsv` weighs each file on +Linux and macOS and `scripts/ci/test-durations.windows.tsv` on Windows (selected by `RUNNER_OS`; +`BUN_TEST_DURATIONS_FILE` overrides both), because Windows per-file cost is not a constant multiple +of the other two and one table cannot balance all three (#6982). The heaviest file goes to the +least-loaded shard, and a file without a row weighs the table's median, so an empty table reproduces sorted round-robin exactly. Every shard computes the whole assignment and refuses to run unless it covers every selected file once. A batch also closes before its predicted duration passes half the process timeout, which only adds process -boundaries. `scripts/ci/test-durations.ts refresh` regenerates the table from hosted job logs. +boundaries. `scripts/ci/test-durations.ts refresh` regenerates each table from its own platforms' +hosted job logs (`--table` selects the Windows table). A test failure, a process timeout and a Bun runtime crash each fail their job on the first occurrence; the batch runner still sweeps a crashed or timed-out batch one file per process, but only to attribute a failure the shard has already taken. The aggregate `ci` gate derives, from the event and the `changes` outputs, which diff --git a/structure/ops/docs-and-release.md b/structure/ops/docs-and-release.md index c14848ad56e..f1fc58b6b17 100644 --- a/structure/ops/docs-and-release.md +++ b/structure/ops/docs-and-release.md @@ -236,7 +236,7 @@ CI setup uses the [Bun runtime and test-runner pins](../runtime.md#bun-runtime-a `scripts/test.ts` owns `SERIAL_FULL_SUITE_FILES`, the shared process-isolation roster. Local full-suite runs, both macOS paths, and `scripts/ci/run-bun-test-batches.sh` execute those files alone with fresh process homes, including launchd repair and standalone home/lease cases. Hosted batches assign shard membership by the per-file durations -in `scripts/ci/test-durations.tsv` (sorted round-robin when nothing is recorded), run each shard's +in `scripts/ci/test-durations.tsv` (Windows: `scripts/ci/test-durations.windows.tsv`; sorted round-robin when nothing is recorded), run each shard's files in sorted order and split only process boundaries; every selected file still runs once. Ordinary macOS shards select 1/3, 2/3 and 3/3 from the full file list; macOS control selects 1/1. Both execute sequential batches of at most 12 files with one worker. diff --git a/tests/ci-workflows/ci-crash-disposition.test.ts b/tests/ci-workflows/ci-crash-disposition.test.ts index c4d8e090233..9f38558ab48 100644 --- a/tests/ci-workflows/ci-crash-disposition.test.ts +++ b/tests/ci-workflows/ci-crash-disposition.test.ts @@ -212,6 +212,11 @@ function runBatches( for (const file of FIXTURE_FILES) writeFileSync(join(directory, "tests", file), ""); writeFileSync(join(directory, "tests", DEDICATED_FILE), ""); for (const file of options.additionalFiles ?? []) writeFileSync(join(directory, "tests", file), ""); + // Pin the weights the runner shards and batches by. Without this every fixture file weighs the + // median of the committed scripts/ci/test-durations.tsv, and the 1-second cases' 500 ms batch + // budget would move with each refresh of that table (#6982). + const durations = join(directory, "durations.tsv"); + writeFileSync(durations, FIXTURE_FILES.map(file => `100\ttests/${file}\n`).join("")); writeFileSync( join(binDirectory, "timeout"), options.timeoutTool === "non-gnu" ? FAKE_NON_GNU_TIMEOUT : FAKE_TIMEOUT, @@ -233,6 +238,7 @@ function runBatches( ...(options.batchTimeoutSeconds === undefined ? {} : { BUN_TEST_BATCH_TIMEOUT_SECONDS: options.batchTimeoutSeconds }), ...(options.killGraceSeconds === undefined ? {} : { BUN_TEST_BATCH_KILL_GRACE_SECONDS: options.killGraceSeconds }), BUN_TEST_FILE_SCOPE: fileScope, + BUN_TEST_DURATIONS_FILE: durations, ...(options.parallel === undefined ? {} : { BUN_TEST_PARALLEL: options.parallel }), OCX_TEST_NO_QUEUE: "1", OPENCODEX_BUN_PATH: join(binDirectory, "bun"), diff --git a/tests/ci-workflows/ci-shard-balance.test.ts b/tests/ci-workflows/ci-shard-balance.test.ts index 42fca7d7c41..a96e5d64d07 100644 --- a/tests/ci-workflows/ci-shard-balance.test.ts +++ b/tests/ci-workflows/ci-shard-balance.test.ts @@ -12,7 +12,7 @@ import { describe, expect, test } from "bun:test"; import { mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { delimiter, join } from "node:path"; -import { mergeDurations, parseJobLog, parseTable, renderTable } from "../../scripts/ci/test-durations"; +import { DURATIONS_TABLE, mergeDurations, parseJobLog, parseTable, renderTable, WINDOWS_DURATIONS_TABLE } from "../../scripts/ci/test-durations"; import { removeTreeWithRetry } from "../helpers/remove-tree"; import { repoPath } from "../helpers/repo-root"; import { SPAWN_BUDGET_MS } from "../helpers/test-budget"; @@ -40,7 +40,20 @@ const FAKE_TIMEOUT = [ "", ].join("\n"); -function runShard(shard: string, options: { durations?: Record | null; batchSize?: string; timeoutSeconds?: string } = {}): string[][] { +type ShardOptions = { + durations?: Record | null; + batchSize?: string; + timeoutSeconds?: string; + /** Leave BUN_TEST_DURATIONS_FILE unset so the runner picks its own committed table. */ + defaultTable?: boolean; + env?: Record; +}; + +function runShard(shard: string, options: ShardOptions = {}): string[][] { + return runShardWithOutput(shard, options).batches; +} + +function runShardWithOutput(shard: string, options: ShardOptions = {}): { batches: string[][]; output: string } { const directory = mkdtempSync(join(tmpdir(), "ocx-shard-balance-")); try { const bin = join(directory, "bin"); @@ -68,17 +81,19 @@ function runShard(shard: string, options: { durations?: Record | TMPDIR: join(directory, "tmp"), BUN_TEST_BATCH_SIZE: options.batchSize ?? "12", ...(options.timeoutSeconds ? { BUN_TEST_BATCH_TIMEOUT_SECONDS: options.timeoutSeconds } : {}), - BUN_TEST_DURATIONS_FILE: durations, + ...(options.defaultTable ? {} : { BUN_TEST_DURATIONS_FILE: durations }), OPENCODEX_BUN_PATH: join(bin, "bun"), FIXTURE_ISOLATED: ISOLATED, FIXTURE_CALLS: calls, + ...options.env, }, stdout: "pipe", stderr: "pipe", }); const output = `${result.stdout.toString()}${result.stderr.toString()}`; expect(`status:${result.exitCode}`, output).toBe("status:0"); - return readFileSync(calls, "utf8").split("\n").filter(Boolean).map(line => line.split(" ")); + const batches = readFileSync(calls, "utf8").split("\n").filter(Boolean).map(line => line.split(" ")); + return { batches, output }; } finally { removeTreeWithRetry(directory); } @@ -126,6 +141,16 @@ describe.skipIf(process.platform === "win32")("duration-weighted shards, execute const batches = runShard("1/1", { durations, batchSize: "3", timeoutSeconds: "2" }); expect(batches.map(batch => batch.length)).toEqual([3, 3]); }, SPAWN_BUDGET_MS); + + test("Windows runners default to their own table, and an explicit table still wins", () => { + const tableUsed = (env: Record, defaultTable = true) => + /Predicted shard time from recorded durations \(([^)]+)\)/ + .exec(runShardWithOutput("1/1", { defaultTable, env, durations: { "alpha.test.ts": 1 } }).output)?.[1]; + expect(tableUsed({ RUNNER_OS: "Windows" })).toBe("test-durations.windows.tsv"); + expect(tableUsed({ RUNNER_OS: "Linux" })).toBe("test-durations.tsv"); + expect(tableUsed({ RUNNER_OS: "macOS" })).toBe("test-durations.tsv"); + expect(tableUsed({ RUNNER_OS: "Windows" }, false)).toBe("durations.tsv"); + }, SPAWN_BUDGET_MS); }); describe("the duration table tool", () => { @@ -163,6 +188,38 @@ describe("the duration table tool", () => { expect(samples.get("tests/b/two.test.ts")).toEqual([400]); }); + test("reads Windows shard headers, which print backslash paths, under forward-slash keys", () => { + const windows = [ + "windows 8/9\tUNKNOWN STEP\t2026-10-12T08:52:18.9199935Z ##[group]shard 8/9 batch 1/45 (6 files)", + "windows 8/9\tUNKNOWN STEP\t2026-10-12T08:52:18.9851789Z ##[group]tests\\adapters\\anthropic\\one.test.ts:", + "windows 8/9\tUNKNOWN STEP\t2026-10-12T08:52:19.5906751Z ##[endgroup]", + ].join("\r\n"); + expect(Object.fromEntries(parseJobLog(windows))).toEqual({ "tests/adapters/anthropic/one.test.ts": [671] }); + }); + + test("refresh --table writes the named table and leaves the shared one alone", () => { + const directory = mkdtempSync(join(tmpdir(), "ocx-durations-table-")); + try { + const log = join(directory, "windows.log"); + const table = join(directory, "windows.tsv"); + writeFileSync(log, [ + "2026-10-12T08:52:18.9199935Z ##[group]shard 8/9 batch 1/45 (1 files)", + "2026-10-12T08:52:18.9851789Z ##[group]tests\\ci-workflows\\ci-shard-balance.test.ts:", + "2026-10-12T08:52:19.5906751Z ##[endgroup]", + ].join("\r\n")); + const shared = readFileSync(DURATIONS_TABLE, "utf8"); + const result = Bun.spawnSync( + [process.execPath, repoPath("scripts", "ci", "test-durations.ts"), "refresh", "--table", table, "--source", "fixture", log], + { stdout: "pipe", stderr: "pipe" }, + ); + expect(result.exitCode, result.stderr.toString()).toBe(0); + expect(parseTable(readFileSync(table, "utf8"))).toEqual(new Map([["tests/ci-workflows/ci-shard-balance.test.ts", 671]])); + expect(readFileSync(DURATIONS_TABLE, "utf8")).toBe(shared); + } finally { + removeTreeWithRetry(directory); + } + }, SPAWN_BUDGET_MS); + test("merges new measurements over kept rows and drops files that no longer exist", () => { const previous = new Map([["tests/kept.test.ts", 70], ["tests/gone.test.ts", 90], ["tests/remeasured.test.ts", 5]]); const measured = new Map([["tests/remeasured.test.ts", [100, 300]], ["tests/zero.test.ts", [0]]]); @@ -171,13 +228,15 @@ describe("the duration table tool", () => { expect(parseTable(renderTable(merged, "fixture"))).toEqual(merged); }); - test("the committed table is well formed", () => { - const text = readFileSync(repoPath("scripts", "ci", "test-durations.tsv"), "utf8"); - const rows = text.split("\n").filter(line => line !== "" && !line.startsWith("#")); - expect(rows.length).toBeGreaterThan(0); - for (const row of rows) expect(row).toMatch(/^\d+\ttests\/\S+$/); - const paths = rows.map(row => row.split("\t")[1]!); - expect(paths).toEqual([...new Set(paths)].sort((a, b) => (a < b ? -1 : a > b ? 1 : 0))); - expect(parseTable(text).size).toBe(rows.length); - }); + for (const table of [DURATIONS_TABLE, WINDOWS_DURATIONS_TABLE]) { + test(`the committed table ${table.split(/[\\/]/).pop()} is well formed`, () => { + const text = readFileSync(table, "utf8"); + const rows = text.split("\n").filter(line => line !== "" && !line.startsWith("#")); + expect(rows.length).toBeGreaterThan(0); + for (const row of rows) expect(row).toMatch(/^\d+\ttests\/\S+$/); + const paths = rows.map(row => row.split("\t")[1]!); + expect(paths).toEqual([...new Set(paths)].sort((a, b) => (a < b ? -1 : a > b ? 1 : 0))); + expect(parseTable(text).size).toBe(rows.length); + }); + } }); From ed07f0f49ded337842e00c4a543821ff9ad75ffb Mon Sep 17 00:00:00 2001 From: lidge-jun Date: Tue, 13 Oct 2026 01:09:29 +0900 Subject: [PATCH 3/9] test(web-search): run the pacing deadline cases on a virtual clock (#6988) (#6993) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Squashed from #6993 for the 2.84.0 release train. Fixes the Windows-shard flake in `tests/web-search/web-search-pacing-deadline.test.ts` (#6988) by running the file on a virtual clock. The source is unchanged: it was traced and is correct. **Why it failed.** The cases measured small header budgets on the real clock. In the reset case the 120–180 ms reconnect backoff deliberately counts against a 300 ms budget, and only the 900 ms pacing wait is paused out of it. In the rotation case the first send spends 70 of 110 ms, leaving about 40 ms. On a loaded Windows runner, scheduler delay used up that slack before the pacing pause took over: - the reset replay returned 504 after 483 ms with a single send; - the rotation case never started its second send (`started=1`). Both results are correct for the real elapsed time. Blocking the event loop locally reproduces them exactly: a 400 ms block during the reset backoff gives 504 after 458 ms with one send, and a 45 ms block after the first rotation send gives `sends=1, started=1`. The CI log carries no phase timestamps, so the exact stall location on the runner is inferred. The source trace found no path that counts an exposed pacing wait, re-arms a spent budget, or admits twice. **Change (test only).** - Every case runs on Bun's fake clock (`jest.useFakeTimers()`), which drives `setTimeout`, `Date.now` and `performance.now`, including references captured at import. This holds on the suite runner Bun 1.4.0 and on 1.4.2. - A `settle()` driver drains real `setImmediate` turns, then advances virtual time 1 ms per step, bounded at 10 s of virtual time. A stalled runner therefore cannot spend a budget. - Each case asserts the virtual time its contract implies, allowing up to 20 ms of virtual slack after the expected time while continuations drain: - queued sends and paced replays succeed exactly when their slot opens (180 ms, or 900 ms for the reset case); - a genuine header stall returns 504 at 60 ms; - the rotated send times out at 220 ms, on the 40 ms it has left. A renewed budget would answer at 250 ms; a pacing wait counted against the budget would expire at 110 ms, before the second send started. Closes #6988 --- .../web-search-pacing-deadline.test.ts | 123 +++++++++++++----- 1 file changed, 91 insertions(+), 32 deletions(-) diff --git a/tests/web-search/web-search-pacing-deadline.test.ts b/tests/web-search/web-search-pacing-deadline.test.ts index 0776e49f18f..92571316a8f 100644 --- a/tests/web-search/web-search-pacing-deadline.test.ts +++ b/tests/web-search/web-search-pacing-deadline.test.ts @@ -1,4 +1,4 @@ -import { afterEach, describe, expect, test } from "bun:test"; +import { afterEach, beforeEach, describe, expect, jest, test } from "bun:test"; import type { ProviderAdapter } from "../../src/adapters/base"; import { createAdapterPhysicalSend } from "../../src/adapters/physical-send"; import { sleepWithAbort } from "../../src/lib/upstream-retry"; @@ -13,8 +13,14 @@ import type { OcxProviderConfig } from "../../src/types"; import { runWithWebSearch, type WebSearchLoopDeps } from "../../src/web-search/loop"; import { createTestTranslatorBudget } from "../helpers/translator-budget"; +// Every deadline, pacing interval and backoff here runs on Bun's fake clock, which also drives +// Date.now() and performance.now(). Virtual time moves only in settle(), one millisecond per step +// after the pending work has drained, so a loaded runner that stalls the event loop cannot spend +// a header budget and each case can assert the exact virtual time its contract implies (#6988). const HEADER_MS = 60; const QUEUE_MS = 180; +// Far below every margin the cases distinguish (40ms of rotation budget, a 60ms header budget). +const DRAIN_SLACK_MS = 20; const NAME = "search-pacing-test"; const MODEL = "model"; const provider: OcxProviderConfig = { @@ -22,7 +28,47 @@ const provider: OcxProviderConfig = { requestPacing: { enabled: true, maxConcurrentRequests: 1 }, }; -afterEach(() => resetProviderRequestPacingForTest()); +// Each case's turn signal is aborted in cleanup, and streams a case opened are torn down, so a case that +// fails mid-flight cannot leave a queued send, a pending deadline or an open stream running into the next +// file in the same Bun process. Real timers come back even if that teardown throws. +let turn = new AbortController(); +let teardown: Array<() => unknown> = []; +beforeEach(() => { + turn = new AbortController(); + teardown = []; + jest.useFakeTimers(); +}); +afterEach(async () => { + try { + turn.abort(new DOMException("test finished", "AbortError")); + for (const step of teardown.splice(0)) await step(); + await drain(); + resetProviderRequestPacingForTest(); + } finally { + jest.clearAllTimers(); + jest.useRealTimers(); + } +}); + +/** Let promise chains and stream reads finish without moving the virtual clock. */ +async function drain(turns = 8) { + for (let turn = 0; turn < turns; turn++) await new Promise(resolve => setImmediate(resolve)); +} + +/** Advance virtual time until `pending` settles; return its value and the virtual ms it took. */ +async function settle(pending: Promise, limitMs = 10_000): Promise<{ value: T; atMs: number }> { + const start = performance.now(); + let done = false; + const observed = pending.finally(() => { done = true; }); + observed.catch(() => {}); + await drain(); + while (!done) { + if (performance.now() - start >= limitMs) throw new Error(`still pending after ${limitMs} virtual ms`); + jest.advanceTimersByTime(1); + await drain(); + } + return { value: await observed, atMs: performance.now() - start }; +} function adapter(adapterSend = false): ProviderAdapter { return { @@ -57,32 +103,42 @@ function run(executor: ProviderFetch, overrides: Partial = {} forwardProvider: { adapter: "openai-responses", baseUrl: "https://chatgpt.test/v1", authMode: "forward" }, hostedTool: { type: "web_search" }, selectedForwardHeaders: new Headers(), settings: { model: "search", reasoning: "low", timeoutMs: 1000 }, maxSearches: 1, - connectTimeoutMs: HEADER_MS, + connectTimeoutMs: HEADER_MS, abortSignal: turn.signal, ...overrides, }); } -async function expectSuccess(pending: Promise) { - const response = await pending; +/** Resolves to the virtual ms at which the final headers arrived. */ +async function expectSuccess(pending: Promise): Promise { + const { value: response, atMs } = await settle(pending); expect(response.status).toBe(200); - const body = await response.text(); + const { value: body } = await settle(response.text()); expect(body).toContain("response.completed"); expect(body).not.toContain("response.failed"); + return atMs; } -async function expectTimeout(pending: Promise, timeoutMs = HEADER_MS) { - const response = await pending; +/** Resolves to the virtual ms at which the 504 was returned. */ +async function expectTimeout(pending: Promise, timeoutMs = HEADER_MS): Promise { + const { value: response, atMs } = await settle(pending); expect(response.status).toBe(504); expect(await response.json()).toEqual({ error: { message: `Provider response-header timeout after ${timeoutMs}ms during web-search`, type: "upstream_error", code: null, } }); + return atMs; +} + +/** A deadline or slot fires at `expectedMs`; observation may trail it by bounded slack while continuations drain. */ +function expectAt(atMs: number, expectedMs: number) { + expect(atMs).toBeGreaterThanOrEqual(expectedMs); + expect(atMs).toBeLessThan(expectedMs + DRAIN_SLACK_MS); } async function waitUntilQueued(configured = provider) { for (let i = 0; i < 1000; i++) { if (providerRequestPacingStatus(NAME, configured).queued > 0) return; - await Bun.sleep(1); + await drain(1); } throw new Error("send never entered pacing queue"); } @@ -93,13 +149,10 @@ describe("web-search pacing and response-header deadline", () => { const held = await waitForProviderRequestSlot(NAME, provider, MODEL); let sends = 0; const executor = fetcher(async () => { sends++; return new Response("ok"); }); - const pending = run(executor, { adapter: adapter(adapterSend) }); - const release = setTimeout(() => held.release(), QUEUE_MS); - try { - await expectSuccess(pending); - expect(sends).toBe(1); - expect(providerRequestPacingStatus(NAME, provider)).toMatchObject({ queued: 0, inFlight: 0 }); - } finally { clearTimeout(release); held.release(); } + setTimeout(() => held.release(), QUEUE_MS); + expectAt(await expectSuccess(run(executor, { adapter: adapter(adapterSend) })), QUEUE_MS); + expect(sends).toBe(1); + expect(providerRequestPacingStatus(NAME, provider)).toMatchObject({ queued: 0, inFlight: 0 }); }); test(`${adapterSend ? "adapter fetchResponse" : "ordinary fetch"} still times out genuine header stalls and releases its lease`, async () => { @@ -107,7 +160,7 @@ describe("web-search pacing and response-header deadline", () => { await sleepWithAbort(10_000, init?.signal ?? undefined); return new Response("unreachable"); }); - await expectTimeout(run(executor, { adapter: adapter(adapterSend) })); + expectAt(await expectTimeout(run(executor, { adapter: adapter(adapterSend) })), HEADER_MS); expect(providerRequestPacingStatus(NAME, provider)).toMatchObject({ queued: 0, inFlight: 0 }); }); } @@ -120,7 +173,7 @@ describe("web-search pacing and response-header deadline", () => { const pending = run(executor, { abortSignal: caller.signal }); await waitUntilQueued(); caller.abort(new DOMException("client left", "AbortError")); - const response = await pending; + const { value: response } = await settle(pending); expect(response.status).toBe(499); expect(sends).toBe(0); expect(providerRequestPacingStatus(NAME, provider)).toMatchObject({ queued: 0, inFlight: 1 }); @@ -132,8 +185,8 @@ describe("web-search pacing and response-header deadline", () => { for (const recovery of ["same-target", "rotation", "reset"] as const) { test(`${recovery} replay excludes its own pacing wait`, async () => { - const configured = { ...provider, requestPacing: { ...provider.requestPacing, - minIntervalMs: recovery === "reset" ? QUEUE_MS * 5 : QUEUE_MS } }; + const intervalMs = recovery === "reset" ? QUEUE_MS * 5 : QUEUE_MS; + const configured = { ...provider, requestPacing: { ...provider.requestPacing, minIntervalMs: intervalMs } }; let sends = 0; const executor = fetcher(async () => { if (++sends === 1) { @@ -142,14 +195,15 @@ describe("web-search pacing and response-header deadline", () => { } return new Response("ok"); }, configured); - await expectSuccess(run(executor, { - // Reset recovery retains its existing 120-180ms backoff inside the budget. + // The replay waits out the whole pacing interval, several header budgets long, and still succeeds the + // moment its slot opens. Reset recovery retains its existing 120-180ms backoff inside the budget. + expectAt(await expectSuccess(run(executor, { connectTimeoutMs: recovery === "reset" ? HEADER_MS * 5 : HEADER_MS, ...(recovery === "same-target" ? { retryOn429Policy: { enabled: true, attempts: 1, intervalMs: 1, maxIntervalMs: 1, respectRetryAfter: false, } } : {}), ...(recovery === "rotation" ? { on429: () => ({ adapter: adapter(true), recoveryKind: "key-429" }) } : {}), - })); + })), intervalMs); expect(sends).toBe(2); expect(providerRequestPacingStatus(NAME, configured)).toMatchObject({ queued: 0, inFlight: 0 }); }); @@ -159,10 +213,8 @@ describe("web-search pacing and response-header deadline", () => { const held = await waitForProviderRequestSlot(NAME, provider, MODEL); const direct = adapter(); direct.fetchResponse = (request, ctx) => ctx!.executor!(request.url, { signal: ctx?.abortSignal }); - const pending = run(fetcher(async () => new Response("ok")), { adapter: direct }); - const release = setTimeout(() => held.release(), QUEUE_MS); - try { await expectSuccess(pending); } - finally { clearTimeout(release); held.release(); } + setTimeout(() => held.release(), QUEUE_MS); + expectAt(await expectSuccess(run(fetcher(async () => new Response("ok")), { adapter: direct })), QUEUE_MS); expect(providerRequestPacingStatus(NAME, provider)).toMatchObject({ queued: 0, inFlight: 0 }); }); @@ -180,7 +232,7 @@ describe("web-search pacing and response-header deadline", () => { await first.body?.cancel(); return send({ url: request.url, recovery: "rate-limit-429", dispatch }); }; - await expectSuccess(run(executor, { adapter: retrying })); + expectAt(await expectSuccess(run(executor, { adapter: retrying })), QUEUE_MS); expect(sends).toBe(2); expect(providerRequestPacingStatus(NAME, configured)).toMatchObject({ queued: 0, inFlight: 0 }); }); @@ -188,12 +240,16 @@ describe("web-search pacing and response-header deadline", () => { test("final headers keep the pacing lease until the streamed body finishes", async () => { let bodyController!: ReadableStreamDefaultController; const upstream = new ReadableStream({ start(controller) { bodyController = controller; } }); - const response = await run(fetcher(async () => new Response(upstream))); + const { value: response } = await settle(run(fetcher(async () => new Response(upstream)))); + teardown.push( + () => response.body?.cancel().catch(() => {}), + () => { try { bodyController.error(new Error("test finished")); } catch { /* already closed */ } }, + ); expect(response.status).toBe(200); expect(providerRequestPacingStatus(NAME, provider).inFlight).toBe(1); bodyController.enqueue(new TextEncoder().encode("ok")); bodyController.close(); - expect(await response.text()).toContain("response.completed"); + expect((await settle(response.text())).value).toContain("response.completed"); expect(providerRequestPacingStatus(NAME, provider).inFlight).toBe(0); }); @@ -206,9 +262,12 @@ describe("web-search pacing and response-header deadline", () => { await sleepWithAbort(70, init?.signal ?? undefined); return ++sends === 1 ? new Response("limited", { status: 429 }) : new Response("ok"); }, configured); - await expectTimeout(run(executor, { connectTimeoutMs: 110, + // The first send spends 70 of 110ms; the pacing wait until 180ms is excluded; the rotated send gets only + // the remaining 40ms. A renewed budget would let it answer at 250ms, and a counted pacing wait would + // expire at 110ms before it started. + expectAt(await expectTimeout(run(executor, { connectTimeoutMs: 110, on429: () => ({ adapter: adapter(), recoveryKind: "key-429" }), - }), 110); + }), 110), QUEUE_MS + 40); expect(sends).toBe(1); expect(started).toBe(2); expect(providerRequestPacingStatus(NAME, configured)).toMatchObject({ queued: 0, inFlight: 0 }); From bbe1fc19da781ce2cafca21c1f69b7d4d17d5cd8 Mon Sep 17 00:00:00 2001 From: lidge-jun Date: Tue, 13 Oct 2026 01:09:32 +0900 Subject: [PATCH 4/9] fix(anthropic-pool): answer spent quota as a usage limit and lift Retry-After quota cooldowns on fresh reads (#6995) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Squashed from #6995 for the 2.84.0 release train. When every Anthropic OAuth account in the pool had spent its five-hour or weekly window, OpenCodex told Codex `429 rate_limit_exceeded`. Codex treats that as transient, so it retried each turn ("reconnecting 1/5 … 5/5") and ended on `exceeded retry limit, last status: 429` without ever saying the quota was spent (#6991). Separately, a quota cooldown whose deadline came from `Retry-After` stayed in place after a fresh usage read showed headroom (plan upgrade, early window reset), so the account stayed out until `ocx account clear-cooldown` (#6989). **Usage limit instead of rate limit (#6991).** For each held account, the latest hold decides whether it is spent quota or something a retry can outlast: | Hold on the account | Classified as | |---|---| | 429 cooldown (only shared-quota refusals record one), any deadline source | spent quota | | model-family exclusion | spent quota, unless its revalidation send is in flight | | transient admission pause (`Retry-After` without a rejected window) | retryable | | entitlement 403 cooldown | retryable | When no account the request's route can use is admissible and every held account is on spent quota, the Responses answer is HTTP 429 with `error.type`/`error.code` `usage_limit_reached`, `Retry-After`, and `resets_at` (epoch seconds), which Codex treats as final and renders as "try again at …". `resets_at` is the earliest upstream window reset. It is computed separately from the admission deadline (`Retry-After`, the 60 s default and the 30 min family-evidence lifetime are not resets) and is omitted when any held account's reset is unknown. A tie between a quota hold and a throttle counts as a throttle. No `plan_type` is sent, so Codex prints its generic usage-limit text rather than ChatGPT upgrade advice. This applies at the local pool refusal in `request-transport.ts`, both cooldown catches in `adapter-dispatch.ts`, the passthrough catch, the final upstream 429 once recovery finds no replacement, and the first iteration of a routed web-search loop. Combo children keep the old `rate_limit_error` body and wording byte for byte, so combo hops and target cooldowns are unchanged. Native `/v1/messages` keeps Anthropic's `rate_limit_error` type with usage-limit wording. Image-bridge terminals, opt-in empty-completion continuations and Chat Completions keep their current identity (documented in `structure/providers/anthropic-account-pool.md`). **Fresh reads lift Retry-After quota cooldowns (#6989).** A fresh usage read may now claim any live 429 cooldown that named a rejected window, whatever set its deadline (`retry-after`, `reset-derived`, `default`). The existing fences are unchanged: the claimed generation must still match, a read older than the claim settles nothing, a claim forces a fresh usage dispatch, and every named window must read below 100%. Entitlement 403s and windowless `Retry-After` refusals are never claimed. #6953's reset-grant admission is unchanged apart from its comment; #6962's API-key billing code is untouched. Logs and response bodies carry no account ids or tokens; the new message names only the pool scope and the reset time. Closes #6989 Closes #6991 --- .../010_plan.md | 100 ++++++ .../fr/reference/configuration/providers.md | 4 + .../ja/reference/configuration/providers.md | 4 + .../ko/reference/configuration/providers.md | 4 + .../docs/reference/configuration/providers.md | 17 + .../ru/reference/configuration/providers.md | 4 + .../tr/reference/configuration/providers.md | 4 + .../reference/configuration/providers.md | 4 + .../reference/configuration/providers.md | 4 + scripts/test-layout/layout.json | 2 +- src/bridge.ts | 2 +- src/bridge/errors.ts | 19 + src/oauth/anthropic-model-quota.ts | 14 +- src/oauth/anthropic-routing.ts | 197 +++++++++-- src/providers/anthropic-reset-admission.ts | 9 +- src/server/responses/adapter-dispatch.ts | 16 +- src/server/responses/core-errors.ts | 30 +- src/server/responses/passthrough-dispatch.ts | 5 +- src/server/responses/request-transport.ts | 11 +- src/server/responses/sidecar-execution.ts | 13 +- structure/providers/anthropic-account-pool.md | 37 ++ structure/transports/responses-failover.md | 2 +- .../anthropic-cooldown-recovery.test.ts | 23 +- .../anthropic/anthropic-model-routes.test.ts | 3 +- .../anthropic-pool-usage-limit.test.ts | 332 ++++++++++++++++++ .../anthropic-quota-cooldown-recovery.test.ts | 161 +++++++++ tests/fixtures/test-layout-expected.json | 2 +- 27 files changed, 970 insertions(+), 53 deletions(-) create mode 100644 devlog/_plan/261012_6991_6989_anthropic_pool_quota/010_plan.md create mode 100644 tests/adapters/anthropic/anthropic-pool-usage-limit.test.ts create mode 100644 tests/adapters/anthropic/anthropic-quota-cooldown-recovery.test.ts diff --git a/devlog/_plan/261012_6991_6989_anthropic_pool_quota/010_plan.md b/devlog/_plan/261012_6991_6989_anthropic_pool_quota/010_plan.md new file mode 100644 index 00000000000..f9dcd03d946 --- /dev/null +++ b/devlog/_plan/261012_6991_6989_anthropic_pool_quota/010_plan.md @@ -0,0 +1,100 @@ +# 010 — Anthropic pool quota exhaustion: terminal usage limit (#6991) and fresh-read recovery (#6989) + +Base: origin/dev dc45bb2ca9. Branch: codex/6989-6991-anthropic-pool-quota-261012. + +## Evidence on dev + +- #6991. Local pool refusals become a retryable 429 `rate_limit_error` / `rate_limit_exceeded` at four sites: + `src/server/responses/request-transport.ts:786-793` (pool-enabled initial selection, `all-cooled`), + `src/server/responses/adapter-dispatch.ts:528-530` and `:788-790` (`AnthropicAccountCooldownError` after pacing / on the recovery leg), + `src/server/responses/passthrough-dispatch.ts:951-953`. `formatErrorResponse` (`src/bridge/errors.ts:19`) classifies every 429 as + `rate_limit_exceeded` (`src/lib/errors.ts:448-458`) and has no slot for `resets_at`. The first exhausted request returns the upstream + 429 as `Provider error 429: ...` with `rate_limit_exceeded` (`adapter-dispatch.ts:1679-1722`) after `rotateAnthropicAccountOnRefusal` + finds no replacement (`src/oauth/anthropic-routing.ts:1062-1073`). +- Codex (openai/codex 6c372fbe, `codex-api/src/api_bridge.rs:181-218`) treats HTTP 429 with `error.type == "usage_limit_reached"` as + non-retryable `UsageLimitReached` and reads `resets_at` as integer epoch seconds; a wrapped WebSocket `error` frame reuses the same + classifier (`responses_websocket.rs:591-662`). OpenCodex's WS bridge keeps the nested error object (`src/server/ws-bridge.ts:369-403`). + Without `plan_type` Codex prints its generic usage-limit copy; a ChatGPT plan type would print ChatGPT upgrade copy, so none is sent. +- #6989. `captureAnthropicCooldownRecovery` and `settleAnthropicCooldownRecovery` (`anthropic-routing.ts:379-412`) only claim + `cooldownSource === "reset-derived"`. A shared-quota 429 that carries Retry-After records source `retry-after` (`:1131-1141`), so a fresh + usage read showing headroom never lifts it. #6953 already records `refusedQuotaWindows` for every 429 regardless of deadline source and + its reset-grant admission documents this gap (`src/providers/anthropic-reset-admission.ts:4-8`). + +## Classification rule (#6991) + +`recordAnthropicAccountRefusal` only records a 429 health entry for a `shared-quota` classification; transient throttles become admission +pauses and family-only rejections become family evidence. For each account the pool would otherwise consider (same filter as +`getAnthropicPoolRetryAfterSeconds`, including route/fallback expansion), collect its blocks: + +| Block | Until | Quota? | +|---|---|---| +| live health entry, refusalStatus 429 | cooldownUntil | yes (shared-quota by construction, any deadline source) | +| live health entry, refusalStatus 403 | cooldownUntil | no (entitlement) | +| admission pause (transient Retry-After) | pause until | no | +| family exclusion, not revalidating | family retry time | yes | +| family exclusion, revalidation in flight (busy) | now + 1 s | no | + +The account is quota-held when its latest block is a quota block (ties favor transient). The pool answers `usage_limit_reached` only when +no route candidate is eligible and every blocked account is quota-held. `resets_at` (epoch seconds) is computed apart from the +admission order: per account, the latest upstream window reset over all its quota holds (unknown if any rejected window lacks a +reset header or a family has no reset), then the earliest across accounts, omitted when any held account is unknown. Retry-After +stays the admission deadline. Anything else keeps today's `rate_limit_error` / `rate_limit_exceeded` with Retry-After. + +## Recovery rule (#6989) + +A cooldown is claimable by a fresh usage read when it is live, `refusalStatus === 429`, and either source is `reset-derived` (unchanged) or +the refusal named at least one rejected window (`refusedQuotaWindows`). Settlement requires the same generation, `quota.updatedAt >= +claimedAt` (unchanged stale-read fence) and `percent < 100` for every window in `rejectedQuotaWindows` if present, else +`refusedQuotaWindows`. 403 refusals and windowless non-reset-derived 429s are never claimed; a reset-derived aggregate refusal stays +claimable as before but is always retained. Claimed probes already force a fresh dispatch +(`requiresFreshDispatch`), so a cached or joined older read cannot settle. + +## Changes (as implemented) + +1. `src/oauth/anthropic-routing.ts`: `AnthropicAccountCooldownError` gains `usageLimit: { resetsAt: number | null } | null` and + `retryableMessage`; its message names the usage limit and, when known, the ISO reset. 429 health entries record `quotaResetAt` + via `rejectedWindowsResetAt` (only when every rejected window carries a reset). `anthropicAccountHold` returns the admission winner + plus the account's quota reset computed over all quota holds. `anthropicPoolCooldownError(now, decision, model)` builds the refusal; + `getAnthropicPoolRetryAfterSeconds` delegates to it; `resolveAnthropicDispatchAccountId` throws it. + `anthropicPoolUsageLimitAfterRefusal` returns it only for a usage limit. Recovery claim/settle widened as above. +2. `src/oauth/anthropic-model-quota.ts`: `anthropicFamilyHoldState` (revalidating flag and family window reset). +3. `src/bridge/errors.ts`: `formatUsageLimitReachedResponse(message, resetsAtMs | null, retryAfterSeconds)`. +4. `src/server/responses/core-errors.ts`: `anthropicPoolCooldownResponse(refusal, { comboAttempt, logCtx })` — usage limit for direct + requests (sets `logCtx.errorCode`), the old `rate_limit_error` body and wording for combo children. +5. Sites: `request-transport.ts` pool-enabled `all-cooled` refusal; both `AnthropicAccountCooldownError` catches in + `adapter-dispatch.ts`; the passthrough catch; the final non-combo upstream 429 in `adapter-dispatch.ts`; the first-iteration 429 of a + routed web-search loop in `sidecar-execution.ts`. +6. Native Messages keeps Anthropic `rate_limit_error`; its message carries the usage-limit wording. +7. Docs: `src/providers/anthropic-reset-admission.ts` comment, `structure/providers/anthropic-account-pool.md` (new section), + `structure/transports/responses-failover.md`, docs-site `reference/configuration/providers.md`. + +## Boundaries (unchanged identity) + +Image bridge terminal, opt-in empty-completion continuation, Chat Completions translation, combo final-error identity +(`core-combo-failure.ts`), Codex-account cooldown path (`src/codex/auth-context.ts`), dashboard reason label for retry-after quota +cooldowns (`src/oauth/health.ts:224`). #6962 shares `adapter-dispatch.ts`, `request-transport.ts` and `messages-native.ts` but not the +edited statements; rerun billing tests on the union once it lands. + +## Tests + +- `tests/adapters/anthropic/anthropic-pool-usage-limit.test.ts` (12): quota-held pool answers `usage_limit_reached` with `resets_at` + (pool on and off); unequal Retry-After/reset; unknown reset omits `resets_at`; partial reset headers; reset independent of admission + winner; transient throttle and 403 keep `rate_limit_exceeded`; last upstream refusal answers the usage limit; throttled sibling keeps + the provider rate limit; family quota vs revalidation; routed web-search loop; combo child body byte-identical; no account material + in bodies or warn logs. +- `tests/adapters/anthropic/anthropic-quota-cooldown-recovery.test.ts` (11): Retry-After quota cooldown lifted by a fresh read (both + instances); partial window retained; stale and superseded claims; default-source recovery; 403 and windowless never claimed; + aggregate reset-derived retained; read started before a refusal cannot lift it; instance isolation; tie with a transient pause. +- Updated: `anthropic-cooldown-recovery.test.ts` (named-window Retry-After/default now clear; windowless/403 do not), + `anthropic-model-routes.test.ts` (helper-named spent window answers `usage_limit_reached`). + +## Verification + +`bun run typecheck`, `bun test --isolate` over `tests/adapters/anthropic/`, `tests/oauth/`, layout/lab/core-module/reset-grant files, +`bun run test:changed`, `bun run structure:check`, `bun run privacy:scan`, `git diff --check`. + +## Independent audit (GPT-6.1-Sol, builder Claude) + +- r1 FAIL: reset vs admission deadline, combo prose, race tests, surfaces, docs — folded except image/continuation/Chat (boundary). +- r2 FAIL: partial reset headers, reset vs admission winner, routed web-search terminal, stale test expectation, plan text — all folded. +- r3 NEAR-PASS: no runtime defect; low finding on this document's stale text, fixed here. diff --git a/docs-site/src/content/docs/fr/reference/configuration/providers.md b/docs-site/src/content/docs/fr/reference/configuration/providers.md index 92910bf7206..ef610f23906 100644 --- a/docs-site/src/content/docs/fr/reference/configuration/providers.md +++ b/docs-site/src/content/docs/fr/reference/configuration/providers.md @@ -251,6 +251,10 @@ rotation automatique peut déclencher des restrictions du fournisseur. Seul un 429 confirmant le rejet d'un quota partagé de cinq heures ou hebdomadaire déclenche la temporisation et le basculement du compte. Une limitation temporaire suspend son admission en conservant l'affinité : au plus un bref nouvel essai sur le même compte et un détour vers un autre compte admissible par requête. Sans en-tête probant, un 429 permet un seul bref nouvel essai sur le même compte, sans refroidir le compte ni inventer Retry-After. Le comportement par défaut avec un seul compte reste identique. Un rejet propre à Fable laisse Sonnet admissible ; choix manuel et affinité vérifient aussi les quotas partagés et ceux de la famille demandée. Les observations passives de famille expirent après trente minutes ou à leur réinitialisation connue et sont revalidées par une seule requête à la fois. Les seuils restent des préférences souples avec le repli existant lorsque tous les candidats sont épuisés, sans plafond strict d'utilisation ou de facturation. L'affinité est locale au processus et de taille bornée. Les erreurs de renouvellement du jeton suivent les règles de réauthentification existantes. Un 403 confirmé lié à un abonnement ou à la facturation du compte permet un basculement avant la sortie, avec une temporisation selon `Retry-After` ou de dix minutes par défaut. Un refus d’autorisation ordinaire ne déclenche pas de basculement. Si tous les comptes admissibles sont en temporisation, les clients reçoivent un 429 accompagné de `Retry-After` lorsqu'il est connu, et non une erreur d'authentification. +Lorsque chaque compte utilisable par la requête est retenu sur un quota épuisé (une fenêtre partagée de cinq heures ou hebdomadaire, ou la fenêtre hebdomadaire de la famille du modèle demandé), ce 429 est une limite d'usage et non une limite de débit : le corps Responses porte `error.type` et `error.code` `usage_limit_reached` et, si la réinitialisation amont de chaque compte retenu est connue, `resets_at` (secondes epoch de la plus proche). Codex s'arrête alors et indique quand réessayer au lieu de réessayer jusqu'à « exceeded retry limit ». Le dernier refus amont pour quota épuisé reçoit la même réponse quand aucun remplaçant ne reste. Si un compte retenu est seulement limité en débit, revalide une famille de modèles ou est refusé pour droits d'accès, la réponse reste `rate_limit_exceeded`, réessayable. Les cibles de combo gardent la forme de limite de débit pour passer à la cible suivante. `/v1/messages` natif garde le type `rate_limit_error` d'Anthropic, avec un message qui nomme la limite d'usage. + +Une temporisation de quota se lève plus tôt dès qu'une lecture d'usage récente (page Fournisseurs du tableau de bord, ou `ocx account list anthropic --quota --refresh`) montre sous 100 % chaque fenêtre rejetée, qu'elle vienne de `Retry-After` ou d'une réinitialisation de fenêtre. Cela couvre une montée d'offre ou une fenêtre rouverte plus tôt. Une lecture antérieure au refus ne la lève jamais, et les temporisations 403 de droits d'accès attendent leur échéance. + :::caution[Expérimental] Laissez cette option désactivée, sauf si vous comprenez les risques liés aux règles d'Anthropic concernant les comptes. En cas de doute, préférez le changement manuel avec `ocx account use anthropic `. diff --git a/docs-site/src/content/docs/ja/reference/configuration/providers.md b/docs-site/src/content/docs/ja/reference/configuration/providers.md index 13184d191e5..b57ae0e51ca 100644 --- a/docs-site/src/content/docs/ja/reference/configuration/providers.md +++ b/docs-site/src/content/docs/ja/reference/configuration/providers.md @@ -227,6 +227,10 @@ affinity を維持します。これらの戦略は provider enforcement を回 共有の5時間・週次クォータが拒否された429だけがアカウントのクールダウンと切り替えを行います。一時的な速度制限はアフィニティを維持して送信受付だけを停止し、リクエストごとに短い同一アカウント再試行1回と別の対象アカウントへの切り替え1回までです。根拠ヘッダーのない429は同一アカウントで1回だけ再試行し、アカウントを冷却せず、Retry-Afterも生成しません。既定の単一アカウント動作は変わりません。Fable専用の拒否はSonnetを制限しません。手動選択とアフィニティもリクエストモデルの共有・ファミリークォータを確認します。パッシブなファミリー情報は30分または既知のリセットで期限切れとなり、1つの送信リクエストで再確認します。使用量しきい値はソフトな優先設定で、全候補消耗時のフォールバックを維持します。使用量・課金のハード上限ではありません。アフィニティはプロセスローカルであり、サイズ制限があります。トークン更新の失敗は既存の再認証ルールに従います。確認済みの契約・アカウント請求の 403 は出力前に別アカウントへ切り替え、`Retry-After` または既定の10分間のクールダウンを記録します。一般的な権限拒否では切り替えません。すべての対象となるアカウントが冷却されている場合、クライアントは、既知の場合、認証エラーではなく、`Retry-After` を含む 429 を受け取ります。 +リクエストが使えるすべてのアカウントが使い切ったクォータ（共有の5時間・週間ウィンドウ、または要求モデルのファミリー週間ウィンドウ）で保留されている場合、その 429 はレート制限ではなく使用量上限です。Responses の本文は `error.type` と `error.code` に `usage_limit_reached` を持ち、保留中の全アカウントの上流リセット時刻が分かる場合は `resets_at`（最も早いリセットのエポック秒）を含みます。Codex は「exceeded retry limit」まで再試行せず、いつ再試行できるかを表示して停止します。最後のアカウントがクォータ切れで拒否し、代替が残っていない場合も同じ応答になります。保留中のアカウントのいずれかが一時的なスロットリング、モデルファミリーの再検証中、または権限による拒否である場合は、再試行可能な `rate_limit_exceeded` のままです。Combo ターゲットは次のターゲットへ進めるようレート制限の形を保ちます。ネイティブ `/v1/messages` は Anthropic の `rate_limit_error` 型を保ち、メッセージで使用量上限を示します。 + +新しい使用量の読み取り（ダッシュボードのプロバイダーページ、または `ocx account list anthropic --quota --refresh`）で、拒否された各ウィンドウが 100% 未満と分かれば、クォータのクールダウンは `Retry-After` 由来でもウィンドウのリセット由来でも早期に解除されます。プランのアップグレードや早期のウィンドウ再開がこれに当たります。拒否より前の読み取りでは解除されず、権限による 403 のクールダウンは期限まで待ちます。 + :::caution[実験的] Anthropic アカウント ポリシーのリスクを理解していない限り、これは無効のままにしてください。不明な場合は、`ocx account use anthropic ` を手動で切り替えることをお勧めします。 ::: diff --git a/docs-site/src/content/docs/ko/reference/configuration/providers.md b/docs-site/src/content/docs/ko/reference/configuration/providers.md index 40db0c8ca04..1b482a58d61 100644 --- a/docs-site/src/content/docs/ko/reference/configuration/providers.md +++ b/docs-site/src/content/docs/ko/reference/configuration/providers.md @@ -229,6 +229,10 @@ affinity 초기화 뒤의 기존 작업도 포함될 수 있습니다. 출력 공유 5시간/주간 할당량의 거부가 확인된 429만 계정 쿨다운과 전환을 유발합니다. 일시적인 속도 제한은 affinity를 유지하며 해당 계정의 전송만 잠시 중지합니다. 요청마다 짧은 동일 계정 재시도 한 번과 적격한 다른 계정 전환 한 번까지만 허용됩니다. 근거 헤더가 없는 429는 동일 계정에서 한 번만 짧게 재시도하며 계정 상태를 냉각하지 않고 Retry-After를 만들어 붙이지 않습니다. 기본 단일 계정 동작은 유지됩니다. Fable 전용 거부는 Sonnet을 제한하지 않습니다. 수동 선택과 affinity도 요청 모델의 공유/계열 할당량을 확인합니다. 수동 조회가 아닌 응답 헤더의 계열 근거는 30분 또는 알려진 리셋 시각에 만료되며 요청 하나로 재확인합니다. 사용량 임계값은 소프트 선호이고 모든 후보가 소진되면 기존 폴백을 유지합니다. 사용량 또는 결제 하드 캡이 아닙니다. 결속은 프로세스 로컬이며 크기가 제한됩니다. 토큰 갱신 실패는 기존 재인증 규칙을 따릅니다. 확인된 구독 또는 계정 결제 403은 출력 전에 다른 계정으로 전환하며 `Retry-After` 또는 기본 10분 동안 냉각합니다. 일반 권한 거부는 전환하지 않습니다. 적격한 계정이 모두 쿨다운 중이면, 클라이언트는 인증 오류가 아니라 알려진 경우 `Retry-After`가 포함된 429를 받습니다. +요청이 쓸 수 있는 모든 계정이 소진된 할당량(공유 5시간·주간 창, 또는 요청 모델 계열의 주간 창)으로 묶여 있으면 그 429는 속도 제한이 아니라 사용량 한도입니다. Responses 본문의 `error.type`과 `error.code`는 `usage_limit_reached`이고, 묶인 모든 계정의 업스트림 리셋 시각을 알면 `resets_at`(가장 이른 리셋의 epoch 초)도 담습니다. Codex는 "exceeded retry limit"까지 재시도하지 않고 언제 다시 시도할지 보여 주며 멈춥니다. 마지막 계정이 할당량 소진으로 거부하고 대체 계정이 없을 때도 같은 응답을 받습니다. 묶인 계정 중 하나라도 일시적 속도 제한, 모델 계열 재확인 중, 권한 거부 상태라면 재시도 가능한 `rate_limit_exceeded`를 유지합니다. Combo 대상은 다음 대상으로 넘어갈 수 있도록 속도 제한 형식을 유지합니다. 네이티브 `/v1/messages`는 Anthropic의 `rate_limit_error` 타입을 유지하고 메시지에서 사용량 한도를 알립니다. + +새 사용량 조회(대시보드 프로바이더 페이지 또는 `ocx account list anthropic --quota --refresh`)에서 거부된 모든 창이 100% 미만으로 나오면, 할당량 쿨다운은 `Retry-After`에서 왔든 창 리셋에서 왔든 일찍 풀립니다. 플랜 업그레이드나 창이 일찍 다시 열린 경우가 여기에 해당합니다. 거부 이전에 시작된 조회는 쿨다운을 풀지 않으며, 권한 403 쿨다운은 기한까지 기다립니다. + :::caution[실험적 기능] Anthropic 계정 정책 위험을 이해하지 못한다면 이 기능은 꺼두십시오. 확신이 없으면 수동 `ocx account use anthropic ` 전환을 우선하십시오. ::: diff --git a/docs-site/src/content/docs/reference/configuration/providers.md b/docs-site/src/content/docs/reference/configuration/providers.md index b07aac6f57a..a333e8cca8b 100644 --- a/docs-site/src/content/docs/reference/configuration/providers.md +++ b/docs-site/src/content/docs/reference/configuration/providers.md @@ -950,6 +950,23 @@ Affinity is process-local and size-bounded. Token-refresh credential failures retain the existing reauthentication policy. Classified pre-output account-entitlement/billing 403s clear affinity and cool the account for `Retry-After`, or ten minutes by default, before trying an eligible replacement. Generic or request-level 403s remain terminal; see [Claude account recovery](/guides/claude-code/). If all eligible accounts are cooling, clients receive 429 with `Retry-After` when known, not an authentication error. +When every account the request can use is held on spent quota (a shared 5-hour or weekly window, +or the requested model family's weekly window), that 429 is a usage limit rather than a rate limit: +the Responses body carries `error.type` and `error.code` `usage_limit_reached` and, when the +upstream reset of every held account is known, `resets_at` (epoch seconds of the earliest reset). +Codex stops on this answer and shows when to try again instead of retrying until +"exceeded retry limit". The last account's own spent-quota refusal gets the same answer once no +replacement is left. If any held account is only throttled, revalidating a model family, or +refused for entitlement, the answer stays the retryable `rate_limit_exceeded`. Combo targets keep +the rate-limit form so the combo can move to its next target. Native `/v1/messages` keeps +Anthropic's `rate_limit_error` type; its message names the usage limit. + +A quota cooldown lifts early once a fresh usage reading (the dashboard Providers page, or +`ocx account list anthropic --quota --refresh`) shows every window the refusal rejected below +100%, whether the cooldown came from `Retry-After` or a window reset. This covers a plan upgrade +or a window that reopened early. A reading taken before the refusal never lifts it, and +entitlement 403 cooldowns wait for their deadline. + Anthropic responses also report the serving account's 5-hour and weekly utilization, and whichever of those two a given response carries is recorded against that account — each window independently, on refusals as well as successes. Usage-aware selection therefore works from the accounts you diff --git a/docs-site/src/content/docs/ru/reference/configuration/providers.md b/docs-site/src/content/docs/ru/reference/configuration/providers.md index 424cd5af5aa..31eb6dcee0d 100644 --- a/docs-site/src/content/docs/ru/reference/configuration/providers.md +++ b/docs-site/src/content/docs/ru/reference/configuration/providers.md @@ -266,6 +266,10 @@ reauth или эффективного порога исчерпания это Если все eligible-аккаунты в cooldown, клиент получает 429 с `Retry-After`, если он известен, а не authentication error. +Если каждый аккаунт, доступный запросу, удерживается исчерпанной квотой (общее пятичасовое или недельное окно либо недельное окно семейства запрошенной модели), этот 429 — лимит использования, а не лимит скорости: тело Responses содержит `error.type` и `error.code` `usage_limit_reached` и, если известен upstream-сброс каждого удерживаемого аккаунта, `resets_at` (epoch-секунды ближайшего сброса). Codex останавливается и показывает, когда повторить, вместо повторов до «exceeded retry limit». Последний отказ upstream из-за исчерпанной квоты получает тот же ответ, когда замены не осталось. Если хотя бы один удерживаемый аккаунт лишь временно ограничен по скорости, перепроверяет семейство моделей или получил отказ по правам доступа, ответ остаётся повторяемым `rate_limit_exceeded`. Цели combo сохраняют форму лимита скорости, чтобы combo перешёл к следующей цели. Нативный `/v1/messages` сохраняет тип Anthropic `rate_limit_error`, а сообщение называет лимит использования. + +Cooldown по квоте снимается раньше, как только свежее чтение использования (страница Providers в dashboard или `ocx account list anthropic --quota --refresh`) показывает каждое отклонённое окно ниже 100%, независимо от того, задан ли cooldown через `Retry-After` или сбросом окна. Это покрывает повышение тарифа и досрочно открывшееся окно. Чтение, начатое до отказа, его никогда не снимает, а cooldown после 403 по правам доступа ждёт своего срока. + :::caution[Experimental] Оставляйте эту функцию выключенной, если не понимаете policy-risk аккаунтов Anthropic. Если сомневаетесь, безопаснее переключать аккаунты вручную через diff --git a/docs-site/src/content/docs/tr/reference/configuration/providers.md b/docs-site/src/content/docs/tr/reference/configuration/providers.md index 7feb6aff39d..18291029c9b 100644 --- a/docs-site/src/content/docs/tr/reference/configuration/providers.md +++ b/docs-site/src/content/docs/tr/reference/configuration/providers.md @@ -282,6 +282,10 @@ Yalnızca ortak 5 saatlik veya haftalık kotanın reddini doğrulayan 429 hesab boyut sınırlıdır. Token yenileme hataları mevcut yeniden kimlik doğrulama kurallarını korur. Doğrulanmış abonelik veya hesap ödeme 403 hatası çıktı başlamadan hesap değiştirebilir ve `Retry-After` veya varsayılan on dakika soğuma uygular. Genel izin reddi hesap değiştirmez. Uygun tüm hesaplar soğuyorsa istemciler bir kimlik doğrulama hatası değil, bilindiğinde `Retry-After` ile 429 alır. +İsteğin kullanabileceği her hesap tükenmiş kotada tutuluyorsa (ortak 5 saatlik veya haftalık pencere ya da istenen model ailesinin haftalık penceresi), bu 429 hız sınırı değil kullanım sınırıdır: Responses gövdesi `error.type` ve `error.code` olarak `usage_limit_reached` taşır ve tutulan her hesabın upstream sıfırlama zamanı biliniyorsa `resets_at` (en erken sıfırlamanın epoch saniyesi) ekler. Codex "exceeded retry limit" olana kadar yeniden denemek yerine durur ve ne zaman tekrar deneneceğini gösterir. Son hesabın tükenmiş kota reddi, yedek kalmadığında aynı yanıtı alır. Tutulan hesaplardan biri yalnızca geçici olarak hız sınırına takılmışsa, bir model ailesini yeniden doğruluyorsa veya yetki nedeniyle reddedilmişse yanıt yeniden denenebilir `rate_limit_exceeded` olarak kalır. Combo hedefleri, combo'nun sonraki hedefe geçebilmesi için hız sınırı biçimini korur. Yerel `/v1/messages` Anthropic'in `rate_limit_error` türünü korur; mesajı kullanım sınırını belirtir. + +Taze bir kullanım okuması (dashboard Sağlayıcılar sayfası veya `ocx account list anthropic --quota --refresh`) reddedilen her pencerenin %100'ün altında olduğunu gösterdiğinde kota soğuması, `Retry-After`'dan da pencere sıfırlamasından da gelse erken kalkar. Plan yükseltmesi veya erken yeniden açılan bir pencere buna örnektir. Retten önce başlayan bir okuma onu asla kaldırmaz; yetki 403 soğumaları sürelerini bekler. + :::caution[Deneysel] Anthropic hesap politikası riskini anlamadığınız sürece bunu devre dışı bırakın. Emin olmadığınızda manuel `ocx account use anthropic ` geçişini tercih edin. diff --git a/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md b/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md index a02f4d243b4..a399340db0d 100644 --- a/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md +++ b/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md @@ -253,6 +253,10 @@ affinity。这些策略不能规避 provider enforcement。 只有共享5小时或每周额度明确拒绝的429才会冷却账户并切换。临时速率限制保留亲和性，只暂停该账户的请求准入；每个请求最多一次短暂的同账户重试和一次合格兄弟账户切换。没有依据响应头的429只允许一次同账户短暂重试，不冷却账户，也不生成Retry-After。默认单账户行为不变。Fable专属拒绝不限制Sonnet；手动选择和亲和性均检查请求模型的共享与家族额度。被动家族信息在30分钟或已知重置时到期，由一个服务请求串行重新验证。使用阈值仍是软偏好，全部候选耗尽时保留原有回退，不是用量或账单硬上限。亲和性是进程本地的，并且有大小上限。令牌刷新失败保留原有重新认证规则。明确的订阅或账户计费 403 可在输出前切换账户，并按 `Retry-After` 或默认十分钟冷却；普通权限拒绝不会切换。如果所有合格账户都在冷却，客户端会在已知时收到带 `Retry-After` 的 429，而不是身份验证错误。 +当请求可用的每个账户都因配额耗尽而被挂起（共享 5 小时或每周窗口，或所请求模型家族的每周窗口）时，这个 429 是用量上限而不是速率限制：Responses 正文的 `error.type` 和 `error.code` 为 `usage_limit_reached`；若每个被挂起账户的上游重置时间都已知，还会带上 `resets_at`（最早重置的 epoch 秒）。Codex 会停止并显示何时可以重试，而不是一直重试到 "exceeded retry limit"。最后一个账户因配额耗尽被拒且没有可替换账户时，也会得到同样的响应。只要有一个被挂起账户只是临时限速、正在重新验证模型家族或因权限被拒，响应就保持可重试的 `rate_limit_exceeded`。Combo 目标保持速率限制形式，以便 combo 转到下一个目标。原生 `/v1/messages` 保持 Anthropic 的 `rate_limit_error` 类型，消息中会说明用量上限。 + +只要一次新的用量读取（仪表盘的提供方页面，或 `ocx account list anthropic --quota --refresh`）显示被拒绝的每个窗口都低于 100%，配额冷却就会提前解除，无论它来自 `Retry-After` 还是窗口重置。这适用于升级套餐或窗口提前重新开放的情况。在拒绝之前开始的读取永远不会解除它，权限类 403 冷却会等到其截止时间。 + :::caution[Experimental] 除非你理解 Anthropic 账户策略风险，否则请保持关闭。若不确定，优先手动使用 `ocx account use anthropic ` 切换。 ::: diff --git a/docs-site/src/content/docs/zh-tw/reference/configuration/providers.md b/docs-site/src/content/docs/zh-tw/reference/configuration/providers.md index 9689311fb77..f89509ceff0 100644 --- a/docs-site/src/content/docs/zh-tw/reference/configuration/providers.md +++ b/docs-site/src/content/docs/zh-tw/reference/configuration/providers.md @@ -187,6 +187,10 @@ API-key 供應商可持有字面值金鑰或環境參考。OAuth 供應商使用 只有共用5小時或每週額度明確拒絕的429才會冷卻帳號並切換。暫時速率限制保留親和性，只暫停該帳號的請求准入；每個請求最多一次短暫的同帳號重試及一次合格兄弟帳號切換。沒有依據回應標頭的429只允許一次同帳號短暫重試，不冷卻帳號，也不產生Retry-After。預設單帳號行為不變。Fable專屬拒絕不限制Sonnet；手動選擇和親和性均檢查請求模型的共用與家族額度。被動家族資訊在30分鐘或已知重設時到期，由一個服務請求依序重新驗證。用量門檻仍是軟偏好，全部候選耗盡時保留原有退回機制，不是用量或帳單硬上限。親和性為行程本地且有界。Token 更新失敗保留原有重新認證規則。已分類的訂閱或帳號計費 403 可在輸出前切換帳號，並按 `Retry-After` 或預設十分鐘冷卻；一般權限拒絕不切換。若所有合格帳號都在冷卻，客戶端收到附帶已知 `Retry-After` 的 429，而非認證錯誤。 +當請求可用的每個帳號都因配額耗盡而被保留（共用 5 小時或每週視窗，或所請求模型家族的每週視窗）時，這個 429 是用量上限而非速率限制：Responses 本文的 `error.type` 與 `error.code` 為 `usage_limit_reached`；若每個被保留帳號的上游重設時間都已知，還會附上 `resets_at`（最早重設的 epoch 秒）。Codex 會停止並顯示何時可再試，而不是一路重試到 "exceeded retry limit"。最後一個帳號因配額耗盡被拒且沒有可替換帳號時，也會得到相同回應。只要有一個被保留帳號只是暫時限速、正在重新驗證模型家族或因權限被拒，回應就維持可重試的 `rate_limit_exceeded`。Combo 目標維持速率限制形式，讓 combo 能轉到下一個目標。原生 `/v1/messages` 維持 Anthropic 的 `rate_limit_error` 類型，訊息中會說明用量上限。 + +只要一次新的用量讀取（儀表板的提供者頁面，或 `ocx account list anthropic --quota --refresh`）顯示每個被拒絕的視窗都低於 100%，配額冷卻就會提前解除，不論它來自 `Retry-After` 或視窗重設。這適用於升級方案或視窗提早重新開放的情況。在拒絕之前開始的讀取永遠不會解除它，權限類 403 冷卻會等到其截止時間。 + :::caution[實驗性] 除非你了解 Anthropic 帳號政策風險，否則保持停用。不確定時偏好手動 `ocx account use anthropic ` 切換。 ::: diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index b0a5fd267ec..fdef21ddfbd 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -13,7 +13,7 @@ "anthropic-instance-isolation.test.ts": "adapters/anthropic", "anthropic-instance-pool-parity.test.ts": "adapters/anthropic", "anthropic-instance-quota.test.ts": "adapters/anthropic", "anthropic-instance-recovery.test.ts": "adapters/anthropic", "anthropic2-fast-parity.test.ts": "adapters/anthropic", "anthropic2-native-routing.test.ts": "claude-integration", - "oauth-anthropic-instance-health.test.ts": "oauth", "oauth-account-plans.test.ts": "oauth", "anthropic2-responses-parity.test.ts": "responses", + "oauth-anthropic-instance-health.test.ts": "oauth", "anthropic-quota-cooldown-recovery.test.ts": "adapters/anthropic", "anthropic-pool-usage-limit.test.ts": "adapters/anthropic", "oauth-account-plans.test.ts": "oauth", "anthropic2-responses-parity.test.ts": "responses", "oauth-anthropic-instance-multiprocess.test.ts": "oauth", "anthropic2-discovery-ownership.test.ts": "oauth", "codex-shim-runtime-preflight.test.ts": "codex-integration", diff --git a/src/bridge.ts b/src/bridge.ts index beb49a3f998..6fb3c483720 100644 --- a/src/bridge.ts +++ b/src/bridge.ts @@ -1,6 +1,6 @@ export { adapterFailureFromMessage } from "./lib/errors"; -export { formatErrorResponse } from "./bridge/errors"; +export { formatErrorResponse, formatUsageLimitReachedResponse } from "./bridge/errors"; export { setOwnedBudgetAbandonedMsForTests } from "./bridge/internal"; export { buildResponseJSON } from "./bridge/response-json"; export { bridgeToResponsesSSE } from "./bridge/sse"; diff --git a/src/bridge/errors.ts b/src/bridge/errors.ts index 5827fcbaf79..c9cb6a2a9c7 100644 --- a/src/bridge/errors.ts +++ b/src/bridge/errors.ts @@ -68,3 +68,22 @@ export function formatErrorResponse( if (refusal) retainReplayRefusal(response); return response; } + +/** + * A spent usage window, in the shape the Codex client treats as final (#6991): HTTP 429 with + * `error.type: "usage_limit_reached"` and, when the reset is known, `resets_at` in epoch + * seconds. Codex stops (and shows the reset time when present) instead of backing off through its + * retry budget, which is what it does with `rate_limit_exceeded`. No `plan_type` is sent: Codex + * would read it as a ChatGPT plan and print that plan's upgrade advice. `formatErrorResponse` + * cannot build this, because every 429 it classifies becomes `rate_limit_exceeded` and its + * payload has no `resets_at`. + */ +export function formatUsageLimitReachedResponse(message: string, resetsAtMs: number | null, retryAfterSeconds: number | null): Response { + const error = { + message, type: "usage_limit_reached", code: "usage_limit_reached", + ...(resetsAtMs !== null ? { resets_at: Math.ceil(resetsAtMs / 1000) } : {}), + }; + const headers = new Headers({ "Content-Type": "application/json" }); + if (retryAfterSeconds !== null) headers.set("Retry-After", String(retryAfterSeconds)); + return new Response(JSON.stringify({ error }), { status: 429, headers }); +} diff --git a/src/oauth/anthropic-model-quota.ts b/src/oauth/anthropic-model-quota.ts index 67430f30ca7..0fe8b249825 100644 --- a/src/oauth/anthropic-model-quota.ts +++ b/src/oauth/anthropic-model-quota.ts @@ -105,10 +105,22 @@ function createAnthropicModelQuota(instance: AnthropicInstanceId) { if (!entry || !anthropicFamilyRejected(id, model, now)) return undefined; return entry.busy ? now + 1000 : Math.min(entry.resetAt ?? Infinity, entry.seenAt + ANTHROPIC_PASSIVE_FAMILY_MAX_AGE_MS); } + /** + * The requested family's exclusion as a usage-limit fact: whether a revalidation send is in flight + * (its answer may reopen the family within the second) and the upstream window reset, when known. + * The admission deadline `anthropicFamilyRetryAt` is not that reset: passive evidence also expires + * after thirty minutes. + */ + function anthropicFamilyHoldState(id: string, model?: string): { revalidating: boolean; resetAt?: number } | undefined { + const family = anthropicModelFamily(model); + const entry = family && ownedObservation(id, family); + return entry ? { revalidating: entry.busy, ...(entry.resetAt !== undefined ? { resetAt: entry.resetAt } : {}) } : undefined; + } return Object.freeze({ instance, anthropicModelExhausted, anthropicFamilyQuotaGeneration, captureAnthropicFamilyQuotaGeneration, observeAnthropicFamilyQuota, anthropicFamilyRejected, claimAnthropicFamilyRevalidation, - clearAnthropicRequestedFamilyQuota, clearAnthropicFamilyQuota, anthropicFamilyRetryAt, reconcileAnthropicFamilyQuota }); + clearAnthropicRequestedFamilyQuota, clearAnthropicFamilyQuota, anthropicFamilyRetryAt, anthropicFamilyHoldState, + reconcileAnthropicFamilyQuota }); } export type AnthropicModelQuota = ReturnType; diff --git a/src/oauth/anthropic-routing.ts b/src/oauth/anthropic-routing.ts index 3338dbbed52..db6ce859104 100644 --- a/src/oauth/anthropic-routing.ts +++ b/src/oauth/anthropic-routing.ts @@ -72,10 +72,25 @@ export type { AnthropicAccountPoolConfig } from "../types/anthropic-account-pool /** Local admission refusal across async dispatch; never treat it as provider reachability evidence. */ export class AnthropicAccountCooldownError extends Error { - constructor(readonly retryAfterSeconds: number | null, readonly routePosition?: number) { - super(routePosition === undefined - ? "All Anthropic OAuth accounts are temporarily rate-limited" - : "Anthropic OAuth accounts for this model route are temporarily rate-limited"); + /** The retryable wording, kept unchanged for combo children: combo failover classifies by prose too. */ + readonly retryableMessage: string; + constructor( + readonly retryAfterSeconds: number | null, + readonly routePosition?: number, + /** + * Set only when no account the route can use is admissible and every held one is out on + * spent quota (#6991). `resetsAt` is the earliest upstream window reset among them (epoch + * ms), or null when any held account's reset is unknown. Null overall means at least one hold + * is a transient throttle or an entitlement refusal, which a client retry may outlast. + */ + readonly usageLimit: { readonly resetsAt: number | null } | null = null, + ) { + const scope = routePosition === undefined ? "All Anthropic OAuth accounts" : "Anthropic OAuth accounts for this model route"; + const retryable = `${scope} are temporarily rate-limited`; + super(usageLimit === null ? retryable + : usageLimit.resetsAt === null ? `${scope} have reached their usage limit` + : `${scope} have reached their usage limit; the first one resets at ${new Date(usageLimit.resetsAt).toISOString()}`); + this.retryableMessage = retryable; this.name = "AnthropicAccountCooldownError"; } } @@ -104,6 +119,18 @@ interface AccountHealth { refusalStatus: 429 | 403; /** Windows a 429 reported as rejected, whatever set the deadline; empty when it named none. */ refusedQuotaWindows: AnthropicQuotaWindow[]; + /** + * When the rejected windows reopen according to upstream's reset headers, kept even when + * Retry-After set the deadline. A usage-limit answer reports this, never an admission deadline. + */ + quotaResetAt?: number; +} + +/** See `anthropicAccountHold`: the latest hold on one account and its known window reset. */ +interface AnthropicAccountHold { + until: number; + quota: boolean; + resetAt?: number; } interface AffinityEntry { @@ -166,6 +193,32 @@ function parseRateLimitReset( return { delayMs: latest - now, rejectedQuotaWindows }; } +/** + * When every window a refusal reported as rejected reopens, per upstream's reset headers; undefined + * unless each rejected window carries a usable reset. Unlike `parseRateLimitReset`, which picks the + * best available admission deadline, this is reported to the client as the usage-limit reset, so a + * weekly window with no reset cannot hide behind a five-hour one that has one. + */ +function rejectedWindowsResetAt(headers: AnthropicRateLimitHeaders | null | undefined, now: number): number | undefined { + if (!headers) return undefined; + const usableReset = (header: string): number | undefined => { + const resetAt = Number(headers.get(header)?.trim()) * 1000; + return Number.isFinite(resetAt) && delayUntil(resetAt, now) !== undefined ? resetAt : undefined; + }; + const rejected = rejectedQuotaWindowsIn(headers); + if (!rejected.length) { + return headers.get("anthropic-ratelimit-unified-status")?.trim() === "rejected" + ? usableReset("anthropic-ratelimit-unified-reset") : undefined; + } + let latest = 0; + for (const window of rejected) { + const resetAt = usableReset(`anthropic-ratelimit-unified-${window === "five-hour" ? "5h" : "7d"}-reset`); + if (resetAt === undefined) return undefined; + latest = Math.max(latest, resetAt); + } + return latest; +} + interface ScoredAccount { accountId: string; hasKnownUsage: boolean; @@ -308,6 +361,8 @@ type RoutingFacadeSymbol = | "hasAnthropicFailoverQuorum" | "forgetAnthropicFailoverQuorum" | "getAnthropicPoolRetryAfterSeconds" + | "anthropicPoolCooldownError" + | "anthropicPoolUsageLimitAfterRefusal" | "pickAlternateAnthropicAccount" | "resolveAnthropicAccountForSession" | "resolveAnthropicDispatchAccountId" @@ -330,7 +385,7 @@ const handlers = new Map(); function createAnthropicRouting(instance: AnthropicInstanceId): AnthropicRouting { const PROVIDER = instance; const poolKey = anthropicPoolKey(instance); - const { anthropicFamilyRejected, anthropicFamilyRetryAt, anthropicModelExhausted, clearAnthropicFamilyQuota } = anthropicModelQuotaFor(instance); + const { anthropicFamilyRejected, anthropicFamilyRetryAt, anthropicFamilyHoldState, anthropicModelExhausted, clearAnthropicFamilyQuota } = anthropicModelQuotaFor(instance); const { anthropicRatePauseUntil, clearAnthropicRatePauses, pauseAnthropicRateAdmission } = anthropicRatePolicyFor(instance); const { anthropicCooldownGeneration, clearAnthropicCooldownGenerations, noteAnthropicCooldownMutation } = anthropicCooldownRecoveryFor(instance); // Request-owned config authorizes selection/commit. Namespace-only credential and roster @@ -371,7 +426,26 @@ function createAnthropicRouting(instance: AnthropicInstanceId): AnthropicRouting /** - * Capture the exact reset-derived refusal a fresh usage probe is allowed to recover. + * The windows a fresh usage read must show open before it may lift this cooldown, or null when + * no usage read can lift it. + * + * Only a 429 is quota evidence: `recordAnthropicAccountRefusal` keeps a 429 only after + * classifying it as shared quota, and a 403 is an entitlement refusal that usage cannot answer. + * The deadline source does not matter (#6989). Anthropic sends Retry-After on a spent window as + * well, set to that window's reset, so a `retry-after` cooldown that named rejected windows + * carries exactly the evidence a `reset-derived` one does; the read decides, not the header. + * A refusal that named no window can only be the aggregate unified status. It stays claimable + * when reset-derived, as before, but no usage reading covers it, so it is always retained. + */ + function recoverableQuotaWindows(entry: AccountHealth): readonly AnthropicQuotaWindow[] | null { + if (entry.refusalStatus !== 429) return null; + if (entry.rejectedQuotaWindows?.length) return entry.rejectedQuotaWindows; + if (entry.refusedQuotaWindows.length) return entry.refusedQuotaWindows; + return entry.cooldownSource === "reset-derived" ? [] : null; + } + + /** + * Capture the exact quota refusal a fresh usage probe is allowed to recover. * * The generation fence matters because the usage request is asynchronous: a newer 429 may * arrive while it is in flight, and an older answer must never erase that newer refusal. @@ -381,13 +455,14 @@ function createAnthropicRouting(instance: AnthropicInstanceId): AnthropicRouting now = Date.now(), ): AnthropicCooldownRecoveryClaim | null { const entry = upstreamHealth.get(accountId); - if (!entry || entry.cooldownUntil <= now || entry.cooldownSource !== "reset-derived") return null; + if (!entry || entry.cooldownUntil <= now || recoverableQuotaWindows(entry) === null) return null; return { instance, accountId, cooldownGeneration: entry.cooldownGeneration, claimedAt: now }; } /** - * Clear only the claimed reset-derived cooldown when a fresh, complete usage result proves - * headroom in every window that upstream previously reported as rejected. + * Clear only the claimed quota cooldown when a fresh, complete usage result proves headroom in + * every window that upstream previously reported as rejected. A result read before the claim + * (`updatedAt < claimedAt`) is stale and settles nothing. */ function settleAnthropicCooldownRecovery( claim: AnthropicCooldownRecoveryClaim, @@ -395,12 +470,12 @@ function createAnthropicRouting(instance: AnthropicInstanceId): AnthropicRouting ): AnthropicCooldownRecoverySettlement { if (claim.instance !== instance) return "superseded"; const entry = upstreamHealth.get(claim.accountId); - if (!entry || entry.cooldownSource !== "reset-derived" + const windows = entry ? recoverableQuotaWindows(entry) : null; + if (!entry || windows === null || entry.cooldownGeneration !== claim.cooldownGeneration || anthropicCooldownGeneration(claim.accountId) !== claim.cooldownGeneration || !Number.isFinite(quota.updatedAt) || quota.updatedAt < claim.claimedAt) return "superseded"; - const windows = entry.rejectedQuotaWindows; - if (!windows?.length) return "retained"; + if (!windows.length) return "retained"; const recovered = windows.every(window => { const percent = window === "five-hour" ? quota.fiveHourPercent : quota.weeklyPercent; return typeof percent === "number" && Number.isFinite(percent) && percent >= 0 && percent < 100; @@ -450,7 +525,7 @@ function createAnthropicRouting(instance: AnthropicInstanceId): AnthropicRouting const required: readonly AnthropicQuotaWindow[] = named.length ? named : ["five-hour", "weekly"]; if (entry.refusalStatus !== 429 || !required.every(window => clearedWindows.has(window))) { // Kept, but reads that started before the grant are now stale. Move the entry to the same - // new generation so a fresh usage read can still recover a reset-derived cooldown. + // new generation so a fresh usage read can still recover a 429 quota cooldown. entry.cooldownGeneration = noteAnthropicCooldownMutation(accountId); return "uncovered"; } @@ -646,23 +721,90 @@ function createAnthropicRouting(instance: AnthropicInstanceId): AnthropicRouting /** Earliest remaining cooldown among cooled Anthropic accounts, for client Retry-After. */ function getAnthropicPoolRetryAfterSeconds(now = Date.now(), decision: AnthropicRouteDecision | null = null, model?: string): number | null { + return anthropicPoolCooldownError(now, decision, model).retryAfterSeconds; + } + + /** + * What keeps one account out of admission, and whether that is spent quota (#6991). + * + * Every hold must lift before the account can serve, so the latest one decides. A 429 health + * entry is quota by construction (only a shared-quota classification records one), whatever its + * deadline source; a 403 entry is an entitlement refusal; an admission pause is a transient + * throttle; a family exclusion is a spent family window unless a revalidation send is already + * in flight. A tie goes to the transient hold, because then a retry at that instant may succeed. + * + * The reported reset is computed apart from that admission order: the account's usage reopens + * when the last of its quota holds' upstream windows resets, and it is unknown when any of them + * lacks a known reset. Admission deadlines are not resets: they may be Retry-After, the + * sixty-second default or the thirty-minute family evidence lifetime. + */ + function anthropicAccountHold(accountId: string, now: number, model?: string): AnthropicAccountHold | null { + const latest = { until: 0, quota: false }; + let quotaResetAt = 0; + let quotaResetKnown = true; + const consider = (until: number | undefined, quota: boolean, reset?: number) => { + if (until === undefined || until <= now) return; + if (quota) { + if (reset === undefined || reset <= now) quotaResetKnown = false; + else quotaResetAt = Math.max(quotaResetAt, reset); + } + if (until < latest.until || (until === latest.until && quota)) return; + latest.until = until; + latest.quota = quota; + }; + const entry = getAnthropicAccountHealthSnapshot(accountId, now) ? upstreamHealth.get(accountId) : undefined; + consider(entry?.cooldownUntil, entry?.refusalStatus === 429, entry?.quotaResetAt); + consider(anthropicRatePauseUntil(accountId, now), false); + const family = anthropicFamilyHoldState(accountId, model); + consider(anthropicFamilyRetryAt(accountId, model, now), family?.revalidating === false, family?.resetAt); + if (latest.until === 0) return null; + return { until: latest.until, quota: latest.quota, + ...(latest.quota && quotaResetKnown && quotaResetAt > 0 ? { resetAt: quotaResetAt } : {}) }; + } + + /** + * The local refusal for a pool (or model route) with no admissible account. Retry-After is the + * earliest hold among accounts the route can use once they recover. The refusal is a usage limit + * only when no account the route can use is admissible and every held one is out on spent quota; + * then a client retry before that instant cannot succeed, and Codex should stop and show the + * reset instead of retrying. The reset is the earliest known window reset, and is omitted when + * any held account's reset is unknown, since that account could reopen first. + */ + function anthropicPoolCooldownError(now = Date.now(), decision: AnthropicRouteDecision | null = null, model?: string): AnthropicAccountCooldownError { const set = getAccountSet(PROVIDER); - if (!set) return null; + if (!set) return new AnthropicAccountCooldownError(null, decision?.position); // Once an explicit fallback route has no eligible declared account, selection may use // the ordinary pool. Its earliest usable cooldown must determine the advertised wait. const fallbackExpanded = decision?.fallback === true && !getEligibleAnthropicAccounts(now, model).some(id => decision.accounts.includes(id)); let earliest: number | null = null; + let allQuota = true; + let earliestReset: number | null = null; + let resetsKnown = true; for (const account of set.accounts) { if (account.paused === true || account.needsReauth === true || !isPoolCredentialUsable(account.id, now)) continue; if (decision && !fallbackExpanded && !decision.accounts.includes(account.id)) continue; - const snap = getAnthropicAccountHealthSnapshot(account.id, now); - const until = Math.max(snap?.cooldownUntil ?? 0, anthropicRatePauseUntil(account.id, now) ?? 0, anthropicFamilyRetryAt(account.id, model, now) ?? 0); - if (until <= now) continue; - if (earliest === null || until < earliest) earliest = until; + const hold = anthropicAccountHold(account.id, now, model); + if (!hold) continue; + if (!hold.quota) allQuota = false; + else if (hold.resetAt === undefined) resetsKnown = false; + else if (earliestReset === null || hold.resetAt < earliestReset) earliestReset = hold.resetAt; + if (earliest === null || hold.until < earliest) earliest = hold.until; } - if (earliest === null || earliest <= now) return null; - return Math.max(1, Math.ceil((earliest - now) / 1000)); + if (earliest === null) return new AnthropicAccountCooldownError(null, decision?.position); + const exhausted = allQuota && routeCandidates(getEligibleAnthropicAccounts(now, model), decision).length === 0; + return new AnthropicAccountCooldownError(Math.max(1, Math.ceil((earliest - now) / 1000)), decision?.position, + exhausted ? { resetsAt: resetsKnown ? earliestReset : null } : null); + } + + /** + * After an upstream refusal found no replacement: the usage-limit refusal when the pool is + * exhausted as defined above, else null. A refusal that left an admissible sibling (a send + * budget ran out, a request-scoped 429) or a sibling on a transient hold is not exhaustion. + */ + function anthropicPoolUsageLimitAfterRefusal(now = Date.now(), decision: AnthropicRouteDecision | null = null, model?: string): AnthropicAccountCooldownError | null { + const refusal = anthropicPoolCooldownError(now, decision, model); + return refusal.usageLimit === null ? null : refusal; } function pickLowestUsage(config: OcxConfig, excludeId: string | undefined, now: number, decision: AnthropicRouteDecision | null = null, model?: string, excludedAccountIds?: ReadonlySet): string | null { @@ -993,7 +1135,7 @@ function createAnthropicRouting(instance: AnthropicInstanceId): AnthropicRouting const now = Date.now(); const selection = resolveAnthropicAccountForSession(sessionKey, config, now, decision, model); if (selection.reason === "all-cooled") { - throw new AnthropicAccountCooldownError(getAnthropicPoolRetryAfterSeconds(now, decision, model), decision?.position); + throw anthropicPoolCooldownError(now, decision, model); } if (selection.reason === "paused" || !selection.accountId || !getEligibleAnthropicAccounts(now, model).includes(selection.accountId)) { const { OAuthAccountPausedError, OAuthLoginRequiredError } = await import("./index"); @@ -1127,6 +1269,7 @@ function createAnthropicRouting(instance: AnthropicInstanceId): AnthropicRouting // account is back in the rotation a minute later. const parsedRetry = parseRetryAfterMs(retryAfterHeader, now); const resetDerived = status === 429 && parsedRetry === undefined ? parseRateLimitReset(rateLimitHeaders, now) : undefined; + const quotaResetAt = status === 429 ? rejectedWindowsResetAt(rateLimitHeaders, now) : undefined; const cooldownMs = parsedRetry ?? resetDerived?.delayMs ?? (status === 403 ? ACCOUNT_REFUSAL_COOLDOWN_MS : DEFAULT_COOLDOWN_MS); const cooldownGeneration = noteAnthropicCooldownMutation(failedAccountId); upstreamHealth.set(failedAccountId, { @@ -1139,6 +1282,7 @@ function createAnthropicRouting(instance: AnthropicInstanceId): AnthropicRouting recordedAt: now, refusalStatus: status, refusedQuotaWindows: status === 429 ? rejectedQuotaWindowsIn(rateLimitHeaders) : [], + ...(quotaResetAt !== undefined ? { quotaResetAt } : {}), }); sweepExpiredOnWrite(now); clearAnthropicSessionAffinityForAccount(failedAccountId); @@ -1320,7 +1464,8 @@ function createAnthropicRouting(instance: AnthropicInstanceId): AnthropicRouting captureAnthropicCooldownRecovery, settleAnthropicCooldownRecovery, getAnthropicAccountHealthSnapshot, clearAnthropicAccountCooldown, clearAnthropicAccountCooldownForReset, sweepExpiredAnthropicRoutingHealth, clearAnthropicAccountPoolState, anthropicSessionAffinitySizeForTests, getEligibleAnthropicAccounts, hasAnthropicFailoverQuorum, - forgetAnthropicFailoverQuorum, getAnthropicPoolRetryAfterSeconds, pickAlternateAnthropicAccount, + forgetAnthropicFailoverQuorum, getAnthropicPoolRetryAfterSeconds, anthropicPoolCooldownError, + anthropicPoolUsageLimitAfterRefusal, pickAlternateAnthropicAccount, resolveAnthropicAccountForSession, resolveAnthropicDispatchAccountId, bindAnthropicSessionAffinity, clearAnthropicSessionAffinityForAccount, rotateAnthropicAccountOn429, rotateAnthropicAccountOnRefusal, recordAnthropicAccount429, recordAnthropicAccountRefusal, promoteAnthropicActiveAccount, @@ -1434,6 +1579,14 @@ export function getAnthropicPoolRetryAfterSeconds(now = Date.now(), decision: An return anthropicRoutingFor("anthropic").getAnthropicPoolRetryAfterSeconds(now, decision, model); } +export function anthropicPoolCooldownError(now = Date.now(), decision: AnthropicRouteDecision | null = null, model?: string): AnthropicAccountCooldownError { + return anthropicRoutingFor("anthropic").anthropicPoolCooldownError(now, decision, model); +} + +export function anthropicPoolUsageLimitAfterRefusal(now = Date.now(), decision: AnthropicRouteDecision | null = null, model?: string): AnthropicAccountCooldownError | null { + return anthropicRoutingFor("anthropic").anthropicPoolUsageLimitAfterRefusal(now, decision, model); +} + export function pickAlternateAnthropicAccount( config: OcxConfig, excludeId: string, diff --git a/src/providers/anthropic-reset-admission.ts b/src/providers/anthropic-reset-admission.ts index f2db67fea88..8e2f8d16f89 100644 --- a/src/providers/anthropic-reset-admission.ts +++ b/src/providers/anthropic-reset-admission.ts @@ -1,10 +1,9 @@ /** * Put one Anthropic account back into the pool after a settled reset grant. * - * A shared-quota 429 that also carries Retry-After is stored with source - * "retry-after" until that deadline. A later usage probe cannot clear that - * source: captureAnthropicCooldownRecovery only claims reset-derived cooldowns. - * The grant itself has to lift admission, but only as far as its evidence goes: + * A fresh usage read lifts a 429 quota cooldown whose rejected windows read + * open, whatever its deadline source (#6989), but a grant should not have to + * wait for one: the grant itself lifts admission, as far as its evidence goes: * * - `claimedAt` is when the operation was first journaled, before its first * claim was sent. A refusal recorded at or after it may already describe the @@ -17,7 +16,7 @@ * Whatever the routing outcome (other than a newer refusal), the cached bars * are dropped and usage is reread with its own request: joining a read that * started before the grant could republish the pre-reset bars, and a fresh - * reading is what lets reset-derived recovery release a cooldown the cleared + * reading is what lets usage recovery release a cooldown the cleared * list did not cover. The reread is best-effort; the grant is already spent. * * Model-family exclusions are not cleared here. Only that reread may release diff --git a/src/server/responses/adapter-dispatch.ts b/src/server/responses/adapter-dispatch.ts index a48ea95ae60..fa0cfc62e4a 100644 --- a/src/server/responses/adapter-dispatch.ts +++ b/src/server/responses/adapter-dispatch.ts @@ -24,7 +24,7 @@ import { sealRequestAttemptIdentity, recordAttemptCredentialSource, } from "../request-log"; -import { clientCancelledResponse, readDisplaySafeErrorText, normalizeUpstreamErrorText } from "./core-errors"; +import { anthropicPoolCooldownResponse, clientCancelledResponse, readDisplaySafeErrorText, normalizeUpstreamErrorText } from "./core-errors"; import { isTranslatorBudgetExceededError } from "../../lib/translator-budget"; import { redactSecretString } from "../../lib/redact"; import { rewriteUpstreamPolicyRefusal } from "./policy-refusal"; @@ -556,8 +556,7 @@ export async function prepareAdapterExchange( return formatErrorResponse(401, "authentication_error", publicOAuthAuthenticationErrorMessage(refusal)); } if (refusal instanceof AnthropicAccountCooldownError) { - return formatErrorResponse(429, "rate_limit_error", refusal.message, - refusal.retryAfterSeconds === null ? undefined : { retryAfter: String(refusal.retryAfterSeconds) }); + return anthropicPoolCooldownResponse(refusal, { comboAttempt: options.comboAttempt, logCtx }); } // A budget refusal is a decision this process made, not an upstream fault. Reporting it as // 502 does more than mislabel it: the Codex client retries 5xx and does not retry a 429, so @@ -835,8 +834,7 @@ export async function prepareAdapterExchange( return { failed: formatErrorResponse(401, "authentication_error", publicOAuthAuthenticationErrorMessage(refusal)) }; } if (refusal instanceof AnthropicAccountCooldownError) { - return { failed: formatErrorResponse(429, "rate_limit_error", refusal.message, - refusal.retryAfterSeconds === null ? undefined : { retryAfter: String(refusal.retryAfterSeconds) }) }; + return { failed: anthropicPoolCooldownResponse(refusal, { comboAttempt: options.comboAttempt, logCtx }) }; } // Same rule on the recovery leg: the ladder refused to send again, so the answer names // this proxy rather than the provider it never reached. @@ -1805,6 +1803,14 @@ export async function prepareAdapterExchange( requestState.subagentFallbackAccountId, ); } + // The last pool account just refused on spent quota and recovery found no replacement: + // answer the usage limit now (#6991), so Codex stops instead of retrying into the local + // refusal that every later attempt meets until the first window reopens. + const poolUsageLimit = upstreamResponse.status === 429 && anthropicInstance && route.provider.authMode === "oauth" + ? anthropicRoutingFor(anthropicInstance).anthropicPoolUsageLimitAfterRefusal(Date.now(), transportState.anthropicRouteDecision, route.modelId) + : null; + // Combo children returned above; the flag is passed anyway so this site states the same rule. + if (poolUsageLimit) return anthropicPoolCooldownResponse(poolUsageLimit, { comboAttempt: options.comboAttempt, logCtx }); // Upstreams occasionally echo request details in error bodies — scrub token-shaped // material before it reaches the client-facing error surface. const upstreamRetryAfter = upstreamResponse.headers.get("retry-after"); diff --git a/src/server/responses/core-errors.ts b/src/server/responses/core-errors.ts index 39dc0ce9604..8e067cad317 100644 --- a/src/server/responses/core-errors.ts +++ b/src/server/responses/core-errors.ts @@ -2,7 +2,7 @@ import { readBoundedResponseBody } from "../../lib/bounded-body"; import { redactSecretString } from "../../lib/redact"; import { isCyberPolicyMessage, isCyberPolicyCode } from "../../lib/errors"; import { isTranslatorBudgetExceededError } from "../../lib/translator-budget"; -import { formatErrorResponse } from "../../bridge"; +import { formatErrorResponse, formatUsageLimitReachedResponse } from "../../bridge"; import { UnsupportedContentEncodingError, DecompressedBodyTooLargeError, @@ -132,6 +132,34 @@ export function clientCancelledResponse(): Response { return formatErrorResponse(499, "client_cancelled", "Client cancelled request"); } +/** + * The Responses answer to a local Anthropic pool refusal (`AnthropicAccountCooldownError`). + * + * When every held account is out on spent quota the refusal is a usage limit (#6991), which the + * Codex client stops on and renders with its reset time; anything else stays the retryable + * `rate_limit_error` it always was. A combo child keeps the retryable form and wording: the combo + * hops on any 429 and rebuilds the body anyway, and `isAnthropicPoolLocalRefusal` in + * `core-combo.ts` deliberately cools the target for the short local fallback, which a usage-limit + * code or usage-limit prose would stretch to the ten-minute exhaustion fallback in + * `combos/failover.ts`. + */ +export function anthropicPoolCooldownResponse( + refusal: { + message: string; + retryableMessage: string; + retryAfterSeconds: number | null; + usageLimit: { readonly resetsAt: number | null } | null; + }, + options: { comboAttempt?: boolean; logCtx?: { errorCode?: string } } = {}, +): Response { + if (refusal.usageLimit !== null && !options.comboAttempt) { + if (options.logCtx) options.logCtx.errorCode = "usage_limit_reached"; + return formatUsageLimitReachedResponse(refusal.message, refusal.usageLimit.resetsAt, refusal.retryAfterSeconds); + } + return formatErrorResponse(429, "rate_limit_error", refusal.retryableMessage, + refusal.retryAfterSeconds === null ? undefined : { retryAfter: String(refusal.retryAfterSeconds) }); +} + export const UNREADABLE_ENCRYPTED_AGENT_TASK_MESSAGE = "Routed V2 worker task is encrypted for the native ChatGPT backend and cannot be read by the selected provider. Use plaintext V2 agent-message delivery or select a native ChatGPT model."; diff --git a/src/server/responses/passthrough-dispatch.ts b/src/server/responses/passthrough-dispatch.ts index f17b98dfbf6..c5cd64a51fd 100644 --- a/src/server/responses/passthrough-dispatch.ts +++ b/src/server/responses/passthrough-dispatch.ts @@ -91,7 +91,7 @@ import { safeHostLabel, } from "./fetch-helpers"; import { classifyPoolRecoveryDispatch } from "../../routing/probe-lease"; -import { clientCancelledResponse } from "./core-errors"; +import { anthropicPoolCooldownResponse, clientCancelledResponse } from "./core-errors"; import { upstreamHostCircuitOpenResponse, usesCodexForwardPoolAuth, @@ -949,8 +949,7 @@ export async function preparePassthroughExchange( return formatErrorResponse(401, "authentication_error", publicOAuthAuthenticationErrorMessage(refusal)); } if (refusal instanceof AnthropicAccountCooldownError) { - return formatErrorResponse(429, "rate_limit_error", refusal.message, - refusal.retryAfterSeconds === null ? undefined : { retryAfter: String(refusal.retryAfterSeconds) }); + return anthropicPoolCooldownResponse(refusal, { comboAttempt: options.comboAttempt, logCtx }); } return formatErrorResponse(403, "permission_error", publicOAuthAuthenticationErrorMessage(refusal)); } diff --git a/src/server/responses/request-transport.ts b/src/server/responses/request-transport.ts index 9ab1a23d14a..9e11b700647 100644 --- a/src/server/responses/request-transport.ts +++ b/src/server/responses/request-transport.ts @@ -88,7 +88,7 @@ import { import type { AttemptRecoveryKind } from "../../usage/log"; import { bindAttemptDeliveryRecorder } from "../../usage/attempt-delivery"; import { resolvePassiveRouteSubjectId } from "../passive-route-linker"; -import { clientCancelledResponse } from "./core-errors"; +import { anthropicPoolCooldownResponse, clientCancelledResponse } from "./core-errors"; import { acquireAccountLease, KIRO_ACCOUNT_WAIT_MS } from "../../oauth/kiro-account-load"; /** Return a renewed Chat wire to the native owner before sending inference. */ @@ -785,12 +785,9 @@ export async function prepareResponsesTransport( // Route names may resemble account IDs; log only the matched rule position. if (anthropicRouteDecision) console.warn(`[${anthropicInstance}-pool] route:#${anthropicRouteDecision.position} ${selection.reason}; answering locally`); if (selection.reason === "all-cooled") { - const retryAfterSec = anthropicRouting!.getAnthropicPoolRetryAfterSeconds(Date.now(), anthropicRouteDecision, route.modelId); - return formatErrorResponse( - 429, - "rate_limit_error", - anthropicRouteDecision ? "Anthropic OAuth accounts for this model route are temporarily rate-limited" : "All Anthropic OAuth accounts are temporarily rate-limited", - retryAfterSec !== null ? { retryAfter: String(retryAfterSec) } : undefined, + return anthropicPoolCooldownResponse( + anthropicRouting!.anthropicPoolCooldownError(Date.now(), anthropicRouteDecision, route.modelId), + { comboAttempt: options.comboAttempt, logCtx }, ); } return formatErrorResponse(401, "authentication_error", anthropicRouteDecision ? "No eligible Anthropic OAuth account for this model route" : "No eligible Anthropic OAuth account available"); diff --git a/src/server/responses/sidecar-execution.ts b/src/server/responses/sidecar-execution.ts index 4b36aa8e978..7f277c4f715 100644 --- a/src/server/responses/sidecar-execution.ts +++ b/src/server/responses/sidecar-execution.ts @@ -32,7 +32,7 @@ import { classifyKiroRefusal } from "../../adapters/kiro-refusal"; import { isNonReplayableResponse } from "../../lib/upstream-retry"; import { noteKiroMonthlyRefusal, noteKiroServedSuccess } from "../../providers/kiro-usage"; import { persistKiroAccountState } from "../../providers/kiro-account-state-disk"; -import { readDisplaySafeErrorText } from "./core-errors"; +import { anthropicPoolCooldownResponse, readDisplaySafeErrorText } from "./core-errors"; import { ANTHROPIC_POOL_MAX_FAILOVERS_PER_REQUEST, anthropicRoutingFor, @@ -555,7 +555,7 @@ export async function executeResponsesSidecars( modelId: route.modelId, ...sidecarBudget.fetchOptions(transportState.adapter), })(input, init)) as typeof globalThis.fetch; - const wsResponse = await runWithWebSearch({ + const loopResponse = await runWithWebSearch({ parsed, adapter: transportState.adapter, onIterationEnd: sidecarBudget.releaseUnsentHop, fetchForRequest: (request, iterParsed, adapter) => providerFetch(route.provider, options.codexWsRuntimeIdentity, { @@ -622,6 +622,15 @@ export async function executeResponsesSidecars( notifyResponseComplete(response); }, }); + // A routed Anthropic model whose last pool account refused on spent quota fails the loop's + // first iteration with a generic 429 before any search runs. Answer the usage limit instead, + // as the main path does (#6991), so Codex stops rather than retrying the whole turn. + const poolUsageLimit = loopResponse.status === 429 && anthropicInstance && route.provider.authMode === "oauth" + && !options.comboAttempt + ? anthropicRoutingFor(anthropicInstance).anthropicPoolUsageLimitAfterRefusal(Date.now(), transportState.anthropicRouteDecision, route.modelId) + : null; + if (poolUsageLimit) void loopResponse.body?.cancel().catch(() => {}); + const wsResponse = poolUsageLimit ? anthropicPoolCooldownResponse(poolUsageLimit, { logCtx }) : loopResponse; // Register the sidecar stream as an active turn so drainAndShutdown waits for (or aborts) // in-flight web-search turns instead of skipping them during graceful shutdown. if (wsResponse.body) { diff --git a/structure/providers/anthropic-account-pool.md b/structure/providers/anthropic-account-pool.md index 9ebbfce6a3a..cf0c0b9ba8e 100644 --- a/structure/providers/anthropic-account-pool.md +++ b/structure/providers/anthropic-account-pool.md @@ -359,6 +359,43 @@ Default single-account users acquire no new retry or admission pause. Regression coverage: `tests/adapters/anthropic/anthropic-429-policy.test.ts`. +## Quota exhaustion answer and fresh-read recovery + +`anthropicPoolCooldownError` in `src/oauth/anthropic-routing.ts` builds the local refusal for a +pool or model route with no admissible account. Per account, the latest hold decides: a 429 health +entry is spent quota whatever its deadline source (only a shared-quota classification records one), +a 403 entry is an entitlement refusal, an admission pause is a transient throttle, and a family +exclusion is spent quota unless its revalidation send is in flight. Ties go to the transient hold. +The refusal carries `usageLimit` only when no route candidate is eligible and every held account is +on spent quota. Its `resetsAt` is computed apart from admission order: an account reopens when the +last of its quota holds' upstream windows resets (`quotaResetAt`, recorded only when every rejected +window carried a reset, even when Retry-After set the deadline; or the family window reset), and the +pool reports the earliest such account. It is null when any held account's reset is unknown. +Retry-After remains the admission deadline. + +`anthropicPoolCooldownResponse` in `src/server/responses/core-errors.ts` answers a usage limit with +`formatUsageLimitReachedResponse` (`src/bridge/errors.ts`): 429, `usage_limit_reached` type and +code, optional `resets_at` in epoch seconds, no `plan_type`, Retry-After, and the +`usage_limit_reached` request-log code. Codex treats that class as final. The local sites are the +pool-enabled selection refusal in `request-transport.ts`, both `AnthropicAccountCooldownError` +catches in `adapter-dispatch.ts` and the passthrough catch; the final non-combo upstream 429 in +`adapter-dispatch.ts` and the first-iteration 429 of a routed web-search loop in `sidecar-execution.ts` +use `anthropicPoolUsageLimitAfterRefusal` after recovery found no replacement. +Combo children keep the earlier `rate_limit_error` body and wording, so combo hop and target-cooldown +classification are unchanged. Native Messages keeps Anthropic's `rate_limit_error` type. The image +bridge terminal, opt-in empty-completion continuations and Chat Completions translation keep their +existing rate-limit identity. + +`captureAnthropicCooldownRecovery` claims any live 429 refusal that named a rejected window, plus a +reset-derived aggregate refusal, regardless of deadline source. Settlement still requires the +claimed generation, a read no older than the claim, and every named window below 100 percent; a +claim forces a fresh usage dispatch, and a refusal recorded while a read is in flight moves the +generation so that read settles nothing. Entitlement 403 cooldowns and windowless Retry-After +refusals are never claimed. + +Regression coverage: `tests/adapters/anthropic/anthropic-pool-usage-limit.test.ts` and +`tests/adapters/anthropic/anthropic-quota-cooldown-recovery.test.ts`. + ## Family weekly admission `src/providers/quota/anthropic-family-headers.ts` attributes 7d_oi only to fixture-confirmed diff --git a/structure/transports/responses-failover.md b/structure/transports/responses-failover.md index d838cb395fd..6c8382e6b5e 100644 --- a/structure/transports/responses-failover.md +++ b/structure/transports/responses-failover.md @@ -1,6 +1,6 @@ # Responses Failover And Replay - `src/server/responses/request-transport.ts` resolves the final Anthropic model ID once for each enabled-pool request and holds its model route through admission. The three account-refusal retry sites in `adapter-dispatch.ts`, `adapter-continuation.ts` and `sidecar-execution.ts` use the same route, keep the original refusal if no replacement exists inside it, and preserve existing send and output replay limits. A local cooldown returns 429 with the earliest known Retry-After among accounts the route can use when they recover; explicit fallback includes usable ordinary-pool accounts even when the saved route IDs have been removed, while a strict route stays route-scoped. Selection, refusal and 429-rotation logs use the rule’s 1-based `route:#` position, never its name. + `src/server/responses/request-transport.ts` resolves the final Anthropic model ID once for each enabled-pool request and holds its model route through admission. The three account-refusal retry sites in `adapter-dispatch.ts`, `adapter-continuation.ts` and `sidecar-execution.ts` use the same route, keep the original refusal if no replacement exists inside it, and preserve existing send and output replay limits. A local cooldown returns 429 with the earliest known Retry-After among accounts the route can use when they recover, and answers a usage limit when every held account is on spent quota ([quota exhaustion answer](../providers/anthropic-account-pool.md#quota-exhaustion-answer-and-fresh-read-recovery)); explicit fallback includes usable ordinary-pool accounts even when the saved route IDs have been removed, while a strict route stays route-scoped. Selection, refusal and 429-rotation logs use the rule’s 1-based `route:#` position, never its name. Classified 429 admission follows the [Anthropic account-pool contract](../providers/anthropic-account-pool.md#classified-429-admission). Classified account-entitlement 403 recovery follows the [Anthropic account-pool contract](../providers/anthropic-account-pool.md#account-entitlement-refusal-recovery). `src/server/responses/compaction-recovery-policy.ts` is a pure eligibility policy, not a dispatcher. It requires explicit configuration and normalized attempt evidence, preserves ordinary requests, diff --git a/tests/adapters/anthropic/anthropic-cooldown-recovery.test.ts b/tests/adapters/anthropic/anthropic-cooldown-recovery.test.ts index 97573223efa..55848b9828d 100644 --- a/tests/adapters/anthropic/anthropic-cooldown-recovery.test.ts +++ b/tests/adapters/anthropic/anthropic-cooldown-recovery.test.ts @@ -6,6 +6,8 @@ import { join } from "node:path"; import { clearAnthropicAccountPoolState, getAnthropicAccountHealthSnapshot, + recordAnthropicAccount429, + recordAnthropicAccountRefusal, } from "../../../src/oauth/anthropic-routing"; import { getAccountSet, saveAccountCredential, saveCredential } from "../../../src/oauth/store"; import { clearAccountQuotaCache, fetchProviderAccountQuotas, getCachedProviderAccountQuota } from "../../../src/providers/quota"; @@ -106,16 +108,33 @@ describe("Anthropic reset-derived cooldown recovery", () => { expect(getAnthropicAccountHealthSnapshot(id)).toBeNull(); }); - test("successful quota reports do not clear Retry-After or guessed cooldowns", async () => { + test("successful quota reports clear named-window Retry-After and guessed cooldowns (#6989)", async () => { const id = await seed(); globalThis.fetch = (async () => quotaResponse(0, 0)) as typeof fetch; + // The shared-quota helper names the five-hour window as rejected, so the read can answer it + // whichever deadline the refusal recorded. rotateAnthropicAccountOn429(config, id, "3600", null, Date.now()); + expect(getAnthropicAccountHealthSnapshot(id)?.cooldownSource).toBe("retry-after"); + await fetchProviderAccountQuotas("anthropic", true); + expect(getAnthropicAccountHealthSnapshot(id)).toBeNull(); + + rotateAnthropicAccountOn429(config, id, null, null, Date.now(), new Headers()); + expect(getAnthropicAccountHealthSnapshot(id)?.cooldownSource).toBe("default"); + await fetchProviderAccountQuotas("anthropic", true); + expect(getAnthropicAccountHealthSnapshot(id)).toBeNull(); + }); + + test("successful quota reports do not clear windowless Retry-After or entitlement cooldowns", async () => { + const id = await seed(); + globalThis.fetch = (async () => quotaResponse(0, 0)) as typeof fetch; + + recordAnthropicAccount429(config, id, "3600", Date.now(), new Headers({ "anthropic-ratelimit-unified-status": "rejected" })); await fetchProviderAccountQuotas("anthropic", true); expect(getAnthropicAccountHealthSnapshot(id)?.cooldownSource).toBe("retry-after"); clearAnthropicAccountPoolState(); - rotateAnthropicAccountOn429(config, id, null, null, Date.now(), new Headers()); + recordAnthropicAccountRefusal(config, id, 403, null, Date.now()); await fetchProviderAccountQuotas("anthropic", true); expect(getAnthropicAccountHealthSnapshot(id)?.cooldownSource).toBe("default"); }); diff --git a/tests/adapters/anthropic/anthropic-model-routes.test.ts b/tests/adapters/anthropic/anthropic-model-routes.test.ts index 8100eb9c8f5..ca2d1700b5c 100644 --- a/tests/adapters/anthropic/anthropic-model-routes.test.ts +++ b/tests/adapters/anthropic/anthropic-model-routes.test.ts @@ -266,7 +266,8 @@ for (const adapter of ["anthropic", "openai-responses"] as const) { expect(Number(response.headers.get("retry-after"))).toBeGreaterThan(50); expect(Number(response.headers.get("retry-after"))).toBeLessThanOrEqual(60); const body = await response.json() as { error: { type: string; message: string } }; - expect(body.error.type).toBe("rate_limit_error"); + // The shared-quota helper names a rejected window, so the held route is out on spent quota (#6991). + expect(body.error.type).toBe("usage_limit_reached"); expect(body.error.message.includes("model route")).toBe(enabled); expect(body.error.message).not.toContain("private-scope"); expect(sends).toEqual([]); diff --git a/tests/adapters/anthropic/anthropic-pool-usage-limit.test.ts b/tests/adapters/anthropic/anthropic-pool-usage-limit.test.ts new file mode 100644 index 00000000000..466904c6600 --- /dev/null +++ b/tests/adapters/anthropic/anthropic-pool-usage-limit.test.ts @@ -0,0 +1,332 @@ +/** + * #6991: a pool whose every held account is out on spent quota answers a usage limit, which the + * Codex client stops on and renders with its reset time. Codex retries `rate_limit_exceeded` + * through its whole budget and ends on "exceeded retry limit, last status: 429", so that answer is + * kept only where a retry can succeed: a transient throttle, an entitlement refusal, a family + * revalidation in flight, or a combo child whose combo owns the next hop. + */ +import { afterEach, beforeEach, expect, spyOn, test } from "bun:test"; +import { acquireOwnedSpendHome } from "../../helpers/owned-spend-home"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + AnthropicAccountCooldownError, anthropicPoolCooldownError, clearAnthropicAccountPoolState, forgetAnthropicFailoverQuorum, + getAnthropicAccountHealthSnapshot, recordAnthropicAccount429, recordAnthropicAccountRefusal, +} from "../../../src/oauth/anthropic-routing"; +import { claimAnthropicFamilyRevalidation, clearAllAnthropicFamilyQuota, observeAnthropicFamilyQuota } from "../../../src/oauth/anthropic-model-quota"; +import { clearGenericFailoverHealth } from "../../../src/oauth/generic-account-failover"; +import { getAccountSet, saveCredential, setActiveAccount } from "../../../src/oauth/store"; +import { clearAccountQuotaCache, resetProviderQuotaReconcileStateForTests } from "../../../src/providers/quota"; +import { clearResponseStateForTests } from "../../../src/responses/state"; +import { handleResponses } from "../../../src/server/responses"; +import { anthropicPoolCooldownResponse } from "../../../src/server/responses/core-errors"; +import type { OcxConfig, OcxProviderConfig } from "../../../src/types"; +import { removeTreeWithRetry } from "../../helpers/remove-tree"; + +const originalHome = process.env.OPENCODEX_HOME; +let originalFetch: typeof globalThis.fetch; +let home: string; +let releaseSpendHome: (() => void) | undefined; +let sent: string[]; + +beforeEach(() => { + originalFetch = globalThis.fetch; + globalThis.fetch = (async () => { throw new Error("Unexpected global fetch in pool usage-limit test"); }) as typeof fetch; + home = mkdtempSync(join(tmpdir(), "ocx-anthropic-pool-usage-limit-")); + process.env.OPENCODEX_HOME = home; + sent = []; + clearAnthropicAccountPoolState(); + forgetAnthropicFailoverQuorum(); + clearAllAnthropicFamilyQuota(); + clearGenericFailoverHealth(); + clearAccountQuotaCache(); + resetProviderQuotaReconcileStateForTests(); + clearResponseStateForTests(); + releaseSpendHome = acquireOwnedSpendHome(); +}); + +afterEach(() => { + releaseSpendHome?.(); + releaseSpendHome = undefined; + clearAccountQuotaCache(); + clearAnthropicAccountPoolState(); + forgetAnthropicFailoverQuorum(); + clearAllAnthropicFamilyQuota(); + clearGenericFailoverHealth(); + resetProviderQuotaReconcileStateForTests(); + clearResponseStateForTests(); + globalThis.fetch = originalFetch; + if (originalHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = originalHome; + removeTreeWithRetry(home); +}); + +const MINUTE = 60_000; + +function credential(index: number) { + return { + access: `synthetic-anthropic-access-${index}`, + refresh: `synthetic-anthropic-refresh-${index}`, + expires: Date.now() + 3_600_000, + accountId: `synthetic-account-${index}`, + }; +} + +async function seed(count = 2): Promise { + for (let index = 0; index < count; index++) await saveCredential("anthropic", credential(index)); + const ids = getAccountSet("anthropic")!.accounts.map(account => account.id); + await setActiveAccount("anthropic", ids[0]!); + return ids; +} + +function spent(window: "5h" | "7d", retryAfterSeconds: number): Response { + return Response.json({ type: "error", error: { type: "rate_limit_error", message: "synthetic window spent" } }, { + status: 429, + headers: { + [`anthropic-ratelimit-unified-${window}-status`]: "rejected", + [`anthropic-ratelimit-unified-${window}-reset`]: String(Math.floor(Date.now() / 1000) + retryAfterSeconds), + "retry-after": String(retryAfterSeconds), + }, + }); +} + +function configFor(poolEnabled: boolean, reply: () => Response = () => { throw new Error("no send expected"); }): OcxConfig { + const transport = (async (_input, init) => { + sent.push(new Headers(init?.headers).get("authorization") ?? ""); + return reply(); + }) as typeof fetch; + const provider: OcxProviderConfig & { fetch: typeof fetch } = { + adapter: "anthropic", baseUrl: "https://anthropic-usage-limit.test", authMode: "oauth", + models: ["claude-sonnet-4-5", "claude-fable-5"], fetch: transport, + }; + return { port: 0, defaultProvider: "anthropic", anthropicAccountPool: { enabled: poolEnabled }, providers: { anthropic: provider } }; +} + +function post(config: OcxConfig, body: Record = {}) { + return handleResponses(new Request("http://localhost/v1/responses", { + method: "POST", headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "anthropic/claude-sonnet-4-5", input: "Answer briefly", stream: false, ...body }), + }), config, { model: "", provider: "" }); +} + +/** + * Quota refusal recorded the way the dispatch path records it: a rejected window with its reset + * and a Retry-After. Anthropic usually sets both to the same instant; `resetSeconds` lets a case + * separate them or drop the reset header. + */ +function holdOnQuota(config: OcxConfig, id: string, retryAfterSeconds: number, now: number, + resetSeconds: number | null = retryAfterSeconds): void { + const headers = new Headers({ "anthropic-ratelimit-unified-7d-status": "rejected" }); + if (resetSeconds !== null) headers.set("anthropic-ratelimit-unified-7d-reset", String(Math.ceil(now / 1000) + resetSeconds)); + expect(recordAnthropicAccount429(config, id, String(retryAfterSeconds), now, headers)).toBe(true); +} + +async function errorOf(response: Response): Promise> { + return ((await response.json()) as { error: Record }).error; +} + +function expectNoAccountMaterial(text: string, ids: readonly string[]): void { + for (const id of ids) expect(text).not.toContain(id); + for (const index of [0, 1]) { + expect(text).not.toContain(credential(index).access); + expect(text).not.toContain(credential(index).accountId); + } +} + +test.each([true, false])("every account held on quota answers usage_limit_reached with resets_at (pool enabled: %p)", async poolEnabled => { + const ids = await seed(2); + const config = configFor(poolEnabled); + const now = Date.now(); + holdOnQuota(config, ids[0]!, 40 * 60, now); + holdOnQuota(config, ids[1]!, 20 * 60, now); + + const response = await post(config); + const text = await response.clone().text(); + const error = await errorOf(response); + expect(response.status).toBe(429); + expect(error).toMatchObject({ type: "usage_limit_reached", code: "usage_limit_reached" }); + // The earliest reopening: the account held for twenty minutes. + expect(error.resets_at).toBe(Math.ceil(now / 1000) + 20 * 60); + expect(Number.isInteger(error.resets_at)).toBe(true); + expect(error).not.toHaveProperty("plan_type"); + expect(String(error.message)).toContain("usage limit"); + expect(Number(response.headers.get("retry-after"))).toBeGreaterThan(19 * 60); + expect(Number(response.headers.get("retry-after"))).toBeLessThanOrEqual(20 * 60); + expect(sent).toHaveLength(0); + expectNoAccountMaterial(text, ids); +}); + +test("resets_at is the upstream window reset, and is omitted when any held account's reset is unknown", async () => { + const ids = await seed(2); + const config = configFor(true); + const now = Date.now(); + // Retry-After (one minute) and the window reset (three hours) disagree: the reset is reported, + // the admission deadline stays Retry-After. + holdOnQuota(config, ids[0]!, 60, now, 3 * 3600); + holdOnQuota(config, ids[1]!, 120, now, 5 * 3600); + let response = await post(config); + let error = await errorOf(response); + expect(error).toMatchObject({ type: "usage_limit_reached", resets_at: Math.ceil(now / 1000) + 3 * 3600 }); + expect(Number(response.headers.get("retry-after"))).toBeLessThanOrEqual(60); + + // A refusal with no reset header could reopen at any time, so no reset is claimed for the pool. + holdOnQuota(config, ids[1]!, 120, now, null); + response = await post(config); + error = await errorOf(response); + expect(response.status).toBe(429); + expect(error).toMatchObject({ type: "usage_limit_reached", code: "usage_limit_reached" }); + expect(error).not.toHaveProperty("resets_at"); + expect(String(error.message)).not.toContain("resets at"); + expect(response.headers.get("retry-after")).not.toBeNull(); +}); + +test("a rejected window without its own reset leaves the pool reset unknown", async () => { + const ids = await seed(2); + const config = configFor(true); + const now = Date.now(); + // Both windows rejected, only the five-hour one dated: the weekly window could hold far longer. + for (const id of ids) { + expect(recordAnthropicAccount429(config, id, "600", now, new Headers({ + "anthropic-ratelimit-unified-5h-status": "rejected", + "anthropic-ratelimit-unified-5h-reset": String(Math.ceil(now / 1000) + 600), + "anthropic-ratelimit-unified-7d-status": "rejected", + }))).toBe(true); + } + const error = await errorOf(await post(config)); + expect(error).toMatchObject({ type: "usage_limit_reached" }); + expect(error).not.toHaveProperty("resets_at"); +}); + +test("the reported reset is the last quota window to reopen, not the admission deadline that wins", async () => { + const ids = await seed(2); + const config = configFor(true); + const now = Date.now(); + const second = Math.ceil(now / 1000) * 1000; + // Shared quota: Retry-After ten minutes, window reset three hours. Family evidence expires for + // admission in thirty minutes, its window resets in one hour. Admission waits on the family; + // usage reopens only when the shared window does. + holdOnQuota(config, ids[0]!, 600, now, 3 * 3600); + observeAnthropicFamilyQuota(ids[0]!, [{ label: "Fable", scope: "model", percent: 100, rejected: true, resetAt: second + 60 * MINUTE }], now, true); + holdOnQuota(config, ids[1]!, 600, now, 4 * 3600); + const refusal = anthropicPoolCooldownError(now, null, "claude-fable-5"); + expect(refusal.usageLimit).toEqual({ resetsAt: second + 3 * 60 * MINUTE }); + expect(refusal.retryAfterSeconds).toBe(600); +}); + +test("a routed web-search turn whose last account refuses on spent quota answers the usage limit", async () => { + const ids = await seed(2); + const config = configFor(true, () => spent("5h", 1800)); + config.webSearchSidecar = { backend: "anthropic", enabled: true }; + const warn = spyOn(console, "warn").mockImplementation(() => {}); + try { + const response = await post(config, { tools: [{ type: "web_search" }] }); + const text = await response.clone().text(); + const error = await errorOf(response); + expect(sent).toHaveLength(2); + expect(response.status).toBe(429); + expect(error).toMatchObject({ type: "usage_limit_reached", code: "usage_limit_reached" }); + expect(Math.abs(Number(error.resets_at) - (Math.floor(Date.now() / 1000) + 1800))).toBeLessThanOrEqual(2); + expectNoAccountMaterial(text, ids); + } finally { + warn.mockRestore(); + } +}); + +test.each(["transient throttle", "entitlement 403"] as const)("one %s among quota holds keeps the retryable rate limit", async kind => { + const ids = await seed(2); + const config = configFor(true); + const now = Date.now(); + holdOnQuota(config, ids[0]!, 20 * 60, now); + if (kind === "entitlement 403") { + expect(recordAnthropicAccountRefusal(config, ids[1]!, 403, "600", now)).toBe(true); + } else { + // A Retry-After with no rejected window is an admission pause, not spent quota. + expect(recordAnthropicAccount429(config, ids[1]!, "30", now, new Headers())).toBe(false); + expect(getAnthropicAccountHealthSnapshot(ids[1]!)).toBeNull(); + } + + const response = await post(config); + const error = await errorOf(response); + expect(response.status).toBe(429); + expect(error).toMatchObject({ type: "rate_limit_error", code: "rate_limit_exceeded" }); + expect(error).not.toHaveProperty("resets_at"); + expect(response.headers.get("retry-after")).not.toBeNull(); + expect(sent).toHaveLength(0); +}); + +test("the last account refusing on spent quota answers the usage limit at once", async () => { + const ids = await seed(2); + const config = configFor(true, () => spent("5h", 1800)); + const warn = spyOn(console, "warn").mockImplementation(() => {}); + const lines: string[] = []; + warn.mockImplementation((...args: unknown[]) => { lines.push(args.map(String).join(" ")); }); + try { + const response = await post(config); + const text = await response.clone().text(); + const error = await errorOf(response); + expect(sent).toEqual([`Bearer ${credential(0).access}`, `Bearer ${credential(1).access}`]); + expect(response.status).toBe(429); + expect(error).toMatchObject({ type: "usage_limit_reached", code: "usage_limit_reached" }); + expect(Math.abs(Number(error.resets_at) - (Math.floor(Date.now() / 1000) + 1800))).toBeLessThanOrEqual(2); + expect(response.headers.get("retry-after")).not.toBeNull(); + expectNoAccountMaterial(text, ids); + expect(lines.some(line => line.includes("no eligible replacement"))).toBe(true); + expectNoAccountMaterial(lines.join("\n"), ids); + } finally { + warn.mockRestore(); + } +}); + +test("an upstream refusal while a sibling is only throttled keeps the provider's rate limit", async () => { + const ids = await seed(2); + const config = configFor(true, () => spent("5h", 1800)); + // The second account is held by a transient pause only, so the pool is not exhausted. + expect(recordAnthropicAccount429(config, ids[1]!, "30", Date.now(), new Headers())).toBe(false); + const response = await post(config); + const error = await errorOf(response); + expect(sent).toHaveLength(1); + expect(response.status).toBe(429); + expect(error.code).toBe("rate_limit_exceeded"); + expect(error).not.toHaveProperty("resets_at"); +}); + +test("a spent model family is quota, but a family revalidation in flight is not", async () => { + const ids = await seed(2); + const config = configFor(true); + const now = Date.now(); + holdOnQuota(config, ids[0]!, 20 * 60, now); + const fable = [{ label: "Fable", scope: "model" as const, percent: 100, rejected: true, resetAt: now + 90 * MINUTE }]; + + observeAnthropicFamilyQuota(ids[1]!, fable, now, true); + const spentFamily = anthropicPoolCooldownError(now, null, "claude-fable-5"); + // Both resets are known (the shared window's header and the family window's), so the earlier one. + expect(spentFamily.usageLimit).toEqual({ resetsAt: Math.ceil(now / 1000) * 1000 + 20 * MINUTE }); + // The family does not hold other models, so the pool is not exhausted for Sonnet at all. + expect(anthropicPoolCooldownError(now, null, "claude-sonnet-4-5").usageLimit).toBeNull(); + + // Passive evidence older than thirty minutes admits one revalidation send; while it is in + // flight the family may reopen within the second, so a retry is the right answer. + observeAnthropicFamilyQuota(ids[1]!, fable, now - 31 * MINUTE, true); + expect(claimAnthropicFamilyRevalidation(ids[1]!, "claude-fable-5", now)).not.toBeNull(); + const revalidating = anthropicPoolCooldownError(now, null, "claude-fable-5"); + expect(revalidating.usageLimit).toBeNull(); + expect(revalidating.retryAfterSeconds).toBe(1); +}); + +test("a combo child keeps the retryable form and wording so its combo still owns the hop and target cooldown", async () => { + const now = Date.now(); + const refusal = new AnthropicAccountCooldownError(60, undefined, { resetsAt: now + MINUTE }); + const logCtx: { errorCode?: string } = {}; + const combo = anthropicPoolCooldownResponse(refusal, { comboAttempt: true, logCtx }); + // Byte-for-byte what a combo child answered before: combo exhaustion also matches usage-limit prose. + expect(await errorOf(combo)).toEqual({ + type: "rate_limit_error", code: "rate_limit_exceeded", message: "All Anthropic OAuth accounts are temporarily rate-limited", + }); + expect(combo.headers.get("retry-after")).toBe("60"); + expect(logCtx.errorCode).toBeUndefined(); + + const direct = anthropicPoolCooldownResponse(refusal, { logCtx }); + expect(await errorOf(direct)).toMatchObject({ type: "usage_limit_reached", resets_at: Math.ceil((now + MINUTE) / 1000) }); + expect(logCtx.errorCode).toBe("usage_limit_reached"); +}); diff --git a/tests/adapters/anthropic/anthropic-quota-cooldown-recovery.test.ts b/tests/adapters/anthropic/anthropic-quota-cooldown-recovery.test.ts new file mode 100644 index 00000000000..1c26ba9bc56 --- /dev/null +++ b/tests/adapters/anthropic/anthropic-quota-cooldown-recovery.test.ts @@ -0,0 +1,161 @@ +/** + * #6989: a fresh usage read that shows every rejected window open lifts a 429 quota cooldown + * whatever set its deadline. Anthropic sends Retry-After on a spent window too, set to that + * window's reset, so a plan upgrade or an early window rollover used to leave the account out + * until the old deadline or a manual clear-cooldown. Stale reads and non-quota refusals still + * settle nothing. + */ +import { afterEach, beforeEach, expect, test } from "bun:test"; +import { createAnthropicInstanceFixture, INSTANCE_FIXTURE_INSTANCES, type AnthropicInstanceFixture } from "../../helpers/anthropic-instance-fixture"; + +let f: AnthropicInstanceFixture; +beforeEach(async () => { f = await createAnthropicInstanceFixture(); await f.seed(); }); +afterEach(async () => { await f?.dispose(); }); + +const HOUR = 3_600_000; + +function rejected(windows: readonly ("5h" | "7d")[], resetAt?: number): Headers { + const headers = new Headers(); + for (const window of windows) { + headers.set(`anthropic-ratelimit-unified-${window}-status`, "rejected"); + if (resetAt !== undefined) headers.set(`anthropic-ratelimit-unified-${window}-reset`, String(Math.floor(resetAt / 1000))); + } + return headers; +} + +function usage(fiveHour: number, weekly: number): void { + const resetsAt = new Date(Date.now() + HOUR).toISOString(); + globalThis.fetch = (async () => Response.json({ + five_hour: { utilization: fiveHour, resets_at: resetsAt }, + seven_day: { utilization: weekly, resets_at: resetsAt }, + limits: [], + })) as typeof fetch; +} + +async function freshRead(instance: "anthropic" | "anthropic2", id: string): Promise { + const { fetchAnthropicAccountQuota } = await import("../../../src/providers/quota/anthropic-account-quota"); + await fetchAnthropicAccountQuota(instance, id, true, undefined, true); +} + +for (const instance of INSTANCE_FIXTURE_INSTANCES) { + test(`${instance}: a fresh read with headroom lifts a Retry-After quota cooldown`, async () => { + const routing = f.routing.anthropicRoutingFor(instance); + const id = f.ids[0]; + const now = Date.now(); + // The shape from the report: a weekly refusal whose Retry-After equals the window reset. + routing.recordAnthropicAccount429(f.config, id, String(4 * 24 * 3600), now, rejected(["5h", "7d"], now + 4 * 24 * HOUR)); + expect(routing.getAnthropicAccountHealthSnapshot(id)).toMatchObject({ cooldownSource: "retry-after" }); + + usage(0, 0); + await freshRead(instance, id); + + expect(routing.getAnthropicAccountHealthSnapshot(id)).toBeNull(); + expect(routing.getEligibleAnthropicAccounts()).toContain(id); + }); +} + +test("a read that still shows one rejected window full keeps the cooldown", async () => { + const routing = f.routing.anthropicRoutingFor("anthropic"); + const id = f.ids[0]; + const now = Date.now(); + routing.recordAnthropicAccount429(f.config, id, "86400", now, rejected(["5h", "7d"])); + usage(0, 100); + await freshRead("anthropic", id); + expect(routing.getAnthropicAccountHealthSnapshot(id)).toMatchObject({ cooldownSource: "retry-after" }); +}); + +test("a read taken before the claim is stale and settles nothing", () => { + const routing = f.routing.anthropicRoutingFor("anthropic"); + const id = f.ids[0]; + const now = Date.now(); + routing.recordAnthropicAccount429(f.config, id, "86400", now, rejected(["7d"])); + const claim = routing.captureAnthropicCooldownRecovery(id, now + 10); + expect(claim).not.toBeNull(); + expect(routing.settleAnthropicCooldownRecovery(claim!, { fiveHourPercent: 0, weeklyPercent: 0, updatedAt: now + 9 })).toBe("superseded"); + expect(routing.getAnthropicAccountHealthSnapshot(id)).not.toBeNull(); + // A newer refusal moves the generation, so the older claim cannot lift it either. + routing.recordAnthropicAccount429(f.config, id, "86400", now + 20, rejected(["7d"])); + expect(routing.settleAnthropicCooldownRecovery(claim!, { fiveHourPercent: 0, weeklyPercent: 0, updatedAt: now + 30 })).toBe("superseded"); + expect(routing.getAnthropicAccountHealthSnapshot(id)).not.toBeNull(); +}); + +test("a quota refusal with no usable deadline is recoverable as well", () => { + const routing = f.routing.anthropicRoutingFor("anthropic"); + const id = f.ids[0]; + const now = Date.now(); + // Rejected status without a reset or Retry-After: the sixty-second default deadline. + routing.recordAnthropicAccount429(f.config, id, null, now, rejected(["5h"])); + expect(routing.getAnthropicAccountHealthSnapshot(id, now)).toMatchObject({ cooldownSource: "default" }); + const claim = routing.captureAnthropicCooldownRecovery(id, now + 1); + expect(routing.settleAnthropicCooldownRecovery(claim!, { fiveHourPercent: 3, weeklyPercent: 100, updatedAt: now + 2 })).toBe("cleared"); + expect(routing.getAnthropicAccountHealthSnapshot(id, now + 3)).toBeNull(); +}); + +test("an entitlement 403 and a refusal naming no window are never claimed", () => { + const routing = f.routing.anthropicRoutingFor("anthropic"); + const now = Date.now(); + routing.recordAnthropicAccountRefusal(f.config, f.ids[0], 403, "600", now); + routing.recordAnthropicAccount429(f.config, f.ids[1], "600", now, new Headers({ "anthropic-ratelimit-unified-status": "rejected" })); + expect(routing.getAnthropicAccountHealthSnapshot(f.ids[1], now)).toMatchObject({ cooldownSource: "retry-after" }); + expect(routing.captureAnthropicCooldownRecovery(f.ids[0], now + 1)).toBeNull(); + expect(routing.captureAnthropicCooldownRecovery(f.ids[1], now + 1)).toBeNull(); +}); + +test("an aggregate-only reset-derived refusal stays claimable but is always retained", () => { + const routing = f.routing.anthropicRoutingFor("anthropic"); + const id = f.ids[0]; + const now = Date.now(); + routing.recordAnthropicAccount429(f.config, id, null, now, new Headers({ + "anthropic-ratelimit-unified-status": "rejected", + "anthropic-ratelimit-unified-reset": String(Math.floor((now + HOUR) / 1000)), + })); + expect(routing.getAnthropicAccountHealthSnapshot(id, now)).toMatchObject({ cooldownSource: "reset-derived" }); + const claim = routing.captureAnthropicCooldownRecovery(id, now + 1); + expect(claim).not.toBeNull(); + expect(routing.settleAnthropicCooldownRecovery(claim!, { fiveHourPercent: 0, weeklyPercent: 0, updatedAt: now + 2 })).toBe("retained"); +}); + +test("a usage read that started before a Retry-After refusal cannot lift it", async () => { + const routing = f.routing.anthropicRoutingFor("anthropic"); + const id = f.ids[0]; + let release!: () => void; + const held = new Promise(resolve => { release = resolve; }); + let entered!: () => void; + const started = new Promise(resolve => { entered = resolve; }); + const resetsAt = new Date(Date.now() + HOUR).toISOString(); + globalThis.fetch = (async () => { + entered(); + await held; + return Response.json({ five_hour: { utilization: 0, resets_at: resetsAt }, seven_day: { utilization: 0, resets_at: resetsAt }, limits: [] }); + }) as typeof fetch; + const read = freshRead("anthropic", id); + await started; + routing.recordAnthropicAccount429(f.config, id, "86400", Date.now(), rejected(["7d"])); + release(); + await read; + expect(routing.getAnthropicAccountHealthSnapshot(id)).toMatchObject({ cooldownSource: "retry-after" }); +}); + +for (const instance of INSTANCE_FIXTURE_INSTANCES) { + const other = instance === "anthropic" ? "anthropic2" : "anthropic"; + test(`${instance}: a pool exhausted on quota is a usage limit only in its own instance`, () => { + const routing = f.routing.anthropicRoutingFor(instance); + const now = Date.now(); + for (const id of f.ids) routing.recordAnthropicAccount429(f.config, id, "600", now, rejected(["7d"], now + 2 * HOUR)); + const refusal = routing.anthropicPoolCooldownError(now); + expect(refusal.usageLimit).toEqual({ resetsAt: Math.floor((now + 2 * HOUR) / 1000) * 1000 }); + expect(refusal.retryAfterSeconds).toBe(600); + const sibling = f.routing.anthropicRoutingFor(other).anthropicPoolCooldownError(now); + expect(sibling.usageLimit).toBeNull(); + expect(sibling.retryAfterSeconds).toBeNull(); + }); +} + +test("a transient hold ending with a quota hold on the same account keeps the refusal retryable", () => { + const routing = f.routing.anthropicRoutingFor("anthropic"); + const now = Date.now(); + for (const id of f.ids) routing.recordAnthropicAccount429(f.config, id, "600", now, rejected(["7d"], now + 600_000)); + expect(routing.anthropicPoolCooldownError(now).usageLimit).not.toBeNull(); + f.ratePolicy.anthropicRatePolicyFor("anthropic").pauseAnthropicRateAdmission(f.ids[0], now + 600_000); + expect(routing.anthropicPoolCooldownError(now).usageLimit).toBeNull(); +}); diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index 88ce20ca193..4dc096c7133 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -22,7 +22,7 @@ "anthropic-instance-isolation.test.ts": "adapters/anthropic", "anthropic-instance-pool-parity.test.ts": "adapters/anthropic", "anthropic-instance-quota.test.ts": "adapters/anthropic", "anthropic-instance-recovery.test.ts": "adapters/anthropic", "anthropic2-fast-parity.test.ts": "adapters/anthropic", "anthropic2-native-routing.test.ts": "claude-integration", - "oauth-anthropic-instance-health.test.ts": "oauth", "oauth-account-plans.test.ts": "oauth", "anthropic2-responses-parity.test.ts": "responses", + "oauth-anthropic-instance-health.test.ts": "oauth", "anthropic-quota-cooldown-recovery.test.ts": "adapters/anthropic", "anthropic-pool-usage-limit.test.ts": "adapters/anthropic", "oauth-account-plans.test.ts": "oauth", "anthropic2-responses-parity.test.ts": "responses", "oauth-anthropic-instance-multiprocess.test.ts": "oauth", "anthropic2-discovery-ownership.test.ts": "oauth", "codex-shim-runtime-preflight.test.ts": "codex-integration", From a061e700868e9d0bcacd55094cd625369cd739d0 Mon Sep 17 00:00:00 2001 From: lidge-jun Date: Tue, 13 Oct 2026 01:09:34 +0900 Subject: [PATCH 5/9] fix(responses): retry and fail over a pooled Codex turn the upstream refuses before any output (#6969) (#6997) Squashed from #6997 for the 2.84.0 release train. Pooled native Codex turns that the ChatGPT backend refuses before any output (`server_is_overloaded` 503, a wrapped 5xx such as `upstream_server_error` 502) now retry the same account with the existing capped backoff and then move to one other eligible Pool account. Before this change every such refusal failed after a single send (`sendCount: 1`, `recoveryKinds: []`, `failureStage: protocol-prelude`). **Why it was missed.** All field failures rode the upstream WebSocket (content-free stage fields: 233 rows with `relayedEvents: 3, controlFrames: 2` = `response.created`, `response.in_progress`, then a terminal `error` frame; 47 rows with the error as the first event). `codexWsExchange` committed a 200 at `response.created`, and `wrappedRejectionResponse` projects only pre-commit **4xx** frames, so `fetchWithTransientRetry` and the Pool transient move (`shouldRetryCodexPoolAccountTransient`) never saw a 5xx. HTTP-status 5xx already took both paths. Native compact ran the same-account ladder but moved accounts only on 429/402. **What changed.** - `codex-ws-exchange.ts`/`codex-ws-wire.ts`: an opt-in hold (at most 10 s and never past half of `connectTimeoutMs` after the send starts, non-resetting; field data puts 97.5% of refusals within 5 s and 98.2% within 10 s) queues empty `response.created`/`in_progress`/`queued` frames without committing. An **authoritative** refusal that follows (bare `error` with one transient-5xx wrapper status, or the recognized overload class; empty `response.failed` of the overload class) settles as a replayable gateway status. Quota/`rate_limit*` identities, 4xx, stream ids, conflicting response ids, competing aliases, or headers that cannot be projected never qualify, and message text can only veto. Every other settle during the hold (output, socket death, silence, abort, queue/size limits, deadline) commits the queued preamble first, so it is the same committed-body failure as before; silence and socket death keep their non-replayable settles and the `retryOnReset` boundary is unchanged. - `passthrough-dispatch.ts`/`core-codex-account.ts`: only Pool forward sends (initial ladder and the alternate-account send) opt in; combo children, native steering/injection, 401-refresh and rebuild legs do not. A projection nothing recovered is restored right after the Pool block to the original event stream (same bytes, markers and stage record), so an exhausted pool answers the client exactly as before. Each refusal's account outcome is written once (move helper, delivery, or the dispatch catch on abort/budget refusal during backoff). - `compact.ts`: a transient 5xx surviving the ladder moves to one alternate Pool account, admitted under the alternate's identity, keeping the first answer whenever the alternate's send never physically dispatched (budget, dispatch guard such as disabled credits, pacing); a 5xx whose body confirms quota keeps the quota path. - 429/401/402/403 semantics are unchanged. Docs: `structure/transports/responses-wire-shapes.md`, `responses-failover.md`, and the Codex integration guide. `tests/responses/responses-account-label.test.ts` (late WS quota case) awaited the handler right after a bare `response.created`; it now emits one output event after it so the commit point is explicit. The quota-attribution assertions are unchanged. Closes #6969 Closes #6969 --- .../000_plan.md | 70 ++++ .../010_plan.md | 217 +++++++++++ .../content/docs/guides/codex-integration.md | 10 + scripts/test-layout/layout.json | 2 +- src/server/responses/codex-ws-exchange.ts | 141 ++++++- src/server/responses/codex-ws-wire.ts | 161 ++++++++ src/server/responses/compact.ts | 174 ++++++--- src/server/responses/core-codex-account.ts | 106 ++++- src/server/responses/fetch-helpers.ts | 9 +- src/server/responses/passthrough-delivery.ts | 5 +- src/server/responses/passthrough-dispatch.ts | 14 +- src/server/responses/ws-upstream.ts | 5 +- structure/transports/responses-failover.md | 2 +- structure/transports/responses-wire-shapes.md | 2 +- tests/fixtures/test-layout-expected.json | 2 +- .../compact-transient-alternate.test.ts | 303 +++++++++++++++ .../responses/responses-account-label.test.ts | 4 + .../responses-pool-overload-failover.test.ts | 307 +++++++++++++++ tests/responses/ws-pre-output-refusal.test.ts | 367 ++++++++++++++++++ 19 files changed, 1838 insertions(+), 63 deletions(-) create mode 100644 devlog/_plan/261012_6969_pool_overload_failover/000_plan.md create mode 100644 devlog/_plan/261012_6969_pool_overload_failover/010_plan.md create mode 100644 tests/responses/compact-transient-alternate.test.ts create mode 100644 tests/responses/responses-pool-overload-failover.test.ts create mode 100644 tests/responses/ws-pre-output-refusal.test.ts diff --git a/devlog/_plan/261012_6969_pool_overload_failover/000_plan.md b/devlog/_plan/261012_6969_pool_overload_failover/000_plan.md new file mode 100644 index 00000000000..230904908d0 --- /dev/null +++ b/devlog/_plan/261012_6969_pool_overload_failover/000_plan.md @@ -0,0 +1,70 @@ +# 6969 — pooled native Codex: retry and fail over on a pre-output upstream overload + +Issue: [#6969](https://github.com/lidge-jun/opencodex/issues/6969). Base: `origin/dev` `dc45bb2ca9`. + +## Symptom + +Pooled native Codex requests to `gpt-6.1-sol` end after one send when the ChatGPT backend +declines before any output with `server_is_overloaded` (503) or `upstream_server_error` (502): +`sendCount: 1`, `recoveryKinds: []`, `failureStage: protocol-prelude`. The pool neither retries +the account nor moves to another one, so the Codex client reports "Selected model is at capacity". + +## Field evidence (content-free stage fields only) + +Aggregated from the reporter's own request log (status, error code, `codexWsStage` counters, +failure stage/cause; no bodies, headers or account identifiers), OpenCodex 2.83.0: + +| Shape | Rows | +| --- | --- | +| WS, `relayedEvents: 3`, `controlFrames: 2`, `protocol-prelude`, 502 `upstream-fault` | 213 | +| WS, `relayedEvents: 1` (error is the first Responses event), 502 `upstream-fault` | 47 | +| WS, `relayedEvents: 3`, 503 `server_is_overloaded`, `upstream-declined` | 20 | + +One further 502 `upstream-fault` row relayed 336 events (output had started) and is an outlier +kept out of the shapes above. Every row carries a `codexWsStage`, so every observed failure rode the +upstream WebSocket. The counters are consistent with `response.created`, `response.in_progress` and a +terminal `error` frame, but they count frames, not event types, and cannot prove which events those +were or that no hosted tool ran. They explain the failure; they authorize nothing — replay +eligibility is decided from the frames themselves at run time (010). Time from the first Responses event to that refusal, over the +281 authoritative refusals (`upstream-declined` and `upstream-fault`): 70% within 2 s, 97.5% within +5 s, 98.2% within 10 s, and no further gain up to 60 s. + +## Why each path misses the existing recovery + +The recovery already exists: `fetchWithTransientRetry` (`src/lib/upstream-retry.ts`) retries a +transient 5xx on the same account up to three sends with capped, jittered, abort-aware backoff +against the request's shared send budget, and the pool block in `passthrough-dispatch.ts` moves a +surviving `shouldRetryCodexPoolAccountTransient` 5xx to one alternate account +(`retryCodexPoolOnAlternateAccount`, guarded by fixed accounts, a spent stored-401 budget, uploaded +files, the account-failover permit and the pool recovery window). Both read the HTTP status. + +- **WebSocket (all field rows).** `codexWsExchange` resolves a 200 SSE Response at the first + non-control frame (`commitResponse`). `wrappedRejectionResponse` projects only a pre-commit + **4xx** `error` frame into an HTTP status, by design ("unlike its native client, this relay + converts only precommit 4xx"). So a 5xx `error` frame — first, or after an empty + `response.created`/`response.in_progress` preamble — is relayed inside a committed 200 body. No + retry layer sees a 5xx; the failure is only classified later from the SSE terminal. +- **HTTP status 5xx (HTTP SSE/JSON).** Handled: the same-account ladder runs, then the pool move. +- **HTTP SSE in-stream zero-output failure** (200 head, then `response.failed`/`error`). Not + retried; direct passthrough returns headers before the first event. No field evidence; out of + scope here and recorded as residual. +- **Compact (`/v1/responses/compact`).** `compact.ts` runs the same-account ladder on an HTTP 5xx, + but its alternate-account move recognises only 429/402, so a 5xx that survives the ladder never + moves accounts. + +## What is deliberately not replayed + +Ingwannu's triage separates an authoritative terminal refusal from socket death or silence after +the send. The latter are settled as non-replayable gateway statuses +(`codexWsPreResponseFailure`) because the origin may still be running the turn; only an operator +`retryOnReset` grant may replace them. That boundary does not move. A refusal frame is different +evidence: the origin itself terminated that response with a structured server status, and the +native Codex client treats the same wrapped frame as a terminal HTTP error even after +`response.created`. That is client-visible terminality, not a backend proof that nothing ran, so +010 replays only when every relayed event was an empty lifecycle frame and the terminal carries no +output — nothing the client could have seen or acted on. + +## Unit map + +- `010_plan.md` — the single implementation work-phase (wp1), diff level. + diff --git a/devlog/_plan/261012_6969_pool_overload_failover/010_plan.md b/devlog/_plan/261012_6969_pool_overload_failover/010_plan.md new file mode 100644 index 00000000000..afc2cc974d2 --- /dev/null +++ b/devlog/_plan/261012_6969_pool_overload_failover/010_plan.md @@ -0,0 +1,217 @@ +# 010 — wp1: recover a pre-output upstream refusal on the native Codex pool + +Research and evidence: [000_plan.md](000_plan.md). Base `dc45bb2ca9`. + +## Loop-spec header + +- Class: C3 (transport contract change on a shared path; no persistence or credential change). +- Outcome: an authoritative upstream 5xx refusal that arrives on the Codex WebSocket before any + output reaches the existing same-account transient ladder and the pool alternate-account move; + compact moves accounts on a surviving transient 5xx. Exhausted recovery delivers exactly what the + client receives today. +- Tool/credential scope: local checkout, isolated temp homes and fake sockets only. No live + `~/.opencodex`, :10100 service, Codex app or GOAT server. Write scope: the files below. +- Resource bounds: none stated; host limits apply. +- Terminal outcomes: DONE (PR open, review PASS, exact-head CI green), BLOCKED/NEEDS_HUMAN only for + secrets, cost, publication or permission decisions. + +## Architect consultation (round 1, gpt-6.1-sol high) and dispositions + +| ID | Proposal | Disposition | +| --- | --- | --- | +| D1 | Internal opt-in threaded ProviderFetchOptions → codexWsUpstreamFetch → codexWsExchange; canonical metadata exchanges only; no native control, sidecars or combos-by-accident | Accepted. Enabled only on Pool forward sends (`usesCodexForwardPoolAuth`), fixed Pool accounts included (same-account ladder only). | +| D2 | Hold only structurally empty lifecycle frames, check every held snapshot; freeze prelude headers and `metadata.commit()` at the first Responses event; non-resetting 10 s bound | Accepted. Bound = `min(10_000, floor(connectMs / 2))`. | +| D3 | Strict refusal identity: correlation first, no foreign `stream_id`, empty preamble and empty terminal output, explicit structured evidence only; no message-text authorization | Accepted. See "Refusal identity". | +| D4 | Commit before failing only when a Responses preamble was actually held | Accepted. | +| D5 | Restore immediately after the Pool block on every loop pass, carrying WS/quota markers and the same stage object; single use | Accepted. Affinity diagnostics (captured before the Pool block) record the true refusal status; they are diagnostics only. | +| D6 | Compact: transient predicate, guards kept, quota precedence kept, admission secured before cancelling A, no promise of B promotion | Accepted. | +| D7 | Exactly-once health outcome: unmoved projection must not be recorded both in the retry helper and by the restored stream's terminal recorder | Accepted as the exactly-once marker (audit dispositions 2 and 4). | +| D8 | Amend the "only precommit 4xx" comment and the transport contract doc | Accepted. | +| Alt | Route Pool over HTTP | Rejected: loses WS latency, still misses HTTP in-stream refusals. | + +## Refusal identity (codex-ws-wire.ts, new `codexWsPreOutputRefusalStatus(payload)`) + +Returns a status only for: + +1. `type: "error"`, no `stream_id`, exactly one integer wrapper alias (`codexWsWrapperStatus`) with + `isTransientUpstreamStatus`; or no wrapper alias at all and `recognizedUpstreamError(error)` is the + overload class (503). +2. `type: "response.failed"` whose `response.output` is absent or empty and whose + `response.error` is the recognized overload class (503). + +Never for: a rate-limit/quota code (`recognizedUpstreamError` 429, `insufficient_quota`, +`usage_limit_reached`, `rate_limit_*`), a message that `isRateLimitOrQuotaFailureMessage` reads as +quota, a 4xx, competing aliases, a malformed `error` member. + +## Reflection round 2 (MISALIGNED, narrow) — corrections folded + +1. The whole new refusal path, first-frame and held, runs only under `holdEnabled && sent && !responseCommitted`; opt-out and native-control exchanges are byte-identical to today. +2. A replay-eligibility latch, independent of correlation being present: it drops on any frame carrying a `stream_id`, on a lifecycle or `response.failed` frame whose `response.id` differs from the first held id, and on any non-empty snapshot. Once dropped, the exchange commits and behaves as today. Cold, reused and one-shot sockets all apply it. +3. First-frame and held refusals settle identically: the terminal frame goes through today's frame-size and queue checks, is recorded as `terminalEvent`, enqueued, the stream closed, the session released as a failed terminal, and only then the projection resolves. A restore therefore always carries the full original bytes. +4. Tests add: foreign id on a cold socket, metadata during the hold (restored headers equal the frozen prelude), queue overflow during the hold, denied alternate admission keeping the original body, single health/quota record, combo consumer receiving restored SSE. + +## Audit round 1 (FAIL, 3 blockers) — dispositions + +1. **Foreign terminal on a cold socket (accepted).** The latch drops on every identity signal + `CodexWsCorrelation.accept` reads, independent of `strict`: any `stream_id`; a lifecycle or + `response.failed` frame whose `response.id` or top-level `response_id` differs from the first held + `response.created` id; a frame carrying two different ids; an `error` frame whose `response_id` + differs from the held id. A refusal with no held id may carry at most one id and no `stream_id`. + Test: held id A, then `response.failed` with `response.id = A`, `response_id = B` → one send. +2. **Late alternate-admission denial (accepted; revised after audit round 2).** The helper's + order is NOT changed (moving A's record/identity after admission charged B's permit under A's + spend identity and lost A's outcome on the credits-off exit). Instead an exactly-once marker: + `noteCodexWsPreOutputRefusalOutcome(projection)` is set whenever the helper records A's outcome for + a projection (`recordFirstOutcome`, `recordUnmovedTransientOutcome`); `recordUnmovedTransientOutcome` + skips a projection already marked (late window/permit denial after `recordFirstOutcome`); the + restore carries the marker to the restored response; delivery's terminal recorder skips the + health write for a marked restored response while still running its other terminal callbacks. + An unmarked restored projection (fixed account, non-Pool forward account, no move attempted) is + recorded once by delivery. Credits-off keeps today's order: A recorded, then the policy refusal. + Residual, pre-existing and shared with the HTTP 5xx path: after a late denial the attempt's log + identity already names B while A's (restored) bytes are delivered; health is attributed to A. + Tests: no alternate available, window denial → one A outcome, A's bytes, zero B sends. +3. **Compact admission and spend identity (accepted, without an early reservation).** The transient + arm switches `logCtx.accountLogLabel`/`outcomeCtx` to B and runs `sendSingleCompactAttempt` for B + with A's body still uncancelled, so every budget/spend check runs under B's identity. A + `SendBudgetExhaustedError` from that helper is always pre-send; on it the arm restores A's label + and context, releases B's probe lease and continues with A's response untouched (recorded once at + the end as today). After B is actually sent (success, HTTP failure or transport failure) A is + recorded with its real 5xx and its body cancelled. `permit.use()` failure inside the helper now + releases its reservation. The quota arm keeps its current order. Tests: B admitted → B serves; + B refused (budget spent) → A's 503 body and one A outcome. + +4. **Abort or local refusal during the same-account backoff (audit rounds 3-4, accepted).** The + initial-send closure in `passthrough-dispatch.ts` keeps the last projection it observed together + with the account context that produced it. One owner per exceptional exit of + `fetchWithTransientRetry`: + - caller abort (`options.abortSignal`/`req.signal` aborted) or `SendBudgetExhaustedError`: + `transportFailureResponse` writes no account health on these exits (it returns at its first two + branches), so the dispatch catch records the unmarked projection once as its real 5xx against + the producing account, marks it, then calls `transportFailureResponse` unchanged; + - any other exception (a later send's timeout/connection error wrapped in + `UpstreamRetryEvidenceError`): the dispatch catch writes nothing and `transportFailureResponse` + remains the single writer, as today. + Recording happens before `transportFailureResponse` releases the probe lease. Tests: abort during + the first backoff → no second socket, exactly one A transient write; 503 then timeout and 503 + then connection error → exactly one A write. +5. **Compact verification matrix (audit rounds 3-4, accepted; all mandatory).** B served from + remaining base sends; B admitted from the final recovery reserve after A's three sends; B refused by + a spent budget (A's body and outcome intact, B's probe released, zero B sends); reservation granted + then `permit.use()` fails → reservation released; caller cancellation while B is in flight; unequal + A/B identity balances on both the base and the reserve path (B's identity governs admission), using + the ledger fixtures from `tests/lib/execution-budget-permits.test.ts`. + +Notes taken: the 10 s hold starts at the first lifecycle event and `fetchWithTransientRetry` +returns an attempt slower than 15 s as-is, so "three same-account sends" holds for refusals within +that budget (tests use an immediate refusal and one slow case). Combo children +(`options.comboAttempt`) do not opt in — their preflight already hops on a zero-output terminal. +401-refresh and rebuild legs do not opt in (documented limitation). Tests assert the restored +attempt's `sendCount`, `recoveryKinds` and `codexWsStage` counters. + +## File change map + +### MODIFY `src/server/responses/codex-ws-wire.ts` +- ADD `codexWsPreOutputRefusalStatus(payload)` (above). +- ADD `isEmptyCodexWsLifecycleEvent(payload)`: `response.created` / `response.in_progress` / + `response.queued` whose `response` is an object with `output` absent or `[]`. +- ADD restore registry: `markCodexWsPreOutputRefusal(projection, restore)`, + `isCodexWsPreOutputRefusal(response)`, `restoreCodexWsPreOutputRefusal(response)` (single use; + returns the response unchanged when not a projection). + +### MODIFY `src/server/responses/codex-ws-exchange.ts` +- ExchangeOptions: `preOutputRefusalHoldMs?: number`. +- `holdEnabled = hold > 0 && metadata && !nativeControl`. +- On a non-control frame while `sent && !responseCommitted`, after correlation: + - first frame is a refusal → project (no preamble). + - `isEmptyCodexWsLifecycleEvent` and holdEnabled → on the first held event freeze + `heldHeaders = metadata.snapshot()`, `metadata.commit()`, arm one non-resetting hold timer + (`commitResponse` on expiry); enqueue the frame; do not resolve. + - held preamble and the frame is a refusal → enqueue, close, `session.release(null)`-style + dispose, resolve the projection. + - anything else → `commitResponse()` (uses `heldHeaders` when present), then today's code. + - The existing pre-commit 4xx projection only runs when nothing was held (today's boundary). +- Projection: JSON `{error:{type,code,message}}` from the frame (bounded via existing + `rejectionHeaders`), the frame's status, `markCodexWsPreludeProjection`, stage record, NOT + non-replayable; restore builds the 200 SSE over the same closed stream with the markers + `commitResponse` sets and the same stage object. +- `failStream` / `cancelExchange`: when a preamble is held, `commitResponse()` first so the + settle is today's committed body failure (no socket-death marker, no new 502/504). +- Clear the hold timer in `cleanup` and `commitResponse`. Amend the "only precommit 4xx" comment. + +### MODIFY `src/server/responses/ws-upstream.ts`, `fetch-helpers.ts` +- `ProviderFetchOptions.preOutputRefusalHoldMs`; pass through `codexWsUpstreamFetch` (new trailing + parameter) to the exchange. + +### MODIFY `src/server/responses/passthrough-dispatch.ts` +- Initial send closure inside `fetchWithTransientRetry` (~1027): pass + `preOutputRefusalHoldMs` when `usesCodexForwardPoolAuth(admissionState.authCtx, route.provider)` + and the leg carries no native control. +- Immediately after the Pool block (~1704), every loop pass: + `upstreamResponse = restoreCodexWsPreOutputRefusal(upstreamResponse)`. + +### MODIFY `src/server/responses/core-codex-account.ts` +- Alternate-account providerFetch (~917): pass the same hold so a second refusal is also a + projection (restored by dispatch). Hold value threaded via `CodexPoolAccountRetryArgs`. +- Exactly-once marker as in audit disposition 2 (record + mark; skip when marked). +- Update the `shouldRetryCodexPoolAccountTransient` doc comment. + +### MODIFY `src/server/responses/compact.ts` +- Alternate condition gains a transient arm: `shouldRetryCodexPoolAccountTransient(upstream)` and + not body-confirmed quota (`shouldRetryCodexPoolAccountQuota`), same guards. Transient arm skips + the workspace-scope quota check and the quota-cache refresh, admits B's send before cancelling A, + records A with its real 5xx (transient) after B was sent and before every B return (success, HTTP failure, transport failure), keeps the conversation-state scrub. Spend/budget identity is B by construction: the label switches before the helper runs and both the base reporter and the reserve path read it lazily. + +### Tests (NEW, registered in `scripts/test-layout/layout.json` and +`tests/fixtures/test-layout-expected.json`) +- `tests/responses/ws-pre-output-refusal.test.ts` — exchange-level: first-frame 503/502 and + after-preamble refusals project with status, stage and restore; output/tool/non-empty snapshot, + socket death and silence before/after preamble, 4xx/429 frames, quota-coded 5xx, foreign + `stream_id`, conflicting aliases, native control, hold expiry, opt-out (no hold option) keep + today's results. +- `tests/responses/responses-pool-overload-failover.test.ts` — handleResponses pool end to end + with fake sockets: single account (3 same-account sends, original stream), two accounts all + overloaded (3 + 1 sends, original stream from B), recovery on B, fixed account (no move), output + started (1 send), abort during backoff (no further send), health recorded once. +- `tests/responses/compact-transient-alternate.test.ts` — compact A 503×3 → B success; B 503; + fixed account; quota-bodied 5xx keeps today's path. + +### Docs +- `structure/transports/responses-wire-shapes.md` (WS prelude contract) and + `structure/transports/responses-failover.md` (pool transient move, compact) — the exception and + its bounds. + +## Accept criteria with activation scenarios + +| Path | Activation | Observable proof | +| --- | --- | --- | +| Refusal after preamble | fake socket sends created, in_progress, `{type:error,status:503}` | dispatch sends again; second socket succeeds; client gets the success stream | +| Same-account bound | every socket refuses, one account | exactly 3 sockets; client body is the original created/in_progress/error stream | +| Account move | two accounts, all refuse | 3 + 1 sockets, B's frame carries B's token | +| Output started | created, output_item.added, error 503 | 1 socket; client sees the error in-stream | +| Socket death / silence | close after created; 90 s silence before any event | today's 200 body error / 504 non-replayable | +| Abort in backoff | abort during first backoff | no second socket | +| Fixed account | pinned Pool account | ≤3 sockets, never another account | +| Compact | A 503 thrice, B 200 | 4 HTTP sends, B serves, A recorded transient | + +## Verifiers (run in C) + +`bun run typecheck`; `bun test` on the three new files plus `tests/responses/ws-*.test.ts`, +`tests/responses/responses-pool-*.test.ts`, compact suites; `bun run test:changed`; +`bun run structure:check`; `bun run privacy:scan`; `bun test tests/test-layout*.test.ts`. + +## Residual risk + +HTTP SSE in-stream zero-output refusals and frames with an unrecognized code and no wrapper status +stay unrecovered. Native steering/injection exchanges keep today's behaviour. + +## Implementation review follow-ups + +- Review round 1 (gpt-6.1-sol high, FAIL): projection headers built before any settle, falling + back to committed delivery when they cannot be projected; `rate_limit*`/`*quota*` veto; dispatch + logic moved to `PoolPreOutputRefusals` in `core-codex-account.ts` for the 2,000-line ratchet; + real spend-ledger, queue-overflow, non-resetting-deadline and header-timeout tests. Round 2: PASS. +- PR review (Codex connector): the hold is now `{ maxMs, until }`, with `until` set per send to + half of `connectTimeoutMs` after it starts, so a late first lifecycle frame cannot hold past the + header deadline; compact keeps A whenever B never physically dispatched (tracked through + `onPhysicalDispatch`), not only on a spent budget. diff --git a/docs-site/src/content/docs/guides/codex-integration.md b/docs-site/src/content/docs/guides/codex-integration.md index f10ab318276..f418d94ce73 100644 --- a/docs-site/src/content/docs/guides/codex-integration.md +++ b/docs-site/src/content/docs/guides/codex-integration.md @@ -959,6 +959,16 @@ ocx service install # persistent: auto-starts on login and respawns on crash `ocx status` shows whether the proxy is running and prints the same restart hint when it is not; `ocx doctor` reports restart safety (service/shim coverage). +### Upstream overload before any output + +When the ChatGPT backend declines a pooled native Codex turn with `server_is_overloaded` or +another server error before it has produced any output, opencodex retries the same account +with a short, capped backoff and then moves the turn to one other eligible Pool account. +A pinned account (an account-qualified model) is retried but never moved. Once any output +has reached Codex the turn is never sent again. If every attempt is declined, Codex receives +the same error it would have received without the retries. Native compact requests move to +another Pool account on a server error in the same way. + ### Codex autostart shim Run `ocx codex-shim install` to install the optional launcher wrapper. It runs diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index fdef21ddfbd..244335dcb8c 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -1942,7 +1942,7 @@ "winsw-stop-hardening.test.ts": "windows", "winsw.test.ts": "service", "workflow-budget.test.ts": "lib", - "ws-ambiguous-resend.test.ts": "responses", + "ws-ambiguous-resend.test.ts": "responses", "ws-pre-output-refusal.test.ts": "responses", "responses-pool-overload-failover.test.ts": "responses", "compact-transient-alternate.test.ts": "responses", "ws-endpoint.test.ts": "responses", "ws-failure-stage.test.ts": "responses", "ws-native-injection.test.ts": "responses", diff --git a/src/server/responses/codex-ws-exchange.ts b/src/server/responses/codex-ws-exchange.ts index df1c98299b6..af37fdc91c4 100644 --- a/src/server/responses/codex-ws-exchange.ts +++ b/src/server/responses/codex-ws-exchange.ts @@ -12,7 +12,8 @@ import { UPGRADE_DEADLINE_MS, CODEX_WS_LIVENESS_PING_INTERVAL_MS, CODEX_WS_RESPO MAX_CODEX_WS_QUEUE_BYTES, markCodexWsResponse, markCodexWsPreludeProjection, normalizeResponsesWsRelayEvent, closedBeforeTerminalMessage, codexWsCreateFrameExceedsLimit, codexWsFailureDetail, codexWsPreResponseFailure, markCodexWsStage, codexWsOcxVersion, markCodexWsSocketDeath, codexWsWrapperStatus, isCodexWsTerminalEvent, type CodexWsFailureStage, type CodexWsStageRecord, - type CodexWsTerminalEvent } from "./codex-ws-wire"; + type CodexWsTerminalEvent, codexWsPreOutputRefusalStatus, isEmptyCodexWsLifecycleEvent, codexWsFrameResponseIds, + markCodexWsPreOutputRefusal, type CodexWsPreOutputRefusalHold } from "./codex-ws-wire"; interface ExchangeOptions { nativeControl?: NativeResponseControl; @@ -27,6 +28,13 @@ interface ExchangeOptions { onPhysicalDispatch?: () => void; /** Bun version string the caller gated on; stamped onto the stage record. */ bunVersion?: string; + /** + * Opt-in (#6969): how long empty lifecycle frames may be held so an authoritative upstream + * refusal that follows them, before any output, can settle as a replayable gateway status instead + * of a committed 200. Absent or zero keeps the exchange exactly as it was. Ignored on native + * control exchanges and on transports without the metadata channel. + */ + preOutputRefusalHold?: CodexWsPreOutputRefusalHold; } const HTTP_HEADER_TOKEN = /^[!#$%&'*+.^_`|~0-9a-z-]+$/i; @@ -76,6 +84,8 @@ function rejectionHeaders(source: Record, prelude: Headers): He * responses_websocket.rs accepts status/status_code and scalar header values; * unlike its native client, this relay converts only precommit 4xx. Returning a * post-send 5xx or fetch rejection could cause the outer retry wrapper to resend. + * The one 5xx exception is the opt-in pre-output refusal below (#6969): a structured upstream + * refusal with nothing relayed but empty lifecycle frames, which the caller may replay. */ function wrappedRejectionResponse(payload: Record, prelude: Headers): Response | null { if (payload.type !== "error" || payload.stream_id !== undefined) return null; @@ -93,9 +103,20 @@ function wrappedRejectionResponse(payload: Record, prelude: Hea }), { status, headers }); } +/** The client-facing error of a pre-output refusal projection, from the refusal's own error object. */ +function refusalErrorBody(source: unknown): Record { + const error = record(source) ? source : {}; + return { + type: typeof error.type === "string" ? error.type : "server_error", + ...(typeof error.code === "string" ? { code: error.code } : {}), + message: typeof error.message === "string" ? error.message : "Upstream declined the request before any output", + }; +} + /** The sole SSE exchange state machine for both one-shot and retained sockets. */ export function codexWsExchange(options: ExchangeOptions): Promise { - const { session, url, init, prepared, sseFallback, onQuota, beforeDispatch, bunVersion, nativeControl, beforeContinuation } = options; + const { session, url, init, prepared, sseFallback, onQuota, beforeDispatch, bunVersion, nativeControl, beforeContinuation, + preOutputRefusalHold } = options; const { frameText, headers } = prepared; const signal = init.signal ?? undefined; return new Promise((resolve, reject) => { @@ -139,6 +160,18 @@ export function codexWsExchange(options: ExchangeOptions): Promise { // The resolved 200, retained so a later body failure can replace its // success-shaped stage record with the failure-shaped one (#4191). let committedResponse: Response | null = null; + // #6969: the pre-output refusal hold. While it is open, empty lifecycle frames are queued on the + // not-yet-exposed stream without resolving it. Every settle other than an eligible refusal + // commits first (deferredCommit), so the outcome is exactly what the exchange produced before. + const holdEnabled = (preOutputRefusalHold?.maxMs ?? 0) > 0 && metadata !== null && !nativeControl; + // Drops for good on any identity the exchange cannot tie to one response: a stream id, two + // different ids in one frame, or an id that differs from the first one seen. Independent of + // socket reuse, so a cold one-shot socket is held to the same rule as a pooled one. + let refusalEligible = holdEnabled; + let refusalResponseId: string | null = null; + let heldHeaders: Headers | null = null; + let deferredCommit = false; + let holdTimer: ReturnType | undefined; const stream = new ReadableStream({ start(c) { controller = c; }, cancel() { @@ -153,6 +186,7 @@ export function codexWsExchange(options: ExchangeOptions): Promise { clearTimeout(upgradeTimer); clearTimeout(silenceTimer); clearTimeout(pingTimer); + clearTimeout(holdTimer); signal?.removeEventListener("abort", onAbort); metadata?.finish(); correlation?.finish(); @@ -211,9 +245,13 @@ export function codexWsExchange(options: ExchangeOptions): Promise { const commitResponse = () => { if (responseCommitted) return; responseCommitted = true; + deferredCommit = false; + clearTimeout(holdTimer); clearTimeout(silenceTimer); clearTimeout(pingTimer); - const responseHeaders = metadata?.snapshot() ?? new Headers(); + // A held preamble froze the prelude headers at the first Responses event, which is the moment + // they were frozen before the hold existed. + const responseHeaders = heldHeaders ? new Headers(heldHeaders) : metadata?.snapshot() ?? new Headers(); responseHeaders.set("content-type", "text/event-stream; charset=utf-8"); const response = new Response(stream, { status: 200, headers: responseHeaders }); metadata?.commit(); @@ -224,9 +262,73 @@ export function codexWsExchange(options: ExchangeOptions): Promise { resolve(response); }; + /** + * Open the hold at the first empty lifecycle frame: freeze the prelude and bound the wait by + * both the hold's own maximum and its absolute end. False when no time is left, so the frame + * commits exactly as it would without the hold. + */ + const beginHold = (): boolean => { + if (heldHeaders) return true; + if (!metadata || !preOutputRefusalHold) return false; + const waitMs = Math.min(preOutputRefusalHold.maxMs, preOutputRefusalHold.until - Date.now()); + if (waitMs <= 0) return false; + heldHeaders = metadata.snapshot(); + metadata.commit(); + holdTimer = setTimeout(() => { + if (!terminal && !responseCommitted) commitResponse(); + }, waitMs); + return true; + }; + + const noteRefusalIdentity = (payload: Record) => { + const ids = codexWsFrameResponseIds(payload); + if (payload.stream_id !== undefined || ids === null || ids.length > 1) { + refusalEligible = false; + return; + } + const id = ids[0]; + if (id === undefined) return; + if (refusalResponseId === null) refusalResponseId = id; + else if (id !== refusalResponseId) refusalEligible = false; + }; + + /** + * Settle an authoritative pre-output refusal (#6969) as the gateway status the upstream stated, + * without the non-replayable marker: nothing but empty lifecycle frames was relayed, and the + * upstream ended that response itself. The caller's transient ladder and pool move may send the + * turn again; if neither does, the registered restore hands the client the original stream, + * which is already complete on `stream`. + */ + const resolveRefusal = (status: number, payload: Record, headers: Headers, prelude: Headers) => { + responseCommitted = true; + deferredCommit = false; + clearTimeout(holdTimer); + const stage = stageRecord(null); + const source = payload.type === "error" + ? payload.error + : record(payload.response) ? payload.response.error : undefined; + const projection = new Response(JSON.stringify({ error: refusalErrorBody(source) }), { status, headers }); + markCodexWsPreludeProjection(projection); + markCodexWsStage(projection, stage); + const observed = Boolean(metadata && onQuota); + markCodexWsPreOutputRefusal(projection, () => { + const restoredHeaders = new Headers(prelude); + restoredHeaders.set("content-type", "text/event-stream; charset=utf-8"); + const restored = new Response(stream, { status: 200, headers: restoredHeaders }); + markCodexWsResponse(restored, observed); + // The same record object, so the attempt that adopted it keeps reading one stage. + markCodexWsStage(restored, stage); + return restored; + }); + resolve(projection); + }; + const failStream = (error: unknown, status: 502 | 504 = 502, { socketDied = false } = {}) => { if (terminal) return; terminal = true; + // A held preamble or refusal was a commit deferred, not avoided: settle as the committed + // body failure it would have been, never as a new pre-response gateway status. + if (deferredCommit) commitResponse(); if (sent && !responseCommitted && metadata) { // Nothing has been promised to the client yet, so the honest answer is a gateway // status, not a 200 whose body then fails. The frame may already be executing @@ -311,7 +413,7 @@ export function codexWsExchange(options: ExchangeOptions): Promise { reject(reason); return; } - if (!responseCommitted && metadata) { + if (!responseCommitted && metadata && !deferredCommit) { // Sent, unacknowledged. The proxy's own connect deadline is an origin-silence // verdict and settles like one; a caller abort is the caller's decision, so the // exchange rejects with that reason and disposing the socket cancels the turn. @@ -503,16 +605,23 @@ export function codexWsExchange(options: ExchangeOptions): Promise { } if (!controlFrame && !type.startsWith("response.") && type !== "error") return; let steeringEnded = false; + let refusalStatus: number | undefined; + let refusalHeaders: Headers | undefined; + let refusalPrelude: Headers | undefined; if (!controlFrame) { firstResponseAt ??= Date.now(); try { if (nativeControl) steeringEnded = nativeControl.observe(normalized.payload); else correlation?.accept(normalized.payload); } catch (error) { failStream(error); return; } + const holdOpen = refusalEligible && sent && !responseCommitted; + if (holdOpen) noteRefusalIdentity(normalized.payload); // Correlation must run first: a reused socket's foreign-stream error settles as a // non-replayable 502 above, never as the refused-create 4xx projection below, which // is the one status family that could authorize an account replay. - if (metadata && sent && !responseCommitted && type === "error") { + // Only before anything was held: after an empty preamble a 4xx frame keeps today's + // in-body delivery, so a held 429 can never become a quota rotation it was not before. + if (metadata && sent && !responseCommitted && !deferredCommit && type === "error") { let rejection: Response | null; try { rejection = wrappedRejectionResponse(normalized.payload, metadata.snapshot()); } catch (error) { failStream(error); return; } @@ -526,7 +635,24 @@ export function codexWsExchange(options: ExchangeOptions): Promise { return; } } - commitResponse(); + let holdFrame = false; + if (holdOpen && refusalEligible) { + refusalStatus = codexWsPreOutputRefusalStatus(normalized.payload); + if (refusalStatus !== undefined) { + // Built before anything settles: a refusal whose metadata cannot be projected within the + // header bounds is delivered the way it was before, in the committed body. + try { + refusalPrelude = heldHeaders ?? metadata?.snapshot() ?? new Headers(); + const declared = normalized.payload.headers; + refusalHeaders = rejectionHeaders(record(declared) ? declared : {}, refusalPrelude); + } catch { + refusalStatus = undefined; + } + } + holdFrame = refusalStatus === undefined && isEmptyCodexWsLifecycleEvent(normalized.payload) && beginHold(); + } + if (refusalStatus === undefined && !holdFrame) commitResponse(); + else deferredCommit = true; // Record the terminal the upstream sent as soon as it is accepted, before the relay-size and // queue checks below: a body failure that follows is still an exchange that received it. if (!nativeControl && isCodexWsTerminalEvent(type)) { @@ -570,6 +696,9 @@ export function codexWsExchange(options: ExchangeOptions): Promise { markCodexWsStage(committedResponse, stageRecord(null)); } session.release(completedId); + if (refusalStatus !== undefined && refusalHeaders && refusalPrelude && !responseCommitted) { + resolveRefusal(refusalStatus, normalized.payload, refusalHeaders, refusalPrelude); + } } }; diff --git a/src/server/responses/codex-ws-wire.ts b/src/server/responses/codex-ws-wire.ts index 71a45be5ed3..59fe27f2699 100644 --- a/src/server/responses/codex-ws-wire.ts +++ b/src/server/responses/codex-ws-wire.ts @@ -1,9 +1,11 @@ import { MAX_CLIENT_SSE_FRAME_BYTES } from "../sse-frame-buffer"; import { + isTransientUpstreamStatus, markResponseNonReplayable, UPSTREAM_CLOSED_BEFORE_RESPONSE_CODE, UPSTREAM_NO_RESPONSE_CODE, } from "../../lib/upstream-retry"; +import { isRateLimitOrQuotaFailureMessage, recognizedUpstreamError } from "../../lib/errors"; import type { RequestFailureCause, RequestFailureStage } from "../../lib/request-failure-model"; import { packageVersion } from "../../lib/package-version"; // If the 101 never arrives (network black hole), give SSE a chance well before @@ -151,6 +153,165 @@ export function codexWsWrapperStatus(payload: Record): number | : undefined; } +function plainRecord(value: unknown): value is Record { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +/** An absent or empty `output` array; anything else may already be visible to the client. */ +function emptyOutput(response: Record): boolean { + const output = response.output; + return output === undefined || (Array.isArray(output) && output.length === 0); +} + +/** + * Upper bound on how long an exchange may hold empty lifecycle frames while it waits to see + * whether the upstream refuses the turn before any output (#6969). + * + * Field rows put 97.5% of authoritative refusals within 5 s of the first Responses event and + * 98.2% within 10 s, with nothing gained after that. + */ +export const CODEX_WS_PRE_OUTPUT_REFUSAL_HOLD_MAX_MS = 10_000; + +/** + * The hold one send may use: at most `maxMs` from the first empty lifecycle frame, and never past + * `until`, an absolute time. + */ +export interface CodexWsPreOutputRefusalHold { + maxMs: number; + until: number; +} + +/** + * The hold for a send starting now. The caller's header deadline (`connectTimeoutMs`) starts no + * earlier than this, so ending every hold by half of it means a held turn always commits well before + * that deadline could abort it, however late the first lifecycle frame arrives. + */ +export function codexWsPreOutputRefusalHold(connectMs: number, now = Date.now()): CodexWsPreOutputRefusalHold { + const half = Math.max(0, Math.floor(connectMs / 2)); + return { maxMs: Math.min(CODEX_WS_PRE_OUTPUT_REFUSAL_HOLD_MAX_MS, half), until: now + half }; +} + +/** + * A lifecycle frame that commits nothing the client could see or act on (#6969): created, + * in-progress or queued, carrying a response object whose output is absent or empty. A snapshot + * with any output item, an unknown event, or a frame addressed to another stream is not one. + */ +export function isEmptyCodexWsLifecycleEvent(payload: Record): boolean { + if (payload.type !== "response.created" && payload.type !== "response.in_progress" + && payload.type !== "response.queued") return false; + if (payload.stream_id !== undefined) return false; + return plainRecord(payload.response) && emptyOutput(payload.response); +} + +const QUOTA_REFUSAL_IDENTITIES: ReadonlySet = new Set(["usage_limit_reached", "usage_not_included"]); + +/** Quota and rate-limit identities keep their own routing: every `rate_limit*` and `*quota*` name. */ +function quotaRefusalIdentity(value: string): boolean { + const name = value.toLowerCase(); + return QUOTA_REFUSAL_IDENTITIES.has(name) || name.startsWith("rate_limit") || name.includes("quota"); +} + +/** + * The status of an authoritative upstream refusal that ended a turn before any output (#6969), + * or undefined. + * + * Only structured evidence counts: a bare `error` frame's single wrapper status (the alias the + * native client turns into an HTTP error of that status, see {@link codexWsWrapperStatus}), or, + * with no wrapper alias at all, the recognized overload class; and a `response.failed` with an + * empty output whose error is the recognized overload class. The status must be a transient + * gateway status. Message wording never authorizes anything here; it can only veto, so a 5xx whose + * error names quota or a rate limit keeps the routing it has today. + */ +export function codexWsPreOutputRefusalStatus(payload: Record): number | undefined { + if (payload.stream_id !== undefined) return undefined; + let error: unknown; + let status: number | undefined; + if (payload.type === "error") { + error = payload.error; + if (error !== undefined && !plainRecord(error)) return undefined; + const statedAlias = Object.hasOwn(payload, "status") || Object.hasOwn(payload, "status_code"); + status = codexWsWrapperStatus(payload); + // A malformed or doubled alias is not evidence of anything. + if (statedAlias && status === undefined) return undefined; + if (status === undefined && plainRecord(error)) status = recognizedUpstreamError(error)?.httpStatus; + } else if (payload.type === "response.failed") { + const response = payload.response; + if (!plainRecord(response) || !emptyOutput(response)) return undefined; + error = response.error; + if (!plainRecord(error)) return undefined; + status = recognizedUpstreamError(error)?.httpStatus; + } else { + return undefined; + } + if (status === undefined || !isTransientUpstreamStatus(status)) return undefined; + if (plainRecord(error)) { + const { code, type, message } = error; + if ([code, type, message].some(value => value != null && typeof value !== "string")) return undefined; + if (typeof code === "string" && quotaRefusalIdentity(code)) return undefined; + if (typeof type === "string" && quotaRefusalIdentity(type)) return undefined; + if (typeof message === "string" && isRateLimitOrQuotaFailureMessage(message)) return undefined; + } + return status; +} + +/** + * The response ids a frame names, for the exchange's replay-eligibility latch (#6969): the nested + * `response.id` and the top-level `response_id`. Null when a present id is not a string, which the + * latch treats as a conflict. + */ +export function codexWsFrameResponseIds(payload: Record): string[] | null { + const ids: string[] = []; + const nested = plainRecord(payload.response) ? payload.response.id : undefined; + for (const value of [nested, payload.response_id]) { + if (value === undefined) continue; + if (typeof value !== "string") return null; + if (!ids.includes(value)) ids.push(value); + } + return ids; +} + +interface PreOutputRefusalRecord { + restore: (() => Response) | null; + outcomeRecorded: boolean; +} + +const preOutputRefusals = new WeakMap(); + +/** + * Register a pre-output refusal projection (#6969): a replayable gateway status standing in for a + * WebSocket exchange whose upstream refused the turn before any output. `restore` rebuilds the + * 200 event stream the client would have received without the projection. + */ +export function markCodexWsPreOutputRefusal(projection: Response, restore: () => Response): void { + preOutputRefusals.set(projection, { restore, outcomeRecorded: false }); +} + +export function isCodexWsPreOutputRefusal(response: Response): boolean { + return preOutputRefusals.has(response); +} + +/** Record that this projection's account outcome was written, so no later owner writes it again. */ +export function noteCodexWsPreOutputRefusalOutcome(response: Response): void { + const record = preOutputRefusals.get(response); + if (record) record.outcomeRecorded = true; +} + +export function codexWsPreOutputRefusalOutcomeRecorded(response: Response): boolean { + return preOutputRefusals.get(response)?.outcomeRecorded === true; +} + +/** + * The response the client should receive in place of a projection no recovery replaced: the + * original event stream, byte for byte. Single use; any other response is returned unchanged. + */ +export function restoreCodexWsPreOutputRefusal(response: Response): Response { + const record = preOutputRefusals.get(response); + const restore = record?.restore; + if (!record || !restore) return response; + record.restore = null; + return restore(); +} + const codexWsStageByResponse = new WeakMap(); export function markCodexWsStage(response: Response, record: CodexWsStageRecord): void { diff --git a/src/server/responses/compact.ts b/src/server/responses/compact.ts index 5215b5a2796..c3c74102035 100644 --- a/src/server/responses/compact.ts +++ b/src/server/responses/compact.ts @@ -200,6 +200,7 @@ import { usesCodexForwardPoolAuth, } from "./core"; import { fetchWithHeaderTimeout, providerFetch, safeHostLabel, safeOriginLabel } from "./fetch-helpers"; +import { shouldRetryCodexPoolAccountQuota, shouldRetryCodexPoolAccountTransient } from "./core-codex-account"; import { mapCodexAuthContextErrorToResponse, nativeMainRefreshFailureResponse } from "./codex-auth-error"; import { linkRequestSessionLane, sessionLaneIdFromRequest } from "../request-log-conversation"; import { recallComboForLane } from "./combo-session-recall"; @@ -1048,7 +1049,11 @@ export async function handleResponsesCompact( targetKey: compactTargetKey, }); if (!decision.allowed) return Promise.reject(new SendBudgetExhaustedError(safeHostLabel(compactUrl))); - if (!decision.permit.use()) return Promise.reject(new SendBudgetExhaustedError(safeHostLabel(compactUrl))); + if (!decision.permit.use()) { + // Nothing was sent, so the reservation goes back rather than holding a send against the request. + decision.permit.release(); + return Promise.reject(new SendBudgetExhaustedError(safeHostLabel(compactUrl))); + } return doFetch(); }; const sendCompactAttempt = ( @@ -1056,6 +1061,7 @@ export async function handleResponsesCompact( sendHeaders: Headers, recovery: "normal" | "single", sendAuthCtx: CodexAuthContext, + onPhysicalDispatch?: () => void, ): Promise => { const doFetch = (upstreamRecovery?: UpstreamSendRecovery) => fetchWithHeaderTimeout( compactUrl, @@ -1072,6 +1078,7 @@ export async function handleResponsesCompact( modelId: route.modelId, beforeDispatch: isCanonicalOpenAiForwardProvider(sendProvider) ? createCodexAuthDispatchGuard(sendAuthCtx, config, selectedModelId, admission) : undefined, + ...(onPhysicalDispatch ? { onPhysicalDispatch } : {}), }), // Every credential-bearing forward send gets manual redirects, not only // pool sends: direct mode carries the caller's credential too (#914). @@ -1217,6 +1224,53 @@ export async function handleResponsesCompact( } } + // The settle for an alternate send that threw, shared by the quota and transient moves. By the + // time it runs the request already names the alternate (outcomeCtx), so health goes to it. + const failedAlternateCompactSend = (err: unknown): Response => { + if (req.signal.aborted) { + recordCompactPoolOutcome(outcomeCtx, 499); + return formatErrorResponse(499, "client_cancelled", "Client cancelled compact request"); + } + const localRefusal = localDispatchRefusal(err); + if (localRefusal) return localRefusal; + const outcome = classifyTransportFailureKind(err); + // Host-level evidence stands regardless of pool membership (#914 review). + if (outcome === "connect_neutral") { + if (compactHostCircuitEnabled) { + recordUpstreamHostFailure(actualCompactHostKey, { + code: transportErrorCode(err), + threshold: config.upstreamHostCircuitThreshold, + lease: compactHostAdmissionLease, + }); + } else { + recordUpstreamHostFailure(actualCompactHostKey, { code: transportErrorCode(err) }); + } + } else { + releaseUpstreamHostAdmission(compactHostAdmissionLease); + } + compactHostAdmissionLease = null; + recordCompactPoolOutcome(outcomeCtx, outcome); + return formatErrorResponse(502, "upstream_error", "Failed to connect to compact upstream"); + }; + const scrubCompactStateForAlternate = (alternateAuthCtx: CodexAuthContext): void => { + const binding = conversationStateBindingFromAuth( + alternateAuthCtx, + (authCtx.kind === "pool" || authCtx.kind === "main-pool") + ? authCtx.affinityKey + : codexPoolAffinityKey(req.headers), + ); + if (!binding) return; + for (const body of [raw, compactBody]) { + applyAccountChangeConversationStateScrub({ + body, + bindingKey: binding.bindingKey, + servingAccountId: binding.accountId, + priorAccountId: authCtx.accountId, + logCtx, + }); + } + }; + // Bounded same-request alternate: the regular /v1/responses path already does this // (core.ts:319-423) and recognizes exactly 429/402. Without it a pool rejection // surfaces to the client, which retries the compact task OUTSIDE the logical request @@ -1317,57 +1371,81 @@ export async function handleResponsesCompact( await upstream.body?.cancel().catch(() => undefined); outcomeCtx = alternate.authCtx; logCtx.accountLogLabel = codexAuthContextLogLabel(alternate.authCtx, config); - { - const binding = conversationStateBindingFromAuth( - alternate.authCtx, - (authCtx.kind === "pool" || authCtx.kind === "main-pool") - ? authCtx.affinityKey - : codexPoolAffinityKey(req.headers), - ); - if (binding) { - applyAccountChangeConversationStateScrub({ - body: raw, - bindingKey: binding.bindingKey, - servingAccountId: binding.accountId, - priorAccountId: authCtx.accountId, - logCtx, - }); - applyAccountChangeConversationStateScrub({ - body: compactBody, - bindingKey: binding.bindingKey, - servingAccountId: binding.accountId, - priorAccountId: authCtx.accountId, - logCtx, - }); - } - } + scrubCompactStateForAlternate(alternate.authCtx); try { upstream = await sendCompactAttempt(alternate.provider, alternate.headers, "single", alternate.authCtx); } catch (err) { - if (req.signal.aborted) { - recordCompactPoolOutcome(outcomeCtx, 499); - return formatErrorResponse(499, "client_cancelled", "Client cancelled compact request"); - } - const localRefusal = localDispatchRefusal(err); - if (localRefusal) return localRefusal; - const outcome = classifyTransportFailureKind(err); - // Host-level evidence stands regardless of pool membership (#914 review). - if (outcome === "connect_neutral") { - if (compactHostCircuitEnabled) { - recordUpstreamHostFailure(actualCompactHostKey, { - code: transportErrorCode(err), - threshold: config.upstreamHostCircuitThreshold, - lease: compactHostAdmissionLease, - }); - } else { - recordUpstreamHostFailure(actualCompactHostKey, { code: transportErrorCode(err) }); - } - } else { - releaseUpstreamHostAdmission(compactHostAdmissionLease); + return failedAlternateCompactSend(err); + } + } + } + // A transient 5xx that survived the same-account ladder above (#6969). The regular path moves + // it to another Pool account; compact used to move only on 429/402, so the client retried the + // compact task outside this request while another account sat idle. One bounded alternate send, + // the same guards as the quota move, and a 5xx whose body confirms quota keeps today's path. + if ( + shouldRetryCodexPoolAccountTransient(upstream) + && !storedPool401ReplayAttempted + && usesCodexForwardPoolAuth(authCtx, route.provider) + && !authCtx.fixedAccount + && route.codexAccountMode + && outcomeCtx === authCtx + && !req.signal.aborted + && !conversationCarriesUploadedFiles(raw) + && !await shouldRetryCodexPoolAccountQuota(upstream, req.signal) + ) { + const firstStatus = upstream.status; + const firstRetryAfter = upstream.headers.get("retry-after"); + const firstResetAt = [ + upstream.headers.get("x-codex-primary-reset-at"), + upstream.headers.get("x-codex-secondary-reset-at"), + upstream.headers.get("x-codex-tertiary-reset-at"), + ].filter(Boolean); + const alternate = await resolveAlternateCompactContext({ + req, + admission, + config, + route, + selectedModelId, + excludeAccountId: authCtx.accountId, + turnAdmissionLease, + }); + if (alternate && req.signal.aborted) { + releaseCodexAuthContextProbeLease(alternate.authCtx); + recordCompactPoolOutcome(outcomeCtx, 499); + void upstream.body?.cancel(req.signal.reason).catch(() => undefined); + return formatErrorResponse(499, "client_cancelled", "Client cancelled compact request"); + } + if (alternate) { + const firstLabel = logCtx.accountLogLabel; + // B's identity before B's admission, so every budget and spend check runs against B. A's + // response stays intact until B is actually sent. + logCtx.accountLogLabel = codexAuthContextLogLabel(alternate.authCtx, config); + scrubCompactStateForAlternate(alternate.authCtx); + const recordFirstMove = () => { + recordCompactPoolOutcome(authCtx, firstStatus, { retryAfter: firstRetryAfter, resetAt: firstResetAt }); + void upstream.body?.cancel().catch(() => undefined); + outcomeCtx = alternate.authCtx; + }; + let alternateResponse: Response | undefined; + // Whether B's send actually left this process. Every local refusal before that point -- a + // spent budget, a dispatch guard such as disabled credits, a pacing refusal -- keeps A. + let alternateDispatched = false; + try { + alternateResponse = await sendCompactAttempt(alternate.provider, alternate.headers, "single", alternate.authCtx, + () => { alternateDispatched = true; }); + } catch (err) { + if (alternateDispatched) { + recordFirstMove(); + return failedAlternateCompactSend(err); } - compactHostAdmissionLease = null; - recordCompactPoolOutcome(outcomeCtx, outcome); - return formatErrorResponse(502, "upstream_error", "Failed to connect to compact upstream"); + // Refused before anything left: A's answer and identity stand, recorded once below. + logCtx.accountLogLabel = firstLabel; + releaseCodexAuthContextProbeLease(alternate.authCtx); + } + if (alternateResponse) { + recordFirstMove(); + upstream = alternateResponse; } } } diff --git a/src/server/responses/core-codex-account.ts b/src/server/responses/core-codex-account.ts index 8fa6dafb1ed..17071ba24d2 100644 --- a/src/server/responses/core-codex-account.ts +++ b/src/server/responses/core-codex-account.ts @@ -22,7 +22,15 @@ import { classifyPoolRecoveryDispatch } from "../../routing/probe-lease"; import { formatErrorResponse } from "../../bridge"; import { readBoundedResponseBody } from "../../lib/bounded-body"; import { upstreamErrorMessageFromPayload, isRateLimitOrQuotaFailureMessage } from "../../lib/errors"; -import { isNonReplayableResponse, isTransientUpstreamStatus } from "../../lib/upstream-retry"; +import { isNonReplayableResponse, isTransientUpstreamStatus, SendBudgetExhaustedError } from "../../lib/upstream-retry"; +import { + codexWsPreOutputRefusalHold, + codexWsPreOutputRefusalOutcomeRecorded, + isCodexWsPreOutputRefusal, + noteCodexWsPreOutputRefusalOutcome, + restoreCodexWsPreOutputRefusal, + type CodexWsPreOutputRefusalHold, +} from "./codex-ws-wire"; import type { RequestLogContext } from "../request-log"; import type { DataPlaneAdmission } from "../auth-cors"; import type { InboundWire } from "../../providers/registry"; @@ -121,6 +129,90 @@ export function usesCodexForwardPoolAuth( && provider.authMode === "forward" && provider.adapter === "openai-responses"; } +/** + * Write a pre-output refusal projection's account outcome once (#6969), for an exit that no other + * owner records: a caller abort or a local send-budget refusal during the same-account backoff, + * where the transport-failure path writes no account health and the restored stream never reaches + * delivery. A projection already recorded by the alternate-account helper is left alone. + */ +export function recordCodexPreOutputRefusalOutcome( + config: OcxConfig, + authCtx: CodexAuthContext, + provider: OcxProviderConfig, + response: Response, + modelId: string, +): void { + if (!isCodexWsPreOutputRefusal(response) || codexWsPreOutputRefusalOutcomeRecorded(response)) return; + if (!usesCodexForwardPoolAuth(authCtx, provider)) return; + noteCodexWsPreOutputRefusalOutcome(response); + recordCodexUpstreamOutcome(config, authCtx.accountId, response.status, { + threadId: authCtx.affinityKey, + fixedAccount: authCtx.fixedAccount, + modelId, + probeLeaseId: codexProbeLeaseId(authCtx), + probeQuotaScope: codexProbeQuotaScope(authCtx), + transientProbe: codexTransientProbeGrant(authCtx), + writerGeneration: authCtx.writerGeneration, + }); +} + +/** + * The pre-output refusal hold for one Pool send starting now (#6969), or undefined. Pool forward + * sends only, and never a combo child, whose own stream preflight already advances on a zero-output + * terminal. Every projection a send produces must later pass through + * {@link PoolPreOutputRefusals.restore}. + */ +export function poolPreOutputRefusalHold( + authCtx: CodexAuthContext, + provider: OcxProviderConfig, + connectMs: number, + comboAttempt: boolean | undefined, +): CodexWsPreOutputRefusalHold | undefined { + return !comboAttempt && usesCodexForwardPoolAuth(authCtx, provider) ? codexWsPreOutputRefusalHold(connectMs) : undefined; +} + +/** + * One dispatch's view of its pre-output refusal projections (#6969): the last one the initial + * transient ladder observed, the single exit where the dispatch itself must record it, and the + * restore that hands the client the original stream. + */ +export class PoolPreOutputRefusals { + private last: Response | undefined; + /** True when the response handed to delivery is a restored projection already recorded. */ + outcomeRecorded = false; + + readonly observe = (response: Response): Response => { + if (isCodexWsPreOutputRefusal(response)) this.last = response; + return response; + }; + + /** + * One owner per exceptional exit of the initial ladder. A caller abort or a local budget refusal + * during the backoff leaves the transport-failure path writing no account health, so the refusal + * already observed is recorded here; any other exception keeps that path as its single writer. + */ + recordOnLocalExit( + error: unknown, + aborted: boolean, + config: OcxConfig, + authCtx: CodexAuthContext, + provider: OcxProviderConfig, + modelId: string, + ): void { + if (!this.last || !(aborted || error instanceof SendBudgetExhaustedError)) return; + recordCodexPreOutputRefusalOutcome(config, authCtx, provider, this.last, modelId); + } + + /** + * Called right after the Pool block on every recovery pass: a projection that survived the ladder + * and the account move becomes the original event stream before any later reader sees it. + */ + restore(response: Response): Response { + this.outcomeRecorded = isCodexWsPreOutputRefusal(response) && codexWsPreOutputRefusalOutcomeRecorded(response); + return restoreCodexWsPreOutputRefusal(response); + } +} + /** Live proof that this plain-main response used the observed main credential. */ export function liveMainQuotaDispatch( @@ -368,6 +460,10 @@ export async function shouldRetryCodexScopedQuotaOnAlternate( * could duplicate a turn the origin may still be running. A 5xx whose body confirms quota is not * routed here either — {@link shouldRetryCodexPoolAccountQuota} classifies that one first and * carries the cooldown with it. + * + * A WebSocket refusal reaches this predicate only as the opt-in pre-output projection (#6969): the + * upstream itself ended the turn with a structured 5xx while nothing but empty lifecycle frames had + * been relayed. It carries no non-replayable marker because nothing could have been observed. */ export function shouldRetryCodexPoolAccountTransient(response: Response): boolean { return !isNonReplayableResponse(response) && isTransientUpstreamStatus(response.status); @@ -420,6 +516,8 @@ export interface CodexPoolAccountRetryArgs { stream: boolean; /** Keep a buffered canonical client on HTTP/SSE after moving to another Pool account. */ httpOnly?: boolean; + /** Pre-output refusal hold for the alternate's WebSocket send (#6969); the caller restores. */ + preOutputRefusalHold?: CodexWsPreOutputRefusalHold; onResponse?: ( response: Response, authCtx: CodexAuthContext, @@ -571,6 +669,10 @@ export async function retryCodexPoolOnAlternateAccount( // keep handing it the next request. const recordUnmovedTransientOutcome = (): void => { if (!isTransientUpstreamStatus(outcomeStatus)) return; + // #6969: one write per projection. After recordFirstOutcome, a late window or permit denial + // must not record the same refusal again, and a recorded projection tells delivery to skip it. + if (codexWsPreOutputRefusalOutcomeRecorded(firstResponse)) return; + noteCodexWsPreOutputRefusalOutcome(firstResponse); recordCodexUpstreamOutcome(config, firstAuthCtx.accountId, outcomeStatus, { threadId: firstAuthCtx.affinityKey, fixedAccount: firstAuthCtx.fixedAccount, @@ -763,6 +865,7 @@ export async function retryCodexPoolOnAlternateAccount( // Retry already advanced the RR ring via excludeAccountId — reuse for promotion. ...(retryAuthCtx.accountId ? { promoteAccountId: retryAuthCtx.accountId } : {}), }); + noteCodexWsPreOutputRefusalOutcome(firstResponse); }; // Only a combo reset-derived outcome is deferred. Retry-After, defaults, and // ordinary requests must block the first account before the alternate send. @@ -918,6 +1021,7 @@ export async function retryCodexPoolOnAlternateAccount( httpOnly, providerName: route.providerName, modelId: route.modelId, + preOutputRefusalHold: args.preOutputRefusalHold, onCodexWsQuota: codexWsQuotaObserver(retryAuthCtx, route.provider, route.modelId), beforeDispatch: isCanonicalOpenAiForwardProvider(route.provider) ? createCodexAuthDispatchGuard(retryAuthCtx, options.codexAuthPolicy ?? config, route.modelId, options.admission, options.visionDescribeTerminal === true) : undefined, diff --git a/src/server/responses/fetch-helpers.ts b/src/server/responses/fetch-helpers.ts index fa1518169a5..1781ea19694 100644 --- a/src/server/responses/fetch-helpers.ts +++ b/src/server/responses/fetch-helpers.ts @@ -1,4 +1,5 @@ import type { NativeResponseControl } from "./native-response-control"; +import type { CodexWsPreOutputRefusalHold } from "./codex-ws-wire"; import type { Server } from "bun"; import { codexWsUpstreamFetch, @@ -244,6 +245,12 @@ export interface ProviderFetchOptions { beforeDispatch?: (headers: Headers) => void; /** One-shot receipt after final local HTTP admission, immediately before executor entry. */ onPhysicalDispatch?: () => void; + /** + * Opt-in (#6969): let a Codex WebSocket send hold empty lifecycle frames so an authoritative + * upstream refusal before any output settles as a replayable gateway status. Only a caller that + * restores the projection afterwards (restoreCodexWsPreOutputRefusal) may set it. + */ + preOutputRefusalHold?: CodexWsPreOutputRefusalHold; /** Revalidate/rebuild a queued request at its physical send boundary, after pacing. */ dispatchOverride?: (input: Parameters[0], init: RequestInit, execute: typeof globalThis.fetch) => Promise; } @@ -360,7 +367,7 @@ export function providerFetch( const fallback = ((url: Parameters[0], fallbackInit?: RequestInit) => httpFetch(url, { ...fallbackInit, [WS_FALLBACK_RECEIPT]: notifyOnce } as RequestInit)) as typeof globalThis.fetch; return codexWsUpstreamFetch(input, init, fallback, runtime, options.onCodexWsQuota, options.beforeDispatch, options.nativeControl, - async () => { (await waitForPacing(init.signal ?? undefined))?.release(); }, notifyOnce); + async () => { (await waitForPacing(init.signal ?? undefined))?.release(); }, notifyOnce, options.preOutputRefusalHold); } return httpFetch(input, init); }; diff --git a/src/server/responses/passthrough-delivery.ts b/src/server/responses/passthrough-delivery.ts index 184e2b1eabb..518b32fb1b4 100644 --- a/src/server/responses/passthrough-delivery.ts +++ b/src/server/responses/passthrough-delivery.ts @@ -408,6 +408,7 @@ export async function deliverPassthroughResponse( | "noteInspectedPayload" | "normalizeFunctionCompletionJson" | "localUpstream" + | "terminalOutcomeRecorded" >, ): Promise { const { route } = requestState; @@ -486,7 +487,9 @@ export async function deliverPassthroughResponse( ? (status: ResponsesTerminalStatus, httpStatusOverride?: number): void => { if (terminalOutcomeRecorded) return; terminalOutcomeRecorded = true; - recordTerminalOutcome(status, httpStatusOverride); + // A restored pre-output refusal whose account outcome dispatch already wrote (#6969) keeps + // every other terminal callback but must not write the same refusal a second time. + if (!nativeExchange.terminalOutcomeRecorded) recordTerminalOutcome(status, httpStatusOverride); } : undefined; const terminalBodyWillRecord = !!terminalRecorder && upstreamResponse.ok && isEventStream; diff --git a/src/server/responses/passthrough-dispatch.ts b/src/server/responses/passthrough-dispatch.ts index c5cd64a51fd..33d80678572 100644 --- a/src/server/responses/passthrough-dispatch.ts +++ b/src/server/responses/passthrough-dispatch.ts @@ -101,6 +101,8 @@ import { shouldRetryCodexPoolAccountQuota, shouldRetryCodexPoolAccountTransient, retryCodexPoolOnAlternateAccount, + PoolPreOutputRefusals, + poolPreOutputRefusalHold, } from "./core-codex-account"; import { clearCodexModelDenialEvidence, @@ -1004,6 +1006,9 @@ export async function preparePassthroughExchange( // Stable per request: the Combo booking (`reserveDispatch`) already started and froze the spend // policy, so `spendEnforced` cannot change before dispatch even though no reporter starts it here. const receiptMode = !sendBudgetState.adapterSendBudget?.spendEnforced && Boolean(options.comboInitialSend); + // #6969: Pool sends hold empty lifecycle frames so a pre-output refusal reaches the ladder below. + const preOutputRefusals = new PoolPreOutputRefusals(); + const preOutputRefusalHold = () => poolPreOutputRefusalHold(admissionState.authCtx, route.provider, connectMs, options.comboAttempt); try { // Transient-5xx pre-stream retry (devlog/_plan/260716_claudecode_hardening/010): // the ChatGPT backend emits transient 502/520s that an immediate retry absorbs. @@ -1034,11 +1039,12 @@ export async function preparePassthroughExchange( onCodexWsQuota: codexWsQuotaObserver(admissionState.authCtx, route.provider, route.modelId), beforeDispatch: isCanonicalOpenAiForwardProvider(route.provider) ? createCodexAuthDispatchGuard(admissionState.authCtx, options.codexAuthPolicy ?? config, route.modelId, options.admission, options.visionDescribeTerminal === true) : undefined, + preOutputRefusalHold: preOutputRefusalHold(), }), route.provider.authMode === "forward") // Every real attempt response — including an intermediate 5xx the // retry wrapper replaces — proves the host was reached (#914 review). - .then(adoptObservedResponse); + .then(adoptObservedResponse).then(preOutputRefusals.observe); }, { abortSignal: upstream.signal, label: safeHostLabel(request.url), attempts: sendBudgetState.initialSendAllowance(transientSendAttempts()), @@ -1049,6 +1055,8 @@ export async function preparePassthroughExchange( }, ); } catch (err) { + preOutputRefusals.recordOnLocalExit(err, options.abortSignal?.aborted === true || req.signal.aborted, + config, admissionState.authCtx, route.provider, route.modelId); return transportFailureResponse(err); } finally { request.releaseBodyObservation?.(); @@ -1675,6 +1683,7 @@ export async function preparePassthroughExchange( passthroughEstimate, stream: upstreamRequestsStream, httpOnly: canonicalBufferedJson, + preOutputRefusalHold: preOutputRefusalHold(), onResponse: (response, retryAuthCtx, retryRequest) => { adoptCodexWsStage(response); captureAffinityResponse( @@ -1701,6 +1710,8 @@ export async function preparePassthroughExchange( } } } + // #6969: a projection nothing above recovered goes on as the original stream, before any reader. + upstreamResponse = preOutputRefusals.restore(upstreamResponse); // The deterministic route record cannot classify history it never observed (restart, expiry, // eviction, or an older transcript). Inspect only a bounded clone of a 4xx whose exact outbound // Responses body still carries opaque state, then rebuild once through the ordinary adapter @@ -1968,6 +1979,7 @@ export async function preparePassthroughExchange( connectMs, localUpstream, upstreamResponse, + terminalOutcomeRecorded: preOutputRefusals.outcomeRecorded, }; } diff --git a/src/server/responses/ws-upstream.ts b/src/server/responses/ws-upstream.ts index b8834dc59b6..b7a7b98a1ca 100644 --- a/src/server/responses/ws-upstream.ts +++ b/src/server/responses/ws-upstream.ts @@ -23,7 +23,7 @@ import { CODEX_RESPONSES_HTTP_URL, CODEX_RESPONSES_WS_URL, CODEX_WS_FRAME_HEADER import { codexWsExchange } from "./codex-ws-exchange"; import { CodexWsSession } from "./codex-ws-session"; import { codexWsPool, codexWsReuseIdentity } from "./codex-ws-pool"; -import { codexWsCreateFrameExceedsLimit } from "./codex-ws-wire"; +import { codexWsCreateFrameExceedsLimit, type CodexWsPreOutputRefusalHold } from "./codex-ws-wire"; import { isLoopbackUrl, rewriteWebSocketDial } from "../../plugins/upstream-hooks"; import { outboundCredentials } from "../../lib/outbound-credential-registry"; export { CODEX_WS_LIVENESS_PING_INTERVAL_MS, CODEX_WS_RESPONSE_PRELUDE_TIMEOUT_MS, MAX_CODEX_WS_FRAME_BYTES, MAX_CODEX_WS_QUEUE_BYTES, @@ -163,6 +163,8 @@ export async function codexWsUpstreamFetch( nativeControl?: NativeResponseControl, beforeContinuation?: () => Promise, onPhysicalDispatch?: () => void, + /** Opt-in pre-output refusal hold (#6969); see codexWsExchange. */ + preOutputRefusalHold?: CodexWsPreOutputRefusalHold, ): Promise { const prepared = prepareCodexWsRequest(url, init); if (!prepared) return sseFallback(url, prepareCodexHttpInit(url, init)); @@ -248,6 +250,7 @@ export async function codexWsUpstreamFetch( session, url, init, prepared, sseFallback, onQuota, beforeDispatch, nativeControl: control, beforeContinuation, onPhysicalDispatch, + preOutputRefusalHold, bunVersion: typeof runtime === "string" ? runtime : runtime.version, }); } diff --git a/structure/transports/responses-failover.md b/structure/transports/responses-failover.md index 6c8382e6b5e..a96d49df479 100644 --- a/structure/transports/responses-failover.md +++ b/structure/transports/responses-failover.md @@ -542,7 +542,7 @@ Copilot Auto replacement, OAuth-refresh and native-429 negotiation follow the [C Refreshed or rotated Copilot native Responses may hand off only to the known Chat adapter, retaining the request-owned inference-401 guard and existing admission, translator and send budget without recursive setup; Chat 401 refresh renegotiates before rebuilding. Transferred 429 permits are rebooked against the actual Chat target and settled by physical-send reporting; unused or untransferred permits refund even on negotiation failure or early sidecar/build exit. Native Responses retains pre-stream OAuth HTTP-429 quorum, cooldown and the generic stable snapshot ceiling below. The complete credential/transport/replay identity refreshes; usage belongs to the serving account. -Single-account installs do not rotate; a missing alternate credential preserves the original error while transient recovery remains available. Translated Anthropic exact revoked-token 401s follow [the account-pool contract](../providers/anthropic-account-pool.md#revoked-oauth-access-token-recovery), with existing output and send-budget gates. +Single-account installs do not rotate; a missing alternate credential preserves the original error while transient recovery remains available. Translated Anthropic exact revoked-token 401s follow [the account-pool contract](../providers/anthropic-account-pool.md#revoked-oauth-access-token-recovery), with existing output and send-budget gates. A pre-output upstream refusal on the native Codex Pool (#6969) uses the transient ladder and the account move that an HTTP 5xx always had. The WebSocket exchange projects the refusal ([WS prelude contract](responses-wire-shapes.md)); `fetchWithTransientRetry` retries the same account within the request's shared send budget, then `retryCodexPoolOnAlternateAccount` makes its one bounded move under the existing fixed-account, stored-401, uploaded-file, account-failover-permit and recovery-window guards. Whatever survives reaches the client as the original stream, so an exhausted pool answers exactly as before. Each projection's account outcome is written once: by the move helper (marked on the projection, so a late denial does not write it again and delivery's terminal recorder skips the restored stream), by delivery when no move was attempted, or by the dispatch catch when the caller aborts or the budget refuses during the backoff; a later send's transport failure stays that path's single writer. Combo children, 401-refresh and rebuild legs do not opt in. Native compact moves a 5xx that survives its own ladder to one alternate account as well: the alternate's identity governs its admission, the first answer stays intact when that send is refused before it physically leaves (budget, dispatch guard, pacing), a 5xx whose body confirms quota keeps the quota path, and the first account is recorded with its real status once the alternate was sent. Covered by `tests/responses/ws-pre-output-refusal.test.ts`, `tests/responses/responses-pool-overload-failover.test.ts` and `tests/responses/compact-transient-alternate.test.ts`. Kiro adapter additionally classifies bounded HTTP 400/403/429 refusals before output. Confirmed monthly exhaustion is persisted for the sent login, suspension is quarantined diff --git a/structure/transports/responses-wire-shapes.md b/structure/transports/responses-wire-shapes.md index 20646910249..edd5721ac7e 100644 --- a/structure/transports/responses-wire-shapes.md +++ b/structure/transports/responses-wire-shapes.md @@ -474,7 +474,7 @@ event, or as an errored SSE body after it, rather than as a retryable fetch failure, so HTTP fallback cannot duplicate that inference. The one exception is a socket that closed or errored before any Responses event on a provider that opted into `retryOnReset`: the passthrough dispatch may spend the request's replacement -grant on one HTTP send (see [ambiguous-resend gate](responses-failover.md#ambiguous-resend-gate)). +grant on one HTTP send (see [ambiguous-resend gate](responses-failover.md#ambiguous-resend-gate)). An authoritative upstream refusal before any output is not one of those ambiguous settles (#6969). On an opt-in exchange (`preOutputRefusalHold`, set only by the native Pool dispatch and its alternate-account send, never with native control) empty `response.created`/`response.in_progress`/`response.queued` frames are queued without committing for at most 10 s and never past half of `connectTimeoutMs` after the send started, with the prelude headers frozen at the first one. A bare `error` frame with a single transient 5xx wrapper status (or, with no wrapper alias, the recognized overload class), or an empty `response.failed` of the overload class, then settles as a replayable gateway status of that status. Quota or rate-limit identities, 4xx frames, stream ids, conflicting response ids or competing aliases never qualify, and message text can only veto. Any other settle during the hold — output, an unqualified frame, socket death, silence, abort, size limits, the hold deadline — first commits the queued preamble, so it is the same committed body failure it was before. The dispatch restores a surviving projection to the original event stream (byte for byte, same markers and stage record) right after its Pool block. A standalone no-response exchange has a 90-second prelude deadline in addition to the upgrade deadline. That prelude deadline is a ceiling, not a floor: the exchange runs under the diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index 4dc096c7133..f0baa213f6a 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -1178,7 +1178,7 @@ "winsw-stop-hardening.test.ts": "windows", "winsw.test.ts": "service", "workflow-budget.test.ts": "lib", - "ws-ambiguous-resend.test.ts": "responses", + "ws-ambiguous-resend.test.ts": "responses", "ws-pre-output-refusal.test.ts": "responses", "responses-pool-overload-failover.test.ts": "responses", "compact-transient-alternate.test.ts": "responses", "ws-endpoint.test.ts": "responses", "ws-failure-stage.test.ts": "responses", "ws-native-injection.test.ts": "responses", diff --git a/tests/responses/compact-transient-alternate.test.ts b/tests/responses/compact-transient-alternate.test.ts new file mode 100644 index 00000000000..4ec20a054ee --- /dev/null +++ b/tests/responses/compact-transient-alternate.test.ts @@ -0,0 +1,303 @@ +import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test"; +import * as authContext from "../../src/codex/auth-context"; +import { createHash } from "node:crypto"; +import { mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { clearAccountNeedsReauth } from "../../src/codex/auth-api"; +import { clearPoolRotationState } from "../../src/codex/pool-rotation"; +import { clearAccountQuota, setAccountQuotaFromParsed } from "../../src/codex/quota"; +import { clearCodexUpstreamHealth, clearThreadAccountMap, getCodexUpstreamHealth } from "../../src/codex/routing"; +import { clearUpstreamHostHealth } from "../../src/codex/upstream-host-health"; +import { handleResponsesCompact } from "../../src/server/responses/compact"; +import { + createRequestExecutionBudget, + CODEX_TEXT_GUARDED_BUDGET_POLICY, + type RequestSendObserver, +} from "../../src/lib/request-execution-budget"; +import { clearPoolRecoveryState } from "../../src/routing/probe-lease"; +import type { RequestLogContext } from "../../src/server/request-log"; +import type { OcxConfig } from "../../src/types"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; +import { removeTreeWithRetry } from "../helpers/remove-tree"; + +/** + * #6969 for native compact: a transient 5xx that survives the same-account ladder moves to one + * alternate Pool account, as the regular Responses path does. Compact used to move only on 429/402. + * The move is admitted under the alternate's identity, leaves the first answer intact when it is + * refused, and records the first account once, as a transient failure. + */ + +const WORK = "work"; +const OTHER = "other"; +const HOME_KEYS = ["HOME", "OPENCODEX_HOME", "CODEX_HOME"] as const; +const PROXY_ENV_KEYS = ["HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "NO_PROXY", "http_proxy", "https_proxy", "all_proxy", "no_proxy"] as const; + +type Answer = () => Response | Promise; + +const RealFetch = globalThis.fetch; +let home = ""; +let previousHomes: Array = []; +let savedProxyEnv: Record = {}; +let releaseSpendHome: (() => void) | undefined; +let sends: Array = []; + +function clearPoolState(): void { + for (const id of [WORK, OTHER]) clearAccountNeedsReauth(id); + clearCodexUpstreamHealth(); + clearUpstreamHostHealth(); + clearThreadAccountMap(); + clearPoolRotationState(); + clearPoolRecoveryState(); + clearAccountQuota(); +} + +beforeEach(() => { + savedProxyEnv = Object.fromEntries(PROXY_ENV_KEYS.map(key => [key, process.env[key]])); + for (const key of PROXY_ENV_KEYS) delete process.env[key]; + previousHomes = HOME_KEYS.map(key => process.env[key]); + home = mkdtempSync(join(tmpdir(), "ocx-compact-overload-")); + for (const key of HOME_KEYS) process.env[key] = home; + releaseSpendHome = acquireOwnedSpendHome(); + clearPoolState(); + for (const id of [WORK, OTHER]) setAccountQuotaFromParsed(id, { weeklyPercent: 10 }); + writeFileSync(join(home, "codex-accounts.json"), JSON.stringify(Object.fromEntries( + [WORK, OTHER].map(id => [id, { + credential: { + accessToken: `${id}-access`, + refreshToken: `${id}-grant`, + expiresAt: Date.now() + 3_600_000, + chatgptAccountId: `acc-${id}`, + }, + generation: 1, + refreshGrantFingerprint: createHash("sha256").update(`codex-refresh-grant:${id}-grant`).digest("hex"), + }]), + ))); + sends = []; +}); + +afterEach(() => { + releaseSpendHome?.(); + releaseSpendHome = undefined; + globalThis.fetch = RealFetch; + clearPoolState(); + for (const [index, key] of HOME_KEYS.entries()) { + if (previousHomes[index] === undefined) delete process.env[key]; + else process.env[key] = previousHomes[index]; + } + for (const key of PROXY_ENV_KEYS) { + if (savedProxyEnv[key] === undefined) delete process.env[key]; + else process.env[key] = savedProxyEnv[key]; + } + removeTreeWithRetry(home); +}); + +/** Answers each compact send in order and records which credential it carried. */ +function upstream(...answers: Answer[]): void { + globalThis.fetch = (async (_input: RequestInfo | URL, init?: RequestInit) => { + sends.push(new Headers(init?.headers).get("authorization")); + const answer = answers[sends.length - 1] ?? answers.at(-1)!; + return answer(); + }) as typeof fetch; +} + +const overloaded = (headers: Record = {}): Answer => () => Response.json( + { error: { type: "server_error", code: "server_is_overloaded", message: "Our servers are currently overloaded." } }, + { status: 503, headers }, +); +// A Retry-After past the wait ceiling ends the ladder after one send, leaving base sends for B. +const overloadedOnce = overloaded({ "retry-after": "600" }); +const compacted: Answer = () => Response.json({ id: "cmp_1", object: "response.compaction", output: [] }); + +function poolConfig(extra: Partial = {}): OcxConfig { + return { + port: 0, + defaultProvider: "openai", + providers: { + openai: { + adapter: "openai-responses", + baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", + codexAccountMode: "pool", + }, + }, + activeCodexAccountId: WORK, + autoSwitchThreshold: 0, + accountPoolStrategy: "round-robin", + codexAccounts: [WORK, OTHER].map(id => ({ id, label: id })), + ...extra, + } as OcxConfig; +} + +async function compact(options: { + config?: OcxConfig; + model?: string; + budget?: ReturnType; + logCtx?: RequestLogContext; + signal?: AbortSignal; +} = {}) { + const logCtx = options.logCtx ?? { model: "", provider: "" }; + const request = new Request("http://localhost/v1/responses/compact", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: options.model ?? "gpt-5.5", input: [{ role: "user", content: "compact this" }] }), + ...(options.signal ? { signal: options.signal } : {}), + }); + const response = await handleResponsesCompact(request, options.config ?? poolConfig(), logCtx, undefined, undefined, + options.budget ? { sendBudget: options.budget } : undefined); + return { response, body: await response.text() }; +} + +function failures(accountId: string): number { + return getCodexUpstreamHealth(accountId)?.consecutiveFailures ?? 0; +} + +describe("native compact moves a surviving transient 5xx to one alternate account (#6969)", () => { + test("A's ladder, then B from the final recovery reserve, serves the compact", async () => { + upstream(overloaded(), overloaded(), overloaded(), compacted); + const { response } = await compact(); + expect(response.status).toBe(200); + expect(sends).toEqual(["Bearer work-access", "Bearer work-access", "Bearer work-access", "Bearer other-access"]); + expect(failures(WORK)).toBe(1); + expect(failures(OTHER)).toBe(0); + }); + + test("B served from the remaining base allowance when A's ladder ended early", async () => { + upstream(overloadedOnce, compacted); + const { response } = await compact(); + expect(response.status).toBe(200); + expect(sends).toEqual(["Bearer work-access", "Bearer other-access"]); + expect(failures(WORK)).toBe(1); + }); + + test("B failing too returns B's answer and records each account once", async () => { + upstream(overloaded(), overloaded(), overloaded(), overloaded()); + const { response, body } = await compact(); + expect(response.status).toBe(503); + expect(body).toContain("server_is_overloaded"); + expect(sends).toHaveLength(4); + expect(failures(WORK)).toBe(1); + expect(failures(OTHER)).toBe(1); + }); + + test("a pinned account never moves", async () => { + upstream(overloaded(), overloaded(), overloaded(), compacted); + const { response } = await compact({ config: poolConfig({ codexAccountNamespaces: { side: WORK } }), model: "side/gpt-5.5" }); + expect(response.status).toBe(503); + expect(sends.every(auth => auth === "Bearer work-access")).toBe(true); + }); + + test("a 5xx whose body confirms quota keeps today's path and does not take the transient move", async () => { + const quota: Answer = () => Response.json({ error: { message: "You have hit your usage limit." } }, { status: 503 }); + upstream(quota, quota, quota, compacted); + await compact(); + expect(sends).not.toContain("Bearer other-access"); + }); + + test("B refused before sending by a spent budget leaves A's answer and outcome intact", async () => { + upstream(overloaded(), overloaded(), overloaded(), compacted); + const budget = createRequestExecutionBudget({ ...CODEX_TEXT_GUARDED_BUDGET_POLICY, finalRecoveryAllowance: 0 }); + const { response, body } = await compact({ budget }); + expect(response.status).toBe(503); + expect(body).toContain("server_is_overloaded"); + expect(sends.every(auth => auth === "Bearer work-access")).toBe(true); + expect(failures(WORK)).toBe(1); + expect(failures(OTHER)).toBe(0); + }); + + test("a granted reservation whose physical start fails is released, and nothing is sent", async () => { + upstream(overloaded(), overloaded(), overloaded(), compacted); + let refunds = 0; + let reserveStarted = false; + const observer: RequestSendObserver = { + enforced: true, + charge: () => true, + refund: () => { refunds += 1; }, + startRequest: () => {}, + beginReporter: () => ({ start() {}, rebindTarget: () => true, report() {}, close() {} }), + // Base sends start; the reserve's physical start is refused. + ensureSeed: target => { + if (target.identityId === undefined) { reserveStarted = true; return undefined; } + return { seed: true } as never; + }, + }; + const budget = createRequestExecutionBudget(undefined, undefined, observer); + const { response } = await compact({ budget }); + expect(reserveStarted).toBe(true); + expect(refunds).toBe(1); + expect(response.status).toBe(503); + expect(sends.every(auth => auth === "Bearer work-access")).toBe(true); + expect(failures(WORK)).toBe(1); + }); + + /** + * Unequal identity balances against the real spend ledger. The identity cap is exactly what the + * first account's own sends consume, so the alternate's send is admitted only if it is charged to + * the alternate's identity; charged to the first account it would be refused and never sent. + */ + async function ledgerCompact(identityCap: number, ...answers: Answer[]) { + const { createSpendReservationLedger, DEFAULT_SPEND_RESERVATION_POLICY } = await import("../../src/lib/spend-reservation-ledger"); + const { createRequestSpendTracker } = await import("../../src/server/responses/request-spend"); + const ledger = createSpendReservationLedger({ salt: "compact-6969", policy: { + ...DEFAULT_SPEND_RESERVATION_POLICY, identity: { maxTokens: identityCap }, pool: { maxTokens: 1_000 }, + } }); + const logCtx: RequestLogContext = { model: "", provider: "", spendInputEstimateTokens: 1, spendOutputCeilingTokens: 0 }; + const tracker = createRequestSpendTracker(logCtx, undefined, ledger); + upstream(...answers); + const config = poolConfig({ codexAccounts: [{ id: WORK, label: WORK, logLabel: "pa0a0a0" }, { id: OTHER, label: OTHER, logLabel: "pb0b0b0" }] } as Partial); + const result = await compact({ config, logCtx, budget: createRequestExecutionBudget(undefined, undefined, tracker) }); + const used = (label: string) => { + const snapshot = ledger.snapshot("identity", label); + return snapshot ? snapshot.reserved + snapshot.settled + snapshot.unresolved : 0; + }; + return { ...result, first: used("pa0a0a0"), alternate: used("pb0b0b0") }; + } + + test("on the base path the alternate is admitted against its own identity balance", async () => { + const result = await ledgerCompact(1, overloadedOnce, compacted); + expect(result.response.status).toBe(200); + expect(sends).toEqual(["Bearer work-access", "Bearer other-access"]); + expect(result.first).toBe(1); + expect(result.alternate).toBe(1); + }); + + test("on the reserve path the alternate is admitted against its own identity balance", async () => { + const result = await ledgerCompact(3, overloaded(), overloaded(), overloaded(), compacted); + expect(result.response.status).toBe(200); + expect(sends.at(-1)).toBe("Bearer other-access"); + expect(result.first).toBe(3); + expect(result.alternate).toBe(1); + }); + + test("a dispatch guard refusing B before it leaves keeps A's answer and outcome", async () => { + upstream(overloaded(), overloaded(), overloaded(), compacted); + const original = authContext.createCodexAuthDispatchGuard; + const guard = spyOn(authContext, "createCodexAuthDispatchGuard").mockImplementation((ctx, ...rest) => { + if (ctx.kind === "pool" && ctx.accountId === OTHER) { + return () => { throw new authContext.CodexPoolAccountCreditsOffError(OTHER, Date.now() + 60_000); }; + } + return original(ctx, ...rest); + }); + try { + const { response, body } = await compact(); + expect(response.status).toBe(503); + expect(body).toContain("server_is_overloaded"); + expect(sends.every(auth => auth === "Bearer work-access")).toBe(true); + expect(failures(WORK)).toBe(1); + expect(failures(OTHER)).toBe(0); + } finally { + guard.mockRestore(); + } + }); + + test("a caller cancelling while B is in flight records A once and answers 499", async () => { + const abort = new AbortController(); + upstream(overloaded(), overloaded(), overloaded(), async () => { + abort.abort(); + throw new DOMException("The operation was aborted.", "AbortError"); + }); + const { response } = await compact({ signal: abort.signal }); + expect(response.status).toBe(499); + expect(failures(WORK)).toBe(1); + }); +}); diff --git a/tests/responses/responses-account-label.test.ts b/tests/responses/responses-account-label.test.ts index bfccb8afb10..fc36df90b15 100644 --- a/tests/responses/responses-account-label.test.ts +++ b/tests/responses/responses-account-label.test.ts @@ -228,6 +228,10 @@ describe("Responses account usage attribution", () => { primary: { used_percent: 10, window_minutes: 10080 }, } }); payload({ type: "response.created", response: { id: "quota-response" } }); + // #6969: a Pool exchange holds empty lifecycle frames; the first output event is what + // commits the response here, so the handler returns before the final quota frame. + payload({ type: "response.output_item.added", response_id: "quota-response", output_index: 0, + item: { id: "quota-item", type: "message", role: "assistant", content: [] } }); void finalQuotaAllowed.then(() => { payload({ type: "codex.rate_limits", rate_limits: { primary: { used_percent: 100, window_minutes: 10080 }, diff --git a/tests/responses/responses-pool-overload-failover.test.ts b/tests/responses/responses-pool-overload-failover.test.ts new file mode 100644 index 00000000000..108fab86254 --- /dev/null +++ b/tests/responses/responses-pool-overload-failover.test.ts @@ -0,0 +1,307 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { createHash } from "node:crypto"; +import { mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { clearAccountNeedsReauth } from "../../src/codex/auth-api"; +import { clearPoolRotationState } from "../../src/codex/pool-rotation"; +import { clearAccountQuota, setAccountQuotaFromParsed } from "../../src/codex/quota"; +import { clearCodexUpstreamHealth, clearThreadAccountMap, getCodexUpstreamHealth } from "../../src/codex/routing"; +import { handleResponses } from "../../src/server/responses"; +import { codexWsPool } from "../../src/server/responses/codex-ws-pool"; +import { createRequestExecutionBudget } from "../../src/lib/request-execution-budget"; +import { clearPoolRecoveryState, configureSharedPoolBackpressure } from "../../src/routing/probe-lease"; +import type { RequestLogContext } from "../../src/server/request-log"; +import type { OcxConfig } from "../../src/types"; +import { BOUNDED_WS_RUNTIME } from "../helpers/ws-upstream-fixtures"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; +import { removeTreeWithRetry } from "../helpers/remove-tree"; + +/** + * #6969 end to end: a pooled native Codex turn that the upstream refuses over the WebSocket before + * any output now takes the same-account transient ladder and then one alternate account, and a + * turn nothing recovers reaches the client as the stream it always received. Output already + * relayed, a pinned account and a cancelled caller keep today's answers. + */ + +const WORK = "work"; +const OTHER = "other"; +const HOME_KEYS = ["HOME", "OPENCODEX_HOME", "CODEX_HOME"] as const; +const PROXY_ENV_KEYS = ["HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "NO_PROXY", "http_proxy", "https_proxy", "all_proxy", "no_proxy"] as const; + +type Listener = (event: unknown) => void; +type Step = "refuse" | "ok" | "output-then-refuse" | "refuse-then-abort" | "transport-error" | "silent"; + +class FakeWebSocket { + static instances: FakeWebSocket[] = []; + static plan: Step[] = []; + static onAbortStep: (() => void) | undefined; + url: string; + headers: Headers; + sent: string[] = []; + closed = false; + listeners = new Map(); + + constructor(url: string, options?: { headers?: HeadersInit }) { + this.url = url; + this.headers = new Headers(options?.headers); + const step = FakeWebSocket.plan[FakeWebSocket.instances.length] ?? "ok"; + FakeWebSocket.instances.push(this); + queueMicrotask(() => this.run(step)); + } + + run(step: Step) { + if (step === "transport-error") { + // Fails before the upgrade: the exchange falls back to HTTP, whose send then fails too. + this.emit("error", {}); + return; + } + this.emit("open", {}); + if (step === "silent") return; + const id = `resp_${FakeWebSocket.instances.indexOf(this)}`; + this.frame({ type: "response.created", response: { id, status: "in_progress", output: [] } }); + this.frame({ type: "response.in_progress", response: { id, status: "in_progress", output: [] } }); + if (step === "ok") { + this.frame({ type: "response.completed", response: { id, status: "completed", output: [] } }); + return; + } + if (step === "output-then-refuse") { + this.frame({ type: "response.output_item.added", response_id: id, output_index: 0, + item: { id: "msg_1", type: "message", role: "assistant", content: [] } }); + } + this.frame({ type: "error", status: 503, + error: { type: "server_error", code: "server_is_overloaded", message: "Our servers are currently overloaded." } }); + if (step === "refuse-then-abort") setTimeout(() => FakeWebSocket.onAbortStep?.(), 100); + } + + addEventListener(type: string, listener: Listener) { + const list = this.listeners.get(type) ?? []; + list.push(listener); + this.listeners.set(type, list); + } + + removeEventListener(type: string, listener: Listener) { + this.listeners.set(type, (this.listeners.get(type) ?? []).filter(value => value !== listener)); + } + + emit(type: string, event: unknown = {}) { + for (const listener of this.listeners.get(type) ?? []) listener(event); + } + + frame(payload: unknown) { + this.emit("message", { data: JSON.stringify(payload) }); + } + + send(data: string) { this.sent.push(data); } + + close() { this.closed = true; } +} + +const RealWebSocket = globalThis.WebSocket; +const RealFetch = globalThis.fetch; +let home = ""; +let previousHomes: Array = []; +let savedProxyEnv: Record = {}; +let releaseSpendHome: (() => void) | undefined; + +function clearPoolState(): void { + for (const id of [WORK, OTHER]) clearAccountNeedsReauth(id); + clearCodexUpstreamHealth(); + clearThreadAccountMap(); + clearPoolRotationState(); + clearAccountQuota(); + clearPoolRecoveryState(); +} + +beforeEach(() => { + savedProxyEnv = Object.fromEntries(PROXY_ENV_KEYS.map(key => [key, process.env[key]])); + for (const key of PROXY_ENV_KEYS) delete process.env[key]; + previousHomes = HOME_KEYS.map(key => process.env[key]); + home = mkdtempSync(join(tmpdir(), "ocx-pool-overload-")); + for (const key of HOME_KEYS) process.env[key] = home; + releaseSpendHome = acquireOwnedSpendHome(); + clearPoolState(); + for (const id of [WORK, OTHER]) setAccountQuotaFromParsed(id, { weeklyPercent: 10 }); + writeFileSync(join(home, "codex-accounts.json"), JSON.stringify(Object.fromEntries( + [WORK, OTHER].map(id => [id, { + credential: { + accessToken: `${id}-access`, + refreshToken: `${id}-grant`, + expiresAt: Date.now() + 3_600_000, + chatgptAccountId: `acc-${id}`, + }, + generation: 1, + refreshGrantFingerprint: createHash("sha256").update(`codex-refresh-grant:${id}-grant`).digest("hex"), + }]), + ))); + FakeWebSocket.instances = []; + FakeWebSocket.plan = []; + FakeWebSocket.onAbortStep = undefined; + globalThis.WebSocket = FakeWebSocket as unknown as typeof WebSocket; + // Every send must ride the WebSocket; an HTTP send here is a failure of the test's premise. + globalThis.fetch = (async () => { throw new Error("unexpected HTTP send"); }) as unknown as typeof fetch; +}); + +afterEach(() => { + releaseSpendHome?.(); + releaseSpendHome = undefined; + globalThis.WebSocket = RealWebSocket; + globalThis.fetch = RealFetch; + codexWsPool.dispose(); + clearPoolState(); + for (const [index, key] of HOME_KEYS.entries()) { + if (previousHomes[index] === undefined) delete process.env[key]; + else process.env[key] = previousHomes[index]; + } + for (const key of PROXY_ENV_KEYS) { + if (savedProxyEnv[key] === undefined) delete process.env[key]; + else process.env[key] = savedProxyEnv[key]; + } + removeTreeWithRetry(home); +}); + +function poolConfig(accounts: string[] = [WORK, OTHER], extra: Partial = {}): OcxConfig { + return { + port: 0, + defaultProvider: "openai", + providers: { + openai: { + adapter: "openai-responses", + baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", + codexAccountMode: "pool", + }, + }, + activeCodexAccountId: WORK, + autoSwitchThreshold: 0, + accountPoolStrategy: "round-robin", + codexAccounts: accounts.map(id => ({ id, label: id })), + ...extra, + } as OcxConfig; +} + +async function send(config: OcxConfig, options: { model?: string; abortSignal?: AbortSignal } = {}) { + const logCtx: RequestLogContext = { model: "", provider: "" }; + const request = new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: options.model ?? "gpt-5.5", input: "hello", stream: true, store: false }), + ...(options.abortSignal ? { signal: options.abortSignal } : {}), + }); + const response = await handleResponses(request, config, logCtx, { + codexWsRuntimeIdentity: BOUNDED_WS_RUNTIME, + sendBudget: createRequestExecutionBudget(), + ...(options.abortSignal ? { abortSignal: options.abortSignal } : {}), + }); + const body = await response.text().catch(error => ``); + return { response, body, logCtx }; +} + +function sockets(): Array { + return FakeWebSocket.instances.map(ws => ws.headers.get("authorization")); +} + +function failures(accountId: string): number { + return getCodexUpstreamHealth(accountId)?.consecutiveFailures ?? 0; +} + +describe("pooled native Codex recovers a pre-output upstream refusal (#6969)", () => { + test("the same account is retried first and serves the turn", async () => { + FakeWebSocket.plan = ["refuse", "ok"]; + const { response, body, logCtx } = await send(poolConfig()); + expect(response.status).toBe(200); + expect(body).toContain("response.completed"); + expect(body).not.toContain("server_is_overloaded"); + expect(sockets()).toEqual(["Bearer work-access", "Bearer work-access"]); + expect(logCtx.activeAttempt?.sendCount).toBe(2); + expect(logCtx.activeAttempt?.recoveryKinds).toContain("transient-5xx"); + }); + + test("after the same-account ladder, one alternate account serves the turn", async () => { + FakeWebSocket.plan = ["refuse", "refuse", "refuse", "ok"]; + const { response, body } = await send(poolConfig()); + expect(response.status).toBe(200); + expect(body).toContain("response.completed"); + expect(sockets()).toEqual(["Bearer work-access", "Bearer work-access", "Bearer work-access", "Bearer other-access"]); + // The refused account is recorded once, as a transient failure. + expect(failures(WORK)).toBe(1); + expect(failures(OTHER)).toBe(0); + }); + + test("with every account overloaded the client receives the original stream, once per account in health", async () => { + FakeWebSocket.plan = ["refuse", "refuse", "refuse", "refuse"]; + const { response, body } = await send(poolConfig()); + expect(response.status).toBe(200); + expect(body).toContain("event: response.created"); + expect(body).toContain("server_is_overloaded"); + expect(FakeWebSocket.instances).toHaveLength(4); + expect(sockets().at(-1)).toBe("Bearer other-access"); + expect(failures(WORK)).toBe(1); + expect(failures(OTHER)).toBe(1); + }); + + test("a single-account pool retries only that account, then delivers the original stream", async () => { + FakeWebSocket.plan = ["refuse", "refuse", "refuse"]; + const { response, body } = await send(poolConfig([WORK])); + expect(response.status).toBe(200); + expect(body).toContain("server_is_overloaded"); + expect(sockets()).toEqual(["Bearer work-access", "Bearer work-access", "Bearer work-access"]); + expect(failures(WORK)).toBe(1); + }); + + test("a pinned account never moves to another account", async () => { + FakeWebSocket.plan = ["refuse", "refuse", "refuse", "ok"]; + const { response, body } = await send(poolConfig([WORK, OTHER], { codexAccountNamespaces: { side: WORK } }), + { model: "side/gpt-5.5" }); + expect(response.status).toBe(200); + expect(body).toContain("server_is_overloaded"); + expect(sockets()).toEqual(["Bearer work-access", "Bearer work-access", "Bearer work-access"]); + }); + + test("a refusal after output started is never sent again", async () => { + FakeWebSocket.plan = ["output-then-refuse", "ok"]; + const { response, body } = await send(poolConfig()); + expect(response.status).toBe(200); + expect(body).toContain("response.output_item.added"); + expect(body).toContain("server_is_overloaded"); + expect(FakeWebSocket.instances).toHaveLength(1); + }); + + test("a caller cancelling during the backoff stops every later send and records the refusal once", async () => { + const abort = new AbortController(); + FakeWebSocket.plan = ["refuse-then-abort", "ok"]; + FakeWebSocket.onAbortStep = () => abort.abort(); + const { response } = await send(poolConfig(), { abortSignal: abort.signal }); + expect(response.status).toBe(499); + expect(FakeWebSocket.instances).toHaveLength(1); + expect(failures(WORK)).toBe(1); + }); + + test("a recovery window that refuses the move keeps the first account's stream and records it once", async () => { + FakeWebSocket.plan = ["refuse", "refuse", "refuse", "ok"]; + configureSharedPoolBackpressure({ windowMs: 10_000, maxRetryRatio: 0, minRecoveryAllowance: 0 }); + const { response, body } = await send(poolConfig()); + expect(response.status).toBe(200); + expect(body).toContain("server_is_overloaded"); + expect(sockets().every(auth => auth === "Bearer work-access")).toBe(true); + expect(failures(WORK)).toBe(1); + expect(failures(OTHER)).toBe(0); + }); + + test("a refusal followed by a failed send is recorded once, by the transport failure", async () => { + FakeWebSocket.plan = ["refuse", "transport-error"]; + const { response } = await send(poolConfig()); + expect(response.status).toBe(502); + expect(FakeWebSocket.instances).toHaveLength(2); + expect(failures(WORK)).toBe(1); + }); + + test("a refusal followed by a header timeout settles the timeout and records the account once", async () => { + FakeWebSocket.plan = ["refuse", "silent"]; + const { response } = await send(poolConfig([WORK, OTHER], { connectTimeoutMs: 1_000 })); + expect(response.status).toBe(504); + expect(FakeWebSocket.instances).toHaveLength(2); + expect(sockets().every(auth => auth === "Bearer work-access")).toBe(true); + expect(failures(WORK)).toBe(1); + }); +}); diff --git a/tests/responses/ws-pre-output-refusal.test.ts b/tests/responses/ws-pre-output-refusal.test.ts new file mode 100644 index 00000000000..2ce9fd2bdac --- /dev/null +++ b/tests/responses/ws-pre-output-refusal.test.ts @@ -0,0 +1,367 @@ +import { afterEach, beforeEach, describe, expect, jest, test } from "bun:test"; +import { codexWsUpstreamFetch } from "../../src/server/responses/ws-upstream"; +import { codexWsExchange } from "../../src/server/responses/codex-ws-exchange"; +import { CodexWsSession } from "../../src/server/responses/codex-ws-session"; +import { prepareCodexWsRequest } from "../../src/server/responses/codex-ws-request"; +import { codexWsPool } from "../../src/server/responses/codex-ws-pool"; +import { + codexWsPreOutputRefusalHold, + codexWsPreOutputRefusalStatus, + codexWsSocketDeathStage, + isCodexWsPreOutputRefusal, + isCodexWsUpstreamResponse, + readCodexWsStage, + restoreCodexWsPreOutputRefusal, +} from "../../src/server/responses/codex-ws-wire"; +import { isNonReplayableResponse } from "../../src/lib/upstream-retry"; +import { BOUNDED_WS_RUNTIME, streamingInit } from "../helpers/ws-upstream-fixtures"; + +/** + * #6969: the Codex WebSocket used to commit a 200 at `response.created`, so an upstream that then + * refused the turn with a structured 5xx and no output reached no retry layer at all. With the + * opt-in hold, such a refusal settles as a replayable gateway status and can be restored to the + * original stream; every other outcome must stay exactly what it was. + */ + +const CODEX_URL = "https://chatgpt.com/backend-api/codex/responses"; +const HOLD_MS = 10_000; + +type Listener = (event: unknown) => void; + +/** Minimal scriptable stand-in for Bun's WebSocket, mirroring `ws-ambiguous-resend.test.ts`. */ +class FakeWebSocket { + static instances: FakeWebSocket[] = []; + static script: (ws: FakeWebSocket) => void = () => {}; + url: string; + sent: string[] = []; + closed = false; + listeners = new Map(); + + constructor(url: string) { + this.url = url; + FakeWebSocket.instances.push(this); + queueMicrotask(() => FakeWebSocket.script(this)); + } + + addEventListener(type: string, listener: Listener) { + const list = this.listeners.get(type) ?? []; + list.push(listener); + this.listeners.set(type, list); + } + + removeEventListener(type: string, listener: Listener) { + this.listeners.set(type, (this.listeners.get(type) ?? []).filter(value => value !== listener)); + } + + emit(type: string, event: unknown = {}) { + for (const listener of this.listeners.get(type) ?? []) listener(event); + } + + frame(payload: unknown) { + this.emit("message", { data: JSON.stringify(payload) }); + } + + send(data: string) { + this.sent.push(data); + } + + close() { this.closed = true; } +} + +const RealWebSocket = globalThis.WebSocket; + +beforeEach(() => { + FakeWebSocket.instances = []; + FakeWebSocket.script = () => {}; +}); + +afterEach(() => { + jest.useRealTimers(); + globalThis.WebSocket = RealWebSocket; + codexWsPool.dispose(); +}); + +function installFake(script: (ws: FakeWebSocket) => void) { + FakeWebSocket.script = script; + globalThis.WebSocket = FakeWebSocket as unknown as typeof WebSocket; +} + +const noFallback = (async () => { + throw new Error("fallback must not run after open"); +}) as unknown as typeof fetch; + +function exchange( + script: (ws: FakeWebSocket) => void, + holdMs = HOLD_MS, + until = Number.MAX_SAFE_INTEGER, +): Promise { + installFake(script); + return codexWsUpstreamFetch( + CODEX_URL, streamingInit(), noFallback, BOUNDED_WS_RUNTIME, + undefined, undefined, undefined, undefined, undefined, { maxMs: holdMs, until }, + ); +} + +const CREATED = { type: "response.created", response: { id: "resp_1", status: "in_progress", output: [] } }; +const IN_PROGRESS = { type: "response.in_progress", response: { id: "resp_1", status: "in_progress", output: [] } }; +const OVERLOADED = { + type: "error", + status: 503, + error: { type: "server_error", code: "server_is_overloaded", message: "Our servers are currently overloaded. Please try again later." }, +}; +const SERVER_ERROR = { + type: "error", + status: 502, + error: { type: "server_error", message: "An error occurred while processing your request." }, +}; +const FAILED_OVERLOADED = { + type: "response.failed", + response: { id: "resp_1", status: "failed", output: [], error: { code: "server_is_overloaded", message: "overloaded" } }, +}; +const OUTPUT_ITEM = { + type: "response.output_item.added", + response_id: "resp_1", + output_index: 0, + item: { id: "msg_1", type: "message", role: "assistant", content: [] }, +}; +const QUOTA = (percent: number) => ({ + type: "codex.rate_limits", + rate_limits: { primary: { used_percent: percent, window_minutes: 10080 } }, +}); + +function opened(...frames: unknown[]): (ws: FakeWebSocket) => void { + return ws => { + ws.emit("open", {}); + for (const frame of frames) ws.frame(frame); + }; +} + +describe("an authoritative refusal before any output becomes a replayable gateway status (#6969)", () => { + test("after an empty created/in_progress preamble", async () => { + const projection = await exchange(opened(CREATED, IN_PROGRESS, OVERLOADED)); + expect(projection.status).toBe(503); + expect(isNonReplayableResponse(projection)).toBe(false); + expect(isCodexWsPreOutputRefusal(projection)).toBe(true); + expect(await projection.clone().json()).toMatchObject({ error: { code: "server_is_overloaded" } }); + const stage = readCodexWsStage(projection); + expect(stage).toMatchObject({ sent: true, relayedEvents: 3, terminalEvent: "error", terminalErrorStatus: 503 }); + + const restored = restoreCodexWsPreOutputRefusal(projection); + expect(restored.status).toBe(200); + expect(restored.headers.get("content-type")).toContain("text/event-stream"); + expect(isCodexWsUpstreamResponse(restored)).toBe(true); + // The same record object: the attempt that adopted it keeps one stage. + expect(readCodexWsStage(restored)).toBe(stage); + const body = await restored.text(); + const order = ["event: response.created", "event: response.in_progress", "event: error"].map(marker => body.indexOf(marker)); + expect(order.every(index => index >= 0)).toBe(true); + expect([...order].sort((a, b) => a - b)).toEqual(order); + expect(body).toContain("server_is_overloaded"); + // Single use. + expect(restoreCodexWsPreOutputRefusal(projection)).toBe(projection); + }); + + test("as the first Responses event, with the wrapper's own 5xx", async () => { + const projection = await exchange(opened(SERVER_ERROR)); + expect(projection.status).toBe(502); + expect(isNonReplayableResponse(projection)).toBe(false); + const body = await restoreCodexWsPreOutputRefusal(projection).text(); + expect(body.startsWith("event: error")).toBe(true); + }); + + test("as an empty response.failed carrying the overload class", async () => { + const projection = await exchange(opened(CREATED, FAILED_OVERLOADED)); + expect(projection.status).toBe(503); + expect(isCodexWsPreOutputRefusal(projection)).toBe(true); + }); + + test("restored headers keep the prelude frozen at the first Responses event", async () => { + const projection = await exchange(opened(QUOTA(10), CREATED, QUOTA(20), OVERLOADED)); + expect(projection.status).toBe(503); + const restored = restoreCodexWsPreOutputRefusal(projection); + expect(restored.headers.get("x-codex-primary-used-percent")).toBe("10"); + }); +}); + +describe("everything else keeps today's settle", () => { + async function committed(response: Response): Promise { + expect(response.status).toBe(200); + expect(isCodexWsPreOutputRefusal(response)).toBe(false); + return response.text(); + } + + test("without the opt-in the refusal is relayed in a committed 200", async () => { + const body = await committed(await exchange(opened(CREATED, IN_PROGRESS, OVERLOADED), 0)); + expect(body).toContain("server_is_overloaded"); + }); + + test("once output started the refusal stays in the body", async () => { + const body = await committed(await exchange(opened(CREATED, OUTPUT_ITEM, OVERLOADED))); + expect(body).toContain("response.output_item.added"); + expect(body).toContain("server_is_overloaded"); + }); + + test("a created snapshot that already carries output commits", async () => { + const resumed = { type: "response.created", response: { id: "resp_1", output: [{ id: "msg_0", type: "message" }] } }; + expect(await committed(await exchange(opened(resumed, OVERLOADED)))).toContain("server_is_overloaded"); + }); + + test.each([ + ["a 429 wrapper", { type: "error", status: 429, error: { code: "rate_limit_exceeded", message: "slow down" } }], + ["a 5xx that names quota", { type: "error", status: 503, error: { code: "usage_limit_reached", message: "limit" } }], + ["a 5xx whose message is quota", { type: "error", status: 502, error: { message: "You exceeded your current quota" } }], + ["a 5xx with a rate_limit_* code", { type: "error", status: 503, error: { code: "rate_limit_requests", message: "Please try again later" } }], + ["a 5xx with a rate_limit_* type", { type: "error", status: 503, error: { type: "rate_limit_tokens", message: "Please try again later" } }], + ["a foreign stream id", { ...OVERLOADED, stream_id: "other" }], + ["doubled wrapper aliases", { ...OVERLOADED, status_code: 503 }], + ["an unrecognized code without a wrapper status", { type: "error", error: { code: "mystery", message: "x" } }], + ])("after the preamble, %s is relayed in the body", async (_name, frame) => { + await committed(await exchange(opened(CREATED, frame))); + }); + + test.each([ + ["first", [] as unknown[]], + ["after the held preamble", [CREATED]], + ])("a refusal whose metadata cannot be projected is delivered in the body when it arrives %s", async (_name, before) => { + const oversized = { ...OVERLOADED, headers: { "x-request-id": "a".repeat(5_000) } }; + expect(await committed(await exchange(opened(...before, oversized)))).toContain("server_is_overloaded"); + }); + + test("a queue overflow while holding fails the committed body, as it did before the hold", async () => { + const padded = (index: number) => ({ + type: "response.in_progress", + response: { id: "resp_1", status: "in_progress", output: [], metadata: { pad: `${index}${"x".repeat(1_500_000)}` } }, + }); + const response = await exchange(opened(CREATED, ...Array.from({ length: 7 }, (_, index) => padded(index)))); + expect(response.status).toBe(200); + expect(isCodexWsPreOutputRefusal(response)).toBe(false); + await expect(response.text()).rejects.toThrow("buffered queue limit"); + }); + + test("lifecycle frames do not extend the hold deadline", async () => { + jest.useFakeTimers(); + let socket: FakeWebSocket | undefined; + let settled: Response | undefined; + const pending = exchange(ws => { + socket = ws; + ws.emit("open", {}); + ws.frame(CREATED); + }); + void pending.then(response => { settled = response; }); + for (let i = 0; i < 5 && !socket; i += 1) await Promise.resolve(); + jest.advanceTimersByTime(HOLD_MS - 4_000); + socket!.frame(IN_PROGRESS); + jest.advanceTimersByTime(4_000); + for (let i = 0; i < 5 && !settled; i += 1) await Promise.resolve(); + expect(settled?.status).toBe(200); + }); + + test("a first-frame 4xx keeps the existing refused-create projection", async () => { + const response = await exchange(opened({ type: "error", status: 429, error: { code: "rate_limit_exceeded", message: "slow" } })); + expect(response.status).toBe(429); + expect(isCodexWsPreOutputRefusal(response)).toBe(false); + }); + + test("a terminal naming another response than the held one commits", async () => { + const foreign = { ...FAILED_OVERLOADED, response_id: "resp_other" }; + await committed(await exchange(opened(CREATED, foreign))); + }); + + test("a socket death after the held preamble is still a failed body, not a socket-death settle", async () => { + const response = await exchange(ws => { + ws.emit("open", {}); + ws.frame(CREATED); + ws.emit("close", { code: 1006 }); + }); + expect(response.status).toBe(200); + expect(codexWsSocketDeathStage(response)).toBeUndefined(); + await expect(response.text()).rejects.toThrow("closed before a Responses terminal event"); + }); + + test("a socket death before any Responses event keeps the non-replayable 502", async () => { + const response = await exchange(ws => { + ws.emit("open", {}); + ws.emit("close", { code: 1006 }); + }); + expect(response.status).toBe(502); + expect(isNonReplayableResponse(response)).toBe(true); + }); + + test("the hold is bounded: the preamble commits when it expires and the turn continues", async () => { + jest.useFakeTimers(); + let socket: FakeWebSocket | undefined; + const pending = exchange(ws => { + socket = ws; + ws.emit("open", {}); + ws.frame(CREATED); + }); + for (let i = 0; i < 5 && !socket; i += 1) await Promise.resolve(); + jest.advanceTimersByTime(HOLD_MS); + const response = await pending; + expect(response.status).toBe(200); + socket!.frame({ type: "response.completed", response: { id: "resp_1", status: "completed", output: [] } }); + const body = await response.text(); + expect(body.indexOf("event: response.created")).toBeLessThan(body.indexOf("event: response.completed")); + }); + + test("a native-control exchange never holds", async () => { + installFake(opened(CREATED, OVERLOADED)); + const init = streamingInit(); + const prepared = prepareCodexWsRequest(CODEX_URL, init)!; + const session = new CodexWsSession("wss://chatgpt.com/backend-api/codex/responses", prepared.headers, true); + const nativeControl = { + kind: "steering" as const, + relayActive: false, + attached: false, + ended: false, + attach() { return () => {}; }, + observe() { return false; }, + steer() {}, + continue() { return false; }, + }; + try { + expect(session.reserve()).toBe(true); + const response = await codexWsExchange({ + session, url: CODEX_URL, init, prepared, nativeControl, sseFallback: noFallback, preOutputRefusalHold: { maxMs: HOLD_MS, until: Number.MAX_SAFE_INTEGER }, + }); + expect(response.status).toBe(200); + expect(isCodexWsPreOutputRefusal(response)).toBe(false); + } finally { session.dispose(); } + }); +}); + +describe("refusal identity and hold bound", () => { + test("only structured transient 5xx evidence counts", () => { + expect(codexWsPreOutputRefusalStatus(OVERLOADED)).toBe(503); + expect(codexWsPreOutputRefusalStatus({ type: "error", error: { code: "server_is_overloaded" } })).toBe(503); + expect(codexWsPreOutputRefusalStatus({ type: "error", status: 500, error: { message: "boom" } })).toBe(500); + expect(codexWsPreOutputRefusalStatus({ type: "error", status: 400, error: { message: "bad" } })).toBeUndefined(); + expect(codexWsPreOutputRefusalStatus({ type: "error", status: "503" })).toBeUndefined(); + expect(codexWsPreOutputRefusalStatus({ type: "error", status: 503, error: { code: "rate_limit_requests" } })).toBeUndefined(); + expect(codexWsPreOutputRefusalStatus({ ...FAILED_OVERLOADED, response: { ...FAILED_OVERLOADED.response, output: [{}] } })) + .toBeUndefined(); + }); + + test("the hold never exceeds 10 s, and every hold ends by half the header deadline from the send", () => { + expect(codexWsPreOutputRefusalHold(200_000, 1_000)).toEqual({ maxMs: 10_000, until: 101_000 }); + expect(codexWsPreOutputRefusalHold(8_000, 1_000)).toEqual({ maxMs: 4_000, until: 5_000 }); + }); + + test("a lifecycle frame after the hold's end commits at once", async () => { + const response = await exchange(opened(CREATED, OVERLOADED), HOLD_MS, Date.now() - 1); + // Nothing was held, so the refusal after it is relayed in the committed body as before. + expect(response.status).toBe(200); + expect(await response.text()).toContain("server_is_overloaded"); + }); + + test("the hold ends at its absolute end even when the maximum is longer", async () => { + const started = Date.now(); + const response = await exchange(ws => { + ws.emit("open", {}); + ws.frame(CREATED); + }, HOLD_MS, Date.now() + 50); + expect(response.status).toBe(200); + expect(Date.now() - started).toBeLessThan(HOLD_MS); + void response.body?.cancel(); + }); +}); From ce97299e159722f5a98e664f2b0069cff29fa457 Mon Sep 17 00:00:00 2001 From: lidge-jun Date: Tue, 13 Oct 2026 01:09:39 +0900 Subject: [PATCH 6/9] fix(desktop): tray Start at Login shows only what Startup safety verifies (#6996) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Squashed from #6996 for the 2.84.0 release train. On macOS the Desktop tray's **Start at Login** checkbox read only whether `~/Library/LaunchAgents/OpenCodex.plist` exists (tauri-plugin-autostart `is_enabled`). Startup safety (`macLoginRegistration` in `src/service/desktop-startup.ts`) also requires launchd not to disable the job and the registered or loaded program to be this app copy. So a job turned off under System Settings → Login Items, another app copy's registration, or a job loaded from an old copy showed a **checked** tray item while the Startup page said Start at Login was not verified, and clicking rewrote the plist without clearing launchd's override. Now the tray uses the same judgment: - `desktop/src-tauri/src/login_registration.rs` ports `macLoginRegistration` step for step (plist shape, `launchctl print-disabled` override, registered app == this running copy, loaded `program`/`path`, or the exact service-not-found answer with an unchanged re-read, 750 ms per command, `access(X_OK)`, JS whitespace and line-terminator semantics). - The item is **checked only for a verified registration**, including a valid one that is not loaded until next login. Every other state is unchecked with the reason in the label: | State | Label | Click | |---|---|---| | verified | Start at Login ✓ | turns it off | | no plist | Start at Login | registers this copy | | launchd disabled this copy | Start at Login (disabled by macOS) | opens Login Items settings | | another copy registered or loaded | Start at Login (another OpenCodex copy) | disabled | | registered app path missing | Start at Login (registered app is missing) | disabled | | unreadable / malformed / timed out / changed while read | Start at Login (status unknown) | disabled | Closes #6972 --- desktop/src-tauri/src/first_run.rs | 22 + desktop/src-tauri/src/lib.rs | 5 + desktop/src-tauri/src/login_registration.rs | 1044 +++++++++++++++ desktop/src-tauri/src/tray.rs | 35 +- desktop/src-tauri/src/tray_login.rs | 163 +++ .../000_plan.md | 228 ++++ .../010_publish.md | 18 + .../src/content/docs/guides/desktop-app.md | 11 +- .../src/content/docs/guides/macos-menu-bar.md | 2 +- .../src/content/docs/ko/guides/desktop-app.md | 10 +- scripts/test-layout/layout.json | 2 +- src/service/desktop-startup.ts | 19 +- structure/INDEX.md | 5 + structure/desktop-shell.md | 3 + structure/desktop-start-at-login.md | 34 + structure/manifest.json | 12 + .../desktop-login-registration-cases.json | 1136 +++++++++++++++++ tests/fixtures/test-layout-expected.json | 2 +- ...service-desktop-login-registration.test.ts | 111 ++ 19 files changed, 2841 insertions(+), 21 deletions(-) create mode 100644 desktop/src-tauri/src/login_registration.rs create mode 100644 desktop/src-tauri/src/tray_login.rs create mode 100644 devlog/_plan/261012_6972_tray_start_at_login/000_plan.md create mode 100644 devlog/_plan/261012_6972_tray_start_at_login/010_publish.md create mode 100644 structure/desktop-start-at-login.md create mode 100644 tests/fixtures/desktop-login-registration-cases.json create mode 100644 tests/service/service-desktop-login-registration.test.ts diff --git a/desktop/src-tauri/src/first_run.rs b/desktop/src-tauri/src/first_run.rs index cf0c5b9e87c..9f1532e79c3 100644 --- a/desktop/src-tauri/src/first_run.rs +++ b/desktop/src-tauri/src/first_run.rs @@ -85,6 +85,11 @@ pub fn apply_start_at_login_default(app: &AppHandle) -> StartAtLogin { /// It runs once, behind its own marker, and only where autostart is already on. It never turns the /// setting on and never turns it off; the worst case is an entry that keeps its old arguments and a /// login launch that shows its window, which is the visible failure rather than the silent one. +/// +/// On macOS the marker lives in a config directory every copy of the app shares, while the plugin +/// rewrites whatever plist exists. The rewrite is therefore limited to this copy's own entry in the +/// exact shape the plugin writes; another or a missing copy, or any key the rewrite would drop, is +/// left untouched and unmarked (#6972). pub fn adopt_launch_origin_argument(app: &AppHandle) { let Ok(dir) = app.path().app_config_dir() else { return; @@ -102,6 +107,23 @@ pub fn adopt_launch_origin_argument(app: &AppHandle) { // that is already enabled, so retrying after a transient registry, LaunchAgent or desktop-file // error is free — and claiming the marker first would suppress the migration permanently and // leave a login launch showing its window forever. + #[cfg(target_os = "macos")] + { + use crate::login_registration::Migration; + let Ok(home) = app.path().home_dir() else { + return; + }; + match crate::login_registration::judge_migration(&home) { + Migration::Rewrite => { + if crate::login_autostart::enable(app).is_err() { + return; + } + } + Migration::Absent => {} + Migration::Leave => return, + } + } + #[cfg(not(target_os = "macos"))] match app.autolaunch().is_enabled() { Ok(true) => { if crate::login_autostart::enable(app).is_err() { diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rs index 06c1fa29540..d4e3dec7e32 100644 --- a/desktop/src-tauri/src/lib.rs +++ b/desktop/src-tauri/src/lib.rs @@ -15,6 +15,9 @@ mod formatting; mod identity; mod logging; mod login_autostart; +// The judgment is portable so Linux CI runs its fixture parity tests; only macOS reads launchd. +#[cfg(any(target_os = "macos", test))] +mod login_registration; #[cfg(any(target_os = "windows", test))] mod windows_autostart_command; // macOS only: it exists to replace one item in a menu no other platform installs. Compiling it @@ -35,6 +38,8 @@ mod popup; mod popup; #[cfg(target_os = "macos")] mod provider_icons; +#[cfg(target_os = "macos")] +mod tray_login; // The macOS build selects native_tray.rs as the popup module; compile the portable popup // module's tests on macOS too so its navigation rules run on the maintainers' platform. #[cfg(all(test, target_os = "macos"))] diff --git a/desktop/src-tauri/src/login_registration.rs b/desktop/src-tauri/src/login_registration.rs new file mode 100644 index 00000000000..2461254c5eb --- /dev/null +++ b/desktop/src-tauri/src/login_registration.rs @@ -0,0 +1,1044 @@ +//! Start at Login on macOS, judged the way Startup safety judges it. +//! +//! The autostart plugin answers "is Start at Login on?" with whether +//! `~/Library/LaunchAgents/OpenCodex.plist` exists. Startup safety +//! (`src/service/desktop-startup.ts`, `macLoginRegistration`) asks more: the plist must have the +//! shape this app writes, launchd must not have the job disabled, the registered app must be this +//! copy, and a loaded job must have been loaded from this copy and this plist. When the two +//! disagreed the tray showed a checked item while the Startup page reported Start at Login as +//! unverified (#6972). +//! +//! [`classify`] is a step-for-step port of that judgment over an injected [`Probe`], so the tray +//! can be tested without launchd. The cases in `tests/fixtures/desktop-login-registration-cases.json` +//! are run against both implementations; a change to either side that is not mirrored in the other +//! fails one of the two suites. Only [`LoginState::On`] is shown checked: every other state is a +//! reason the registration could not be credited, and none of them may read as on. + +use serde_json::Value; +use std::io; +use std::path::{Path, PathBuf}; + +const SERVICE: &str = "OpenCodex"; +const APP_SUFFIX: &str = "/Contents/MacOS/opencodex-desktop"; +const AUTOSTART_ARGUMENT: &str = "--autostart"; +/// System Settings → General → Login Items & Extensions, where a disabled job is turned back on. +#[cfg(target_os = "macos")] +pub const LOGIN_ITEMS_SETTINGS: &str = + "x-apple.systempreferences:com.apple.LoginItems-Settings.extension"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum LoginState { + /// This copy's registration is enabled and either loaded from this copy or valid for the next + /// login. The only state Startup safety credits, and the only one shown checked. + On, + /// No LaunchAgent plist. + Off, + /// This copy is registered but launchd has the job disabled (System Settings or + /// `launchctl disable`). Rewriting the plist does not clear that, so the item opens Login + /// Items settings instead. + DisabledBySystem, + /// The registration, or the job launchd loaded, belongs to another OpenCodex copy. The tray + /// never rewrites it. + OtherCopy, + /// The registration names an app path that does not exist. It is still not this copy's. + MissingCopy, + /// The evidence could not be read, was malformed, timed out or changed while it was read. + Unknown, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ClickAction { + Disable, + Enable, + OpenLoginItems, + Nothing, +} + +impl LoginState { + pub fn checked(self) -> bool { + self == Self::On + } + + pub fn label(self) -> &'static str { + match self { + Self::On | Self::Off => "Start at Login", + Self::DisabledBySystem => "Start at Login (disabled by macOS)", + Self::OtherCopy => "Start at Login (another OpenCodex copy)", + Self::MissingCopy => "Start at Login (registered app is missing)", + Self::Unknown => "Start at Login (status unknown)", + } + } + + pub fn click_action(self) -> ClickAction { + match self { + Self::On => ClickAction::Disable, + Self::Off => ClickAction::Enable, + Self::DisabledBySystem => ClickAction::OpenLoginItems, + // A missing path may be a copy being replaced or on a volume that is not mounted, so it + // is no more this copy's to take over than an existing one. + Self::OtherCopy | Self::MissingCopy | Self::Unknown => ClickAction::Nothing, + } + } + + pub fn clickable(self) -> bool { + self.click_action() != ClickAction::Nothing + } +} + +/// A failed command. `Exit` is an ordinary non-zero exit; everything else (spawn failure, +/// timeout, signal, oversized or non-UTF-8 output) is `Failed`, which never counts as absence. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum RunError { + Exit { status: i32, stderr: String }, + Failed, +} + +/// The filesystem and process reads the judgment needs, injectable for tests. +pub trait Probe { + /// An identity of the path itself (not its target) that changes when it is replaced or edited. + fn identity(&self, path: &Path) -> io::Result; + fn realpath(&self, path: &Path) -> io::Result; + fn executable(&self, path: &Path) -> io::Result<()>; + /// Trimmed stdout of a successful run. + fn run(&self, program: &str, args: &[&str]) -> Result; +} + +/// JavaScript's `\s`, so trimming and the regex ports below match the TypeScript judgment. +fn js_space(c: char) -> bool { + matches!( + c, + '\t' | '\n' | '\u{0b}' | '\u{0c}' | '\r' | ' ' | '\u{a0}' | '\u{1680}' | '\u{2000}' + ..='\u{200a}' + | '\u{2028}' + | '\u{2029}' + | '\u{202f}' + | '\u{205f}' + | '\u{3000}' + | '\u{feff}' + ) +} + +pub fn js_trim(text: &str) -> &str { + text.trim_matches(js_space) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Enabled { + Yes, + Disabled, + Unknown, +} + +/// `/"OpenCodex"\s*=>\s*(enabled|false|disabled|true)(?=\s*(?:\r?\n|[,;}]|$))/`, first match. +fn override_value(text: &str) -> Option<&'static str> { + let needle = "\"OpenCodex\""; + let mut from = 0; + while let Some(found) = text[from..].find(needle) { + let start = from + found; + from = start + needle.len(); + let rest = text[from..].trim_start_matches(js_space); + let Some(rest) = rest.strip_prefix("=>") else { + continue; + }; + let rest = rest.trim_start_matches(js_space); + for word in ["enabled", "false", "disabled", "true"] { + let Some(after) = rest.strip_prefix(word) else { + continue; + }; + let spaces = after.len() - after.trim_start_matches(js_space).len(); + let (gap, next) = after.split_at(spaces); + if gap.contains('\n') || next.is_empty() || next.starts_with([',', ';', '}']) { + return Some(word); + } + } + } + None +} + +/// The print-disabled half of `loginEnabled` in `macLoginRegistration`. +fn enabled(disabled: &str, plist_disabled: Option) -> Enabled { + let body = js_trim(disabled); + if !(body.starts_with("disabled services = {") && body.ends_with('}')) { + return Enabled::Unknown; + } + match disabled.matches("\"OpenCodex\"").count() { + // launchd's override is absent, so the plist's own Disabled key decides. auto-launch never + // writes one; a plist that disables itself was edited by something else. + 0 if plist_disabled == Some(true) => Enabled::Unknown, + 0 => Enabled::Yes, + 1 => match override_value(disabled) { + Some("enabled" | "false") => Enabled::Yes, + Some(_) => Enabled::Disabled, + None => Enabled::Unknown, + }, + _ => Enabled::Unknown, + } +} + +/// ECMAScript line terminators: what `^`, `$` and `.` honour under the `m` flag. +fn js_line_end(c: char) -> bool { + matches!(c, '\n' | '\r' | '\u{2028}' | '\u{2029}') +} + +/// `/^\s*key(.+)$/m`, first match. +fn line_value<'a>(text: &'a str, key: &str) -> Option<&'a str> { + text.split(js_line_end).find_map(|line| { + line.trim_start_matches(js_space) + .strip_prefix(key) + .filter(|value| !value.is_empty()) + }) +} + +/// The registered program when the plist has the shape auto-launch writes for this app. +fn registered_app(plist: &Value) -> Option<(&str, Option)> { + let args = plist.get("ProgramArguments")?.as_array()?; + let app = args.first()?.as_str()?; + let disabled = match plist.get("Disabled") { + None => None, + Some(Value::Bool(value)) => Some(*value), + Some(_) => return None, + }; + let shaped = plist.get("Label").and_then(Value::as_str) == Some(SERVICE) + && plist.get("RunAtLoad") == Some(&Value::Bool(true)) + && args.len() == 2 + && args[1].as_str() == Some(AUTOSTART_ARGUMENT) + && app.ends_with(APP_SUFFIX) + && plist + .get("Program") + .is_none_or(|program| program.as_str() == Some(app)); + shaped.then_some((app, disabled)) +} + +#[derive(Debug, PartialEq, Eq)] +struct Capture { + app: PathBuf, + proxy: PathBuf, + registration: PathBuf, + plist_identity: String, + target_identity: String, + definition: String, + app_identity: String, + proxy_identity: String, + disabled: String, + enabled: Enabled, +} + +fn unknown(_: T) -> LoginState { + LoginState::Unknown +} + +fn capture( + probe: &dyn Probe, + plist: &Path, + domain: &str, + current_app: &Path, +) -> Result { + let plist_identity = probe.identity(plist).map_err(|error| { + if error.kind() == io::ErrorKind::NotFound { + LoginState::Off + } else { + LoginState::Unknown + } + })?; + let registration = probe.realpath(plist).map_err(unknown)?; + let target_identity = probe.identity(®istration).map_err(unknown)?; + let plist_arg = plist.to_str().ok_or(LoginState::Unknown)?; + let definition = probe + .run( + "/usr/bin/plutil", + &["-convert", "json", "-o", "-", plist_arg], + ) + .map_err(unknown)?; + let parsed: Value = serde_json::from_str(&definition).map_err(unknown)?; + let (registered, plist_disabled) = registered_app(&parsed).ok_or(LoginState::Unknown)?; + let registered = Path::new(registered); + let app = probe.realpath(registered).map_err(|error| { + if error.kind() == io::ErrorKind::NotFound { + LoginState::MissingCopy + } else { + LoginState::Unknown + } + })?; + if app != current_app { + return Err(LoginState::OtherCopy); + } + let sibling = app.parent().ok_or(LoginState::Unknown)?.join("ocx"); + let proxy = probe.realpath(&sibling).map_err(unknown)?; + probe.executable(&app).map_err(unknown)?; + probe.executable(&proxy).map_err(unknown)?; + let app_identity = probe.identity(&app).map_err(unknown)?; + let proxy_identity = probe.identity(&proxy).map_err(unknown)?; + let disabled = probe + .run("/bin/launchctl", &["print-disabled", domain]) + .map_err(unknown)?; + let enabled = enabled(&disabled, plist_disabled); + // Every path and identity is read again: a registration replaced during the probes is not the + // registration that was judged. + let stable = probe.realpath(plist).ok().as_ref() == Some(®istration) + && probe.identity(plist).ok().as_ref() == Some(&plist_identity) + && probe.identity(®istration).ok().as_ref() == Some(&target_identity) + && probe.realpath(registered).ok().as_ref() == Some(&app) + && probe.realpath(&sibling).ok().as_ref() == Some(&proxy) + && probe.identity(&app).ok().as_ref() == Some(&app_identity) + && probe.identity(&proxy).ok().as_ref() == Some(&proxy_identity); + if !stable { + return Err(LoginState::Unknown); + } + Ok(Capture { + app, + proxy, + registration, + plist_identity, + target_identity, + definition, + app_identity, + proxy_identity, + disabled, + enabled, + }) +} + +/// Judge this copy's Start at Login registration. `current_app` must already be resolved. +pub fn classify(probe: &dyn Probe, home: &Path, uid: u32, current_app: &Path) -> LoginState { + let plist = home + .join("Library") + .join("LaunchAgents") + .join(format!("{SERVICE}.plist")); + let domain = format!("gui/{uid}"); + let first = match capture(probe, &plist, &domain, current_app) { + Ok(first) => first, + Err(state) => return state, + }; + match first.enabled { + Enabled::Yes => {} + Enabled::Disabled => return LoginState::DisabledBySystem, + Enabled::Unknown => return LoginState::Unknown, + } + let target = format!("{domain}/{SERVICE}"); + match probe.run("/bin/launchctl", &["print", &target]) { + Ok(loaded) => { + let Some(program) = line_value(&loaded, "program = ") else { + return LoginState::Unknown; + }; + match probe.realpath(Path::new(program)) { + Ok(program) if program == first.app => {} + Ok(_) => return LoginState::OtherCopy, + Err(error) if error.kind() == io::ErrorKind::NotFound => { + return LoginState::OtherCopy + } + Err(_) => return LoginState::Unknown, + } + let Some(path) = line_value(&loaded, "path = ") else { + return LoginState::Unknown; + }; + match probe.realpath(Path::new(path)) { + Ok(path) if path == first.registration => LoginState::On, + _ => LoginState::Unknown, + } + } + // launchctl's own service-not-found answer is absence; any other failure is unverified. + Err(RunError::Exit { + status: 113, + stderr, + }) => { + let missing = + format!("Could not find service \"{SERVICE}\" in domain for user gui: {uid}"); + let stderr = js_trim(&stderr); + if stderr != missing && stderr != format!("Bad request.\n{missing}") { + return LoginState::Unknown; + } + // An unloaded job leaves no durable evidence, so the whole registration is read again + // and must not have changed. + match capture(probe, &plist, &domain, current_app) { + Ok(last) if last == first => LoginState::On, + _ => LoginState::Unknown, + } + } + Err(_) => LoginState::Unknown, + } +} + +/// What the once-per-installation launch-origin migration in `first_run.rs` may do. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Migration { + /// No LaunchAgent: nothing to migrate. + Absent, + /// This copy's own enabled entry, in the shape auto-launch writes with or without + /// `--autostart`: rewriting it changes only the arguments. + Rewrite, + /// Anything else: another or a missing copy, a `Program` override, a definition that disables + /// itself, an edited shape or unreadable evidence. Rewriting would change more than arguments. + Leave, +} + +pub fn migration(probe: &dyn Probe, home: &Path, current_app: &Path) -> Migration { + let plist = home + .join("Library") + .join("LaunchAgents") + .join(format!("{SERVICE}.plist")); + let plist_identity = match probe.identity(&plist) { + Ok(identity) => identity, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Migration::Absent, + Err(_) => return Migration::Leave, + }; + let Ok(registration) = probe.realpath(&plist) else { + return Migration::Leave; + }; + let Ok(target_identity) = probe.identity(®istration) else { + return Migration::Leave; + }; + let Some(plist_arg) = plist.to_str() else { + return Migration::Leave; + }; + let Ok(definition) = probe.run( + "/usr/bin/plutil", + &["-convert", "json", "-o", "-", plist_arg], + ) else { + return Migration::Leave; + }; + let Ok(parsed) = serde_json::from_str::(&definition) else { + return Migration::Leave; + }; + let Some(args) = parsed.get("ProgramArguments").and_then(Value::as_array) else { + return Migration::Leave; + }; + let Some(app) = args.first().and_then(Value::as_str) else { + return Migration::Leave; + }; + let arguments = match args.len() { + 1 => true, + 2 => args[1].as_str() == Some(AUTOSTART_ARGUMENT), + _ => false, + }; + // The plugin rewrites the whole file with exactly these three keys, so any other key — + // `Program`, `Disabled`, `KeepAlive`, environment — would be dropped by the rewrite. + let keys = parsed.as_object().is_some_and(|plist| { + plist.len() == 3 + && ["Label", "ProgramArguments", "RunAtLoad"] + .iter() + .all(|key| plist.contains_key(*key)) + }); + let own_shape = arguments + && keys + && parsed.get("Label").and_then(Value::as_str) == Some(SERVICE) + && parsed.get("RunAtLoad") == Some(&Value::Bool(true)) + && app.ends_with(APP_SUFFIX); + if !own_shape { + return Migration::Leave; + } + let Ok(resolved) = probe.realpath(Path::new(app)) else { + return Migration::Leave; + }; + // As in [`classify`]: a plist replaced while it was read is not the plist that was judged. + let stable = probe.identity(&plist).ok().as_ref() == Some(&plist_identity) + && probe.realpath(&plist).ok().as_ref() == Some(®istration) + && probe.identity(®istration).ok().as_ref() == Some(&target_identity) + && probe.realpath(Path::new(app)).ok().as_ref() == Some(&resolved); + if stable && resolved == current_app { + Migration::Rewrite + } else { + Migration::Leave + } +} + +/// Running launchctl and plutil with a deadline, without ever killing a process. +/// +/// The shell never kills a process (`tests/clients/desktop-exit-ownership.test.ts`), so a +/// command that outlives its deadline is abandoned: the caller gets [`RunError::Failed`] at the +/// deadline and a detached thread reaps the child and its output readers later. One process-wide +/// slot is held until all three have finished, and no command starts while it is held, so a +/// launchd that never answers costs at most one abandoned command however often the tray asks. +#[cfg(any(target_os = "macos", all(test, unix)))] +mod command { + use super::{js_trim, RunError}; + use std::io::Read; + use std::process::{Child, Command, Stdio}; + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::mpsc::{channel, Receiver}; + use std::thread::JoinHandle; + use std::time::{Duration, Instant}; + + const MAX_OUTPUT: u64 = 128 * 1024; + static BUSY: AtomicBool = AtomicBool::new(false); + + /// Occupancy of the one command slot; released when dropped. + struct Slot; + + impl Slot { + fn take() -> Option { + BUSY.compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .ok() + .map(|_| Self) + } + } + + impl Drop for Slot { + fn drop(&mut self) { + BUSY.store(false, Ordering::Release); + } + } + + type Reader = (Receiver>, JoinHandle<()>); + + fn drain(stream: Option) -> Reader { + let (sender, receiver) = channel(); + let handle = std::thread::spawn(move || { + let mut bytes = Vec::new(); + if let Some(stream) = stream { + let _ = stream.take(MAX_OUTPUT + 1).read_to_end(&mut bytes); + } + let _ = sender.send(bytes); + }); + (receiver, handle) + } + + /// Hand the slot to a thread that waits for the child and both readers, then frees it. + fn abandon(slot: Slot, mut child: Child, readers: [Reader; 2]) -> RunError { + std::thread::spawn(move || { + let _ = child.wait(); + for (_, handle) in readers { + let _ = handle.join(); + } + drop(slot); + }); + RunError::Failed + } + + pub(super) fn run(program: &str, args: &[&str], timeout: Duration) -> Result { + let slot = Slot::take().ok_or(RunError::Failed)?; + let deadline = Instant::now() + timeout; + let mut child = Command::new(program) + .args(args) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .map_err(|_| RunError::Failed)?; + let readers = [drain(child.stdout.take()), drain(child.stderr.take())]; + let status = loop { + match child.try_wait() { + Ok(Some(status)) => break status, + Ok(None) if Instant::now() < deadline => { + std::thread::sleep(Duration::from_millis(5)) + } + _ => return Err(abandon(slot, child, readers)), + } + }; + let mut output = Vec::new(); + for (receiver, _) in &readers { + let left = deadline.saturating_duration_since(Instant::now()); + match receiver.recv_timeout(left) { + Ok(bytes) => output.push(bytes), + Err(_) => return Err(abandon(slot, child, readers)), + } + } + for (_, handle) in readers { + let _ = handle.join(); + } + drop(slot); + let [stdout, stderr] = [&output[0], &output[1]]; + if stdout.len() as u64 > MAX_OUTPUT || stderr.len() as u64 > MAX_OUTPUT { + return Err(RunError::Failed); + } + // Node decodes `encoding: "utf8"` output lossily; so does this. + if status.success() { + return Ok(js_trim(&String::from_utf8_lossy(stdout)).to_string()); + } + match status.code() { + Some(status) => Err(RunError::Exit { + status, + stderr: String::from_utf8_lossy(stderr).into_owned(), + }), + None => Err(RunError::Failed), + } + } + + #[cfg(test)] + mod tests { + use super::*; + + #[test] + fn a_command_past_its_deadline_holds_the_only_slot_until_it_ends() { + let started = Instant::now(); + assert_eq!( + run("/bin/sleep", &["1"], Duration::from_millis(50)), + Err(RunError::Failed) + ); + assert!(started.elapsed() < Duration::from_millis(500)); + // Repeated timeouts cannot add work: nothing runs while the abandoned command lives. + for _ in 0..20 { + assert_eq!( + run("/bin/echo", &["ready"], Duration::from_secs(5)), + Err(RunError::Failed) + ); + } + let freed = Instant::now() + Duration::from_secs(10); + while BUSY.load(Ordering::Acquire) { + assert!( + Instant::now() < freed, + "the abandoned command was never reaped" + ); + std::thread::sleep(Duration::from_millis(20)); + } + assert_eq!( + run("/bin/echo", &["ready"], Duration::from_secs(5)), + Ok("ready".into()) + ); + assert_eq!( + run( + "/bin/sh", + &["-c", "echo nope >&2; exit 113"], + Duration::from_secs(5) + ), + Err(RunError::Exit { + status: 113, + stderr: "nope\n".into() + }) + ); + } + } +} + +#[cfg(target_os = "macos")] +pub use system::{judge, judge_migration}; + +#[cfg(target_os = "macos")] +mod system { + use super::*; + use std::ffi::CString; + use std::os::raw::c_char; + use std::os::unix::ffi::OsStrExt; + use std::os::unix::fs::MetadataExt; + use std::time::Duration; + + /// TypeScript's `run` deadline in `desktop-startup.ts`. + const TIMEOUT: Duration = Duration::from_millis(750); + const X_OK: i32 = 1; + + unsafe extern "C" { + fn getuid() -> u32; + fn access(path: *const c_char, mode: i32) -> i32; + } + + pub struct SystemProbe; + + impl Probe for SystemProbe { + fn identity(&self, path: &Path) -> io::Result { + let meta = std::fs::symlink_metadata(path)?; + Ok(format!( + "{}:{}:{}:{}:{}.{}:{}.{}", + meta.dev(), + meta.ino(), + meta.size(), + meta.mode(), + meta.mtime(), + meta.mtime_nsec(), + meta.ctime(), + meta.ctime_nsec() + )) + } + + fn realpath(&self, path: &Path) -> io::Result { + std::fs::canonicalize(path) + } + + fn executable(&self, path: &Path) -> io::Result<()> { + let path = CString::new(path.as_os_str().as_bytes()) + .map_err(|_| io::Error::from(io::ErrorKind::InvalidInput))?; + // SAFETY: `path` is a NUL-terminated string that outlives the call. + if unsafe { access(path.as_ptr(), X_OK) } == 0 { + Ok(()) + } else { + Err(io::Error::last_os_error()) + } + } + + fn run(&self, program: &str, args: &[&str]) -> Result { + command::run(program, args, TIMEOUT) + } + } + + fn current_app() -> Option { + std::fs::canonicalize(std::env::current_exe().ok()?).ok() + } + + /// Decide the launch-origin migration against the live LaunchAgent. Blocking. + pub fn judge_migration(home: &Path) -> Migration { + match current_app() { + Some(app) => migration(&SystemProbe, home, &app), + None => Migration::Leave, + } + } + + /// Read the live registration. Blocking: run off the main thread. + pub fn judge(home: &Path) -> LoginState { + let Some(app) = current_app() else { + return LoginState::Unknown; + }; + // SAFETY: getuid has no preconditions and cannot fail. + let uid = unsafe { getuid() }; + classify(&SystemProbe, home, uid, &app) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::cell::Cell; + use std::collections::{BTreeMap, BTreeSet}; + + const CASES: &str = + include_str!("../../../tests/fixtures/desktop-login-registration-cases.json"); + const HOME: &str = "/fixture/home"; + const APP: &str = "/fixture/OpenCodex.app/Contents/MacOS/opencodex-desktop"; + const OTHER_APP: &str = "/fixture/Other/OpenCodex.app/Contents/MacOS/opencodex-desktop"; + const MISSING_APP: &str = "/fixture/Gone/OpenCodex.app/Contents/MacOS/opencodex-desktop"; + const PLIST: &str = "/fixture/home/Library/LaunchAgents/OpenCodex.plist"; + const REAL_PLIST: &str = "/fixture/home/Real.plist"; + const OTHER_PLIST: &str = "/fixture/home/Other.plist"; + + fn sibling(app: &str) -> String { + format!("{}/ocx", app.rsplit_once('/').unwrap().0) + } + + struct Fake { + files: BTreeSet, + links: BTreeMap, + plist: Option, + disabled: Option, + disabled_after: Option, + loaded: Result, + touch: bool, + disabled_reads: Cell, + plutil_calls: Cell, + } + + impl Fake { + fn new(plist: Option) -> Self { + let mut files: BTreeSet = [APP, OTHER_APP, OTHER_PLIST] + .into_iter() + .map(String::from) + .collect(); + files.insert(sibling(APP)); + files.insert(sibling(OTHER_APP)); + if plist.is_some() { + files.insert(PLIST.into()); + } + Self { + files, + links: BTreeMap::new(), + plist, + disabled: Some("disabled services = {\n}".into()), + disabled_after: None, + loaded: Err(RunError::Failed), + touch: false, + disabled_reads: Cell::new(0), + plutil_calls: Cell::new(0), + } + } + + fn known(&self, path: &Path) -> Option { + let path = path.to_str()?.to_string(); + (self.files.contains(&path) || self.links.contains_key(&path)).then_some(path) + } + } + + fn not_found() -> io::Error { + io::Error::from(io::ErrorKind::NotFound) + } + + impl Probe for Fake { + fn identity(&self, path: &Path) -> io::Result { + let path = self.known(path).ok_or_else(not_found)?; + let generation = if self.touch && path == PLIST { + self.plutil_calls.get() + } else { + 0 + }; + Ok(format!("{path}#{generation}")) + } + + fn realpath(&self, path: &Path) -> io::Result { + let path = self.known(path).ok_or_else(not_found)?; + Ok(PathBuf::from( + self.links.get(&path).cloned().unwrap_or(path), + )) + } + + fn executable(&self, path: &Path) -> io::Result<()> { + self.known(path).map(|_| ()).ok_or_else(not_found) + } + + fn run(&self, program: &str, args: &[&str]) -> Result { + match (program, args.first().copied()) { + ("/usr/bin/plutil", _) => { + self.plutil_calls.set(self.plutil_calls.get() + 1); + self.plist + .as_ref() + .map(Value::to_string) + .ok_or(RunError::Failed) + } + ("/bin/launchctl", Some("print-disabled")) => { + let reads = self.disabled_reads.get() + 1; + self.disabled_reads.set(reads); + let after = self.disabled_after.as_ref().filter(|_| reads > 1); + after + .or(self.disabled.as_ref()) + .cloned() + .ok_or(RunError::Failed) + } + ("/bin/launchctl", Some("print")) => self.loaded.clone(), + other => panic!("unexpected command {other:?}"), + } + } + } + + fn fill(text: &str) -> String { + text.replace("{APP}", APP) + .replace("{OTHER_APP}", OTHER_APP) + .replace("{MISSING_APP}", MISSING_APP) + .replace("{PLIST}", PLIST) + .replace("{OTHER_PLIST}", OTHER_PLIST) + .replace("{UID}", "501") + } + + fn fake_for(case: &Value) -> Fake { + let case: Value = serde_json::from_str(&fill(&case.to_string())).unwrap(); + let plist = case.get("plist").filter(|plist| !plist.is_null()).cloned(); + let present = plist.is_some(); + let mut fake = Fake::new(plist); + if case.get("plutilFails") == Some(&Value::Bool(true)) { + fake.plist = None; + } + if case.get("proxy") == Some(&Value::Bool(false)) { + fake.files.remove(&sibling(APP)); + } + if present && case.get("plistLink") == Some(&Value::Bool(true)) { + fake.files.remove(PLIST); + fake.files.insert(REAL_PLIST.into()); + fake.links.insert(PLIST.into(), REAL_PLIST.into()); + } + fake.touch = case.get("touchPlistOnRead") == Some(&Value::Bool(true)); + fake.disabled = case["disabled"].as_str().map(String::from); + fake.disabled_after = case + .get("disabledAfterFirstRead") + .and_then(Value::as_str) + .map(String::from); + let loaded = &case["loaded"]; + fake.loaded = if let Some(stdout) = loaded.get("stdout").and_then(Value::as_str) { + Ok(stdout.into()) + } else if let (Some(status), None) = (loaded.get("status"), loaded.get("signal")) { + Err(RunError::Exit { + status: status.as_i64().unwrap() as i32, + stderr: loaded["stderr"].as_str().unwrap().into(), + }) + } else { + Err(RunError::Failed) + }; + fake + } + + fn name(state: LoginState) -> &'static str { + match state { + LoginState::On => "on", + LoginState::Off => "off", + LoginState::DisabledBySystem => "disabled-by-system", + LoginState::OtherCopy => "other-copy", + LoginState::MissingCopy => "missing-copy", + LoginState::Unknown => "unknown", + } + } + + #[test] + fn shared_fixture_matches_startup_safety() { + let fixture: Value = serde_json::from_str(CASES).unwrap(); + let cases = fixture["cases"].as_array().unwrap(); + assert!(cases.len() > 40); + for case in cases { + let label = case["name"].as_str().unwrap(); + let fake = fake_for(case); + let state = classify(&fake, Path::new(HOME), 501, Path::new(APP)); + assert_eq!(name(state), case["expect"]["tray"], "{label}"); + assert_eq!( + Some(state.checked()), + case["expect"]["loginEnabled"].as_bool(), + "{label}" + ); + } + } + + #[test] + fn only_a_verified_registration_is_checked_or_rewritable() { + let table = [ + (LoginState::On, true, ClickAction::Disable, "Start at Login"), + ( + LoginState::Off, + false, + ClickAction::Enable, + "Start at Login", + ), + ( + LoginState::DisabledBySystem, + false, + ClickAction::OpenLoginItems, + "Start at Login (disabled by macOS)", + ), + ( + LoginState::OtherCopy, + false, + ClickAction::Nothing, + "Start at Login (another OpenCodex copy)", + ), + ( + LoginState::MissingCopy, + false, + ClickAction::Nothing, + "Start at Login (registered app is missing)", + ), + ( + LoginState::Unknown, + false, + ClickAction::Nothing, + "Start at Login (status unknown)", + ), + ]; + for (state, checked, action, label) in table { + assert_eq!(state.checked(), checked, "{state:?}"); + assert_eq!(state.click_action(), action, "{state:?}"); + assert_eq!( + state.clickable(), + action != ClickAction::Nothing, + "{state:?}" + ); + assert_eq!(state.label(), label, "{state:?}"); + } + } + + #[test] + fn migration_rewrites_only_this_copys_own_enabled_entry() { + let entry = |extra: Value| { + let mut plist = serde_json::json!({ + "Label": "OpenCodex", + "RunAtLoad": true, + "ProgramArguments": [APP], + }); + for (key, value) in extra.as_object().unwrap() { + plist[key] = value.clone(); + } + Some(plist) + }; + let cases = [ + ("absent", None, Migration::Absent), + ( + "legacy entry", + entry(serde_json::json!({})), + Migration::Rewrite, + ), + ( + "current arguments", + entry(serde_json::json!({ "ProgramArguments": [APP, "--autostart"] })), + Migration::Rewrite, + ), + ( + "Program equal", + entry(serde_json::json!({ "Program": APP })), + Migration::Leave, + ), + ( + "Disabled false", + entry(serde_json::json!({ "Disabled": false })), + Migration::Leave, + ), + ( + "a key the rewrite would drop", + entry(serde_json::json!({ "KeepAlive": true })), + Migration::Leave, + ), + ( + "another copy", + entry(serde_json::json!({ "ProgramArguments": [OTHER_APP] })), + Migration::Leave, + ), + ( + "missing copy", + entry(serde_json::json!({ "ProgramArguments": [MISSING_APP, "--autostart"] })), + Migration::Leave, + ), + ( + "Program names another copy", + entry(serde_json::json!({ "Program": OTHER_APP })), + Migration::Leave, + ), + ( + "Program names a missing copy", + entry(serde_json::json!({ "Program": MISSING_APP })), + Migration::Leave, + ), + ( + "disabled by its own definition", + entry(serde_json::json!({ "Disabled": true })), + Migration::Leave, + ), + ( + "Disabled is not a boolean", + entry(serde_json::json!({ "Disabled": "yes" })), + Migration::Leave, + ), + ( + "unknown argument", + entry(serde_json::json!({ "ProgramArguments": [APP, "--hidden"] })), + Migration::Leave, + ), + ( + "extra argument", + entry(serde_json::json!({ "ProgramArguments": [APP, "--autostart", "x"] })), + Migration::Leave, + ), + ( + "another label", + entry(serde_json::json!({ "Label": "com.example.OpenCodex" })), + Migration::Leave, + ), + ( + "RunAtLoad false", + entry(serde_json::json!({ "RunAtLoad": false })), + Migration::Leave, + ), + ]; + for (label, plist, expected) in cases { + let fake = Fake::new(plist); + assert_eq!( + migration(&fake, Path::new(HOME), Path::new(APP)), + expected, + "{label}" + ); + } + let mut unreadable = Fake::new(entry(serde_json::json!({}))); + unreadable.plist = None; + assert_eq!( + migration(&unreadable, Path::new(HOME), Path::new(APP)), + Migration::Leave + ); + // Another copy replacing the plist while it is read leaves it alone. + let mut replaced = Fake::new(entry(serde_json::json!({}))); + replaced.touch = true; + assert_eq!( + migration(&replaced, Path::new(HOME), Path::new(APP)), + Migration::Leave + ); + } + + #[test] + fn javascript_whitespace_and_line_terminators() { + assert_eq!(js_trim("\u{feff}\u{a0} x \u{3000}"), "x"); + assert_eq!(js_trim("\u{85}x"), "\u{85}x"); + assert_eq!( + line_value("a\u{2028}program = /x\u{2029}y", "program = "), + Some("/x") + ); + assert_eq!( + line_value("program = \rprogram = /y", "program = "), + Some("/y") + ); + } +} diff --git a/desktop/src-tauri/src/tray.rs b/desktop/src-tauri/src/tray.rs index e8e76697ec9..c702bc7c240 100644 --- a/desktop/src-tauri/src/tray.rs +++ b/desktop/src-tauri/src/tray.rs @@ -14,6 +14,7 @@ use tauri::{ tray::{MouseButton, MouseButtonState, TrayIconBuilder, TrayIconEvent}, AppHandle, Manager, Wry, }; +#[cfg(not(target_os = "macos"))] use tauri_plugin_autostart::ManagerExt; use tauri_plugin_opener::OpenerExt; @@ -28,6 +29,8 @@ pub struct TrayMenu { check_updates: MenuItem, install_update: MenuItem, stop: MenuItem, + #[cfg(target_os = "macos")] + pub(crate) login: CheckMenuItem, } impl Default for TrayState { @@ -79,6 +82,10 @@ pub fn install(app: &AppHandle) -> tauri::Result<()> { true, None::<&str>, )?; + // macOS shows only what Startup safety would verify, and probes it off the main thread. + #[cfg(target_os = "macos")] + let login = crate::tray_login::item(app)?; + #[cfg(not(target_os = "macos"))] let login = CheckMenuItem::with_id( app, "start-at-login", @@ -133,8 +140,12 @@ pub fn install(app: &AppHandle) -> tauri::Result<()> { check_updates: check_updates.clone(), install_update: install_update.clone(), stop: stop.clone(), + #[cfg(target_os = "macos")] + login: login.clone(), }); } + #[cfg(target_os = "macos")] + crate::tray_login::refresh(app); let builder = TrayIconBuilder::with_id("main") .icon(icon()) @@ -151,6 +162,11 @@ pub fn install(app: &AppHandle) -> tauri::Result<()> { let builder = builder.show_menu_on_left_click(false); let tray = builder .on_tray_icon_event(|tray, event| { + #[cfg(target_os = "macos")] + if let TrayIconEvent::Enter { .. } = event { + crate::tray_login::hover(tray.app_handle()); + return; + } if let TrayIconEvent::Click { button: MouseButton::Left, button_state: MouseButtonState::Up, @@ -213,11 +229,16 @@ pub fn install(app: &AppHandle) -> tauri::Result<()> { .open_url(format!("{}#/usage", endpoint.url("/")), None::); } "start-at-login" => { - let enabled = app.autolaunch().is_enabled().unwrap_or(false); - if enabled { - let _ = app.autolaunch().disable(); - } else { - let _ = crate::login_autostart::enable(app); + #[cfg(target_os = "macos")] + crate::tray_login::click(app); + #[cfg(not(target_os = "macos"))] + { + let enabled = app.autolaunch().is_enabled().unwrap_or(false); + if enabled { + let _ = app.autolaunch().disable(); + } else { + let _ = crate::login_autostart::enable(app); + } } } "stop-proxy" => { @@ -254,6 +275,8 @@ pub fn install(app: &AppHandle) -> tauri::Result<()> { let mut tick = 0; loop { tokio::time::sleep(std::time::Duration::from_secs(60)).await; + #[cfg(target_os = "macos")] + crate::tray_login::refresh(&app); let Some(proxy) = app .try_state::() .and_then(|state| state.proxy()) @@ -287,7 +310,7 @@ fn refresh(app: &AppHandle, tray: &tauri::tray::TrayIcon) { /// the main thread and takes this same mutex while doing so, so calling a setter with the lock held /// is a cycle: a background update owns the mutex and waits for the main thread, and the main /// thread waits for the mutex. The app would stop answering Quit. -fn menu_handles(app: &AppHandle) -> Option { +pub(crate) fn menu_handles(app: &AppHandle) -> Option { let state = app.try_state::()?; let handles = state.menu.lock().ok()?; handles.as_ref().cloned() diff --git a/desktop/src-tauri/src/tray_login.rs b/desktop/src-tauri/src/tray_login.rs new file mode 100644 index 00000000000..5126c1f44e7 --- /dev/null +++ b/desktop/src-tauri/src/tray_login.rs @@ -0,0 +1,163 @@ +//! The macOS tray's Start at Login item (#6972). +//! +//! The item shows what [`login_registration::judge`] verifies, the judgment Startup safety uses, +//! and is checked only for [`LoginState::On`]. Reading launchd runs subprocesses, so every probe +//! runs on a blocking worker; one mutex serializes probes with the mutation a click performs. +//! +//! Presentation is applied in one closure on the main thread, the thread that also delivers +//! clicks, so a click and a refresh can never interleave their setters. muda flips a check item +//! before the click reaches the app; the click handler puts back what was shown and holds the +//! item disabled until its own job answers, and refresh results that land meanwhile are dropped. + +use crate::login_registration::{self, ClickAction, LoginState}; +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Mutex, PoisonError, +}; +use std::time::{Duration, Instant}; +use tauri::{menu::CheckMenuItem, AppHandle, Manager, Wry}; +use tauri_plugin_autostart::ManagerExt; +use tauri_plugin_opener::OpenerExt; + +const HOVER_INTERVAL: Duration = Duration::from_secs(2); + +#[derive(Default)] +pub struct LoginTray { + /// The state the item shows; `None` until the first probe answers. + shown: Mutex>, + /// Clicks whose job has not applied its answer yet. + pending: AtomicUsize, + /// Serializes probes and the mutation a click performs. + probe: Mutex<()>, + /// When hovering last asked for a refresh. + hovered: Mutex>, +} + +/// The item as built, before anything has been verified: unchecked and inert. +pub fn item(app: &AppHandle) -> tauri::Result> { + app.manage(LoginTray::default()); + let state = LoginState::Unknown; + CheckMenuItem::with_id( + app, + "start-at-login", + state.label(), + state.clickable(), + state.checked(), + None::<&str>, + ) +} + +fn judge(app: &AppHandle) -> LoginState { + match app.path().home_dir() { + Ok(home) => login_registration::judge(&home), + Err(_) => LoginState::Unknown, + } +} + +/// Show `state`. A refresh (`click == false`) yields to any click still waiting for its answer. +fn present(app: &AppHandle, state: LoginState, click: bool) { + let handle = app.clone(); + let _ = app.run_on_main_thread(move || { + let Some(login) = handle.try_state::() else { + return; + }; + if click { + login.pending.fetch_sub(1, Ordering::AcqRel); + } else if login.pending.load(Ordering::Acquire) > 0 { + return; + } + *login.shown.lock().unwrap_or_else(PoisonError::into_inner) = Some(state); + let busy = login.pending.load(Ordering::Acquire) > 0; + if let Some(menu) = crate::tray::menu_handles(&handle) { + let _ = menu.login.set_text(state.label()); + let _ = menu.login.set_checked(state.checked()); + let _ = menu.login.set_enabled(state.clickable() && !busy); + } + }); +} + +/// Probe again unless a probe or a click is already running. +pub fn refresh(app: &AppHandle) { + let app = app.clone(); + tauri::async_runtime::spawn_blocking(move || { + let Some(login) = app.try_state::() else { + return; + }; + let Ok(_probe) = login.probe.try_lock() else { + return; + }; + let state = judge(&app); + present(&app, state, false); + }); +} + +/// The pointer entered the status item: the menu may be about to open. +pub fn hover(app: &AppHandle) { + let Some(login) = app.try_state::() else { + return; + }; + { + let mut hovered = login.hovered.lock().unwrap_or_else(PoisonError::into_inner); + let now = Instant::now(); + if hovered.is_some_and(|last| now.duration_since(last) < HOVER_INTERVAL) { + return; + } + *hovered = Some(now); + } + refresh(app); +} + +/// Handle a click on the item. Runs on the main thread and never waits for a probe. +pub fn click(app: &AppHandle) { + let Some(login) = app.try_state::() else { + return; + }; + let shown = login + .shown + .lock() + .unwrap_or_else(PoisonError::into_inner) + .unwrap_or(LoginState::Unknown); + login.pending.fetch_add(1, Ordering::AcqRel); + if let Some(menu) = crate::tray::menu_handles(app) { + let _ = menu.login.set_checked(shown.checked()); + let _ = menu.login.set_enabled(false); + } + let app = app.clone(); + tauri::async_runtime::spawn_blocking(move || { + let state = match app.try_state::() { + Some(login) => { + let _probe = login.probe.lock().unwrap_or_else(PoisonError::into_inner); + let fresh = judge(&app); + // The click answered what was shown. If the registration has changed since, the + // person has not seen the new state yet: show it and do nothing else. + if fresh == shown && fresh.click_action() != ClickAction::Nothing { + perform(&app, fresh.click_action()); + judge(&app) + } else { + fresh + } + } + None => LoginState::Unknown, + }; + present(&app, state, true); + }); +} + +fn perform(app: &AppHandle, action: ClickAction) { + let result = match action { + ClickAction::Disable => app + .autolaunch() + .disable() + .map_err(|error| error.to_string()), + ClickAction::Enable => crate::login_autostart::enable(app), + // launchd keeps its override whatever the plist says; the person turns it back on. + ClickAction::OpenLoginItems => app + .opener() + .open_url(login_registration::LOGIN_ITEMS_SETTINGS, None::) + .map_err(|error| error.to_string()), + ClickAction::Nothing => Ok(()), + }; + if let Err(error) = result { + crate::logging::log_once("start at login", &error); + } +} diff --git a/devlog/_plan/261012_6972_tray_start_at_login/000_plan.md b/devlog/_plan/261012_6972_tray_start_at_login/000_plan.md new file mode 100644 index 00000000000..a0f617280a0 --- /dev/null +++ b/devlog/_plan/261012_6972_tray_start_at_login/000_plan.md @@ -0,0 +1,228 @@ +# 6972 — Tray Start at Login uses the Startup safety judgment + +Issue: lidge-jun/opencodex#6972. Related: #6980 (Startup page label only), #6963. +Branch: `codex/6972-tray-start-at-login-launchd-261012`, target `dev`. + +## Problem + +On macOS the tray's **Start at Login** `CheckMenuItem` is built from +`app.autolaunch().is_enabled()` (tauri-plugin-autostart 2.5.0 → auto-launch 0.5.0), which +is `Path::exists(~/Library/LaunchAgents/OpenCodex.plist)`. Startup safety +(`src/service/desktop-startup.ts` `macLoginRegistration`) credits the registration only when: + +1. the plist has the auto-launch shape (`Label` OpenCodex, `RunAtLoad` true, + `ProgramArguments = [<…>/Contents/MacOS/opencodex-desktop, --autostart]`, `Program` + absent or equal, `Disabled` absent or boolean), app and sibling `ocx` resolve and are + executable; +2. `launchctl print-disabled gui/` is well formed and either does not mention + `"OpenCodex"` (then plist `Disabled !== true`) or mentions it exactly once as + `enabled`/`false`; +3. `launchctl print gui//OpenCodex` either names this app as `program` and the + registration plist as `path` (`loaded`), or fails with the exact service-not-found result + (status 113, exact stderr) and a full re-capture is unchanged (`registered-not-loaded`); +4. the registered app equals the app being judged (unowned supervision path: + `login.app === app`). + +So a launchd-disabled job, another app copy, or a loaded job from an old copy show a checked +tray item while Startup reports Start at Login unverified. Clicking writes the plist again, +which neither clears a launchd override nor respects another copy's registration. + +## Decision + +Port the judgment to Rust (`desktop/src-tauri/src/login_registration.rs`) as a pure +classifier over an injected probe, and prove parity with one shared fixture file read by both +a Rust unit test and a bun test that drives the TypeScript judgment. The tray checks the item +only for the verified state. The classifier subdivides "not verified" so the tray can explain +and choose a safe click action: + +| State | TS `loginEnabled` | Checked | Label | Clickable | Click action (after a fresh re-probe that still returns the same state) | +|---|---|---|---|---|---| +| `On` (loaded or registered-not-loaded, this copy) | true | yes | Start at Login | yes | plugin `disable()` (unchanged) | +| `Off` (plist absent) | false | no | Start at Login | yes | `login_autostart::enable` (unchanged) | +| `DisabledBySystem` (this copy; print-disabled override disabled/true) | false | no | Start at Login (off in System Settings) | yes | open `x-apple.systempreferences:com.apple.LoginItems-Settings.extension`; never rewrites the plist | +| `OtherCopy` (registered app or loaded program resolves to another existing copy) | false | no | Start at Login (another OpenCodex copy) | no | none | +| `MissingCopy` (registered app path does not exist) | false | no | Start at Login (registered app is missing) | yes | `login_autostart::enable` for this copy | +| `Unknown` (probe failure, timeout, malformed output, shape mismatch, plist Disabled=true without override, path mismatch, changed during probe, not yet probed) | false | no | Start at Login (status unknown) | no | none | + +Invariant: `checked ⇔ state == On ⇔ TS loginEnabled for this app`. Nothing but `On` is +checked; an unverified result can never become checked. + +Why the Rust port instead of asking the running proxy: the tray must work when no proxy is +attached or when the attached proxy is an older runtime, and the proxy's Startup diagnostics +judge the supervising app, which is not necessarily this copy. The shared fixture is the +drift guard. + +## Diff plan + +### `desktop/src-tauri/src/login_registration.rs` (new) + +- `pub enum LoginState { On, Off, DisabledBySystem, OtherCopy, MissingCopy, Unknown }` with + `checked()`, `label()`, `clickable()`, `click_action() -> ClickAction`. +- `pub trait Probe { identity(&Path) -> io::Result; realpath(&Path) -> io::Result; + executable(&Path) -> io::Result<()>; run(&str, &[&str]) -> Result }`, + `RunError::{Exit { status, stderr }, Failed}` (timeout, signal, spawn failure, oversize → + `Failed`). +- `pub fn classify(probe, home, uid, current_app) -> LoginState` mirroring + `macLoginRegistration` step for step: capture (realpath + identity of plist and target, + plutil JSON, shape, app/ocx realpath + X_OK + identity, print-disabled parse, re-check of + every path/identity), then loaded/not-loaded branch with full re-capture comparison. + Parsers mirror the TS regexes by hand (no new crate). +- `#[cfg(target_os = "macos")] pub struct SystemProbe`: `std::fs` realpath and + `MetadataExt` identity (dev, ino, size, mode, mtime ns, ctime ns), and + `std::process::Command` with a 750 ms deadline (try_wait poll, kill on timeout), 128 KiB + cap and trimmed UTF-8 stdout, matching TS `run`. `getuid` and `access(X_OK)` are declared + through `extern "C"` (precedent: `geteuid` in `cli_command_record.rs`), so executability + matches TS `accessSync(X_OK)` with no new dependency. +- `#[cfg(test)]` parity test reading `../../../tests/fixtures/desktop-login-registration-cases.json` + through `include_str!` with a fake probe; plus the invariant `checked == startupLoginEnabled`. + +### `desktop/src-tauri/src/tray.rs` + +- macOS: build the item unchecked with the `Unknown` label and disabled, store it in + `TrayMenu`, then spawn the first probe. Keep the plugin path on Linux/Windows unchanged. +- `"start-at-login"` on macOS: `spawn_blocking` → take the probe lock, re-probe; if the + fresh state differs from the state the menu displayed, apply it and stop; otherwise run the + state's click action, re-probe and apply. Always re-apply so muda's automatic check toggle + never survives. +- Refresh (try-lock, skip when busy): at install, `TrayIconEvent::Enter` (throttled to 2 s), + and the existing 60 s loop (before the "no proxy" `continue`). +- Setters run only on background threads and never while holding the menu mutex + (existing `menu_handles` rule). + +### `desktop/src-tauri/src/lib.rs` + +- `#[cfg(any(target_os = "macos", test))] mod login_registration;` + +### `src/service/desktop-startup.ts` + +- Extract the existing unowned-path lambda into + `export function macLoginFactsFor(app, deps)` (no behavior change) so the parity test + judges exactly what Startup safety uses for "this app". + +### Tests + +- `tests/fixtures/desktop-login-registration-cases.json` (shared cases, placeholders + `{APP}`, `{OTHER_APP}`, `{MISSING_APP}`, `{PLIST}`, `{OTHER_PLIST}`, `{UID}`). + Cases: loaded current; registered-not-loaded (both stderr forms); override enabled / + false; override disabled / true; plist absent; plist Disabled=true without override; + plist Disabled=true with enabled override; other copy registered; loaded job from other + copy; loaded path mismatch; registered app missing; ocx missing; plutil failure; shape + mismatches (no --autostart, wrong Label, wrong suffix); print-disabled failure, malformed, + duplicate mentions; print failure non-113, timeout/signal, 113 with other stderr; + registration changed during the not-loaded re-capture. +- `tests/service/service-desktop-login-registration.test.ts`: materializes the fixture in a + temp home, drives `macLoginFactsFor`, asserts `loginEnabled`/`loginRegistration` and the + tray invariant. Registered in `scripts/test-layout/layout.json` and + `tests/fixtures/test-layout-expected.json`. + +### Docs + +- `structure/desktop-shell.md`: tray Start at Login judgment, states, click rules, parity + fixture. +- `docs-site/src/content/docs/guides/macos-menu-bar.md` and `desktop-app.md` (English): + the checkbox shows only a verified registration; reason labels. + +## Out of scope + +Linux/Windows tray, `first_run.rs` (its once-per-install migration still uses the plugin), +`gui/`, `app/`, TS judgment semantics, any real launchd/LaunchAgents state. + +## Verification + +- `cargo fmt --manifest-path desktop/src-tauri/Cargo.toml --check` +- `cargo clippy --manifest-path desktop/src-tauri/Cargo.toml --all-targets -- -D warnings` +- `cargo test --manifest-path desktop/src-tauri/Cargo.toml` +- `bun test tests/service/service-desktop-login-registration.test.ts tests/service/service-desktop-startup.test.ts tests/service/service-desktop-startup-health.test.ts` +- `bun run typecheck`, `bun run structure:check`, `bun test tests/test-layout.test.ts tests/test-layout-tooling.test.ts`, `bun run privacy:scan` +- No real launchd: tests use fakes and temp homes only. + + +## Audit round 1 (GPT-6.1-Sol high): FAIL → folded + +1. **MissingCopy takeover** — accepted. `MissingCopy` is non-clickable like `OtherCopy`; a + missing path may be an installation being replaced or on an unmounted volume. Table: + MissingCopy → Clickable no, action none. +2. **first_run migration rewrites another copy** — accepted. On macOS + `adopt_launch_origin_argument` only rewrites when the plist's `ProgramArguments[0]` + resolves to this running copy (legacy entries without `--autostart` included, via + `login_registration::registered_to`). Foreign, missing or unreadable registrations are left + untouched and the marker is not written, so a later launch can still migrate this copy's own + entry. Covered by Rust unit cases (this copy / other copy / missing / legacy shape / plutil + failure / absent). +3. **auto-toggle before probe** — accepted. The menu callback (main thread; Tauri runs setters + inline there, `tauri-runtime-wry` `send_user_message`) immediately restores the last + applied presentation and disables the item, then spawns the blocking job. It never waits on + the probe lock. Jobs and refreshes are serialized by one probe mutex; refreshes use + `try_lock`. The residual is AppKit's own toggle between the click and the callback. +4. **X_OK** — already folded: `access(path, X_OK)` via `extern "C"`. +5. **parser boundaries** — accepted: fixture adds delimiter/newline/EOF, invalid suffix + (`enabledx`), duplicate mentions, malformed envelope, non-boolean `Disabled`, `Program` + equal/different, symlinked plist, 113 with signal-style failure, wrong uid in stderr, + changed registration during re-capture. +6. **Linux dead code** — accepted: portable tests exercise `label`/`clickable`/ + `click_action`; macOS-only constants and helpers sit behind `cfg(target_os = "macos")`; + CI clippy runs on Linux, local run on macOS, both `--all-targets -D warnings`. +7. **label overclaims** — accepted: "Start at Login (disabled by macOS)"; click opens Login + Items settings, opener failure is logged once, and nothing is credited until a fresh probe. +8. **translations** — accepted: update `ko/guides/desktop-app.md` and any locale that + describes the checkbox as file existence; update `structure/desktop-shell.md` first-run + paragraph. + + +## Audit round 2 (GPT-6.1-Sol high): FAIL → folded + +1. **first-run guard and `Program`** — accepted. `login_registration::migration` returns + `Rewrite` only for `Label` OpenCodex, `RunAtLoad` true, `ProgramArguments` = + `[<…>/Contents/MacOS/opencodex-desktop]` or `[…, --autostart]`, `Program` absent or equal to + argument zero, and argument zero resolving to this running copy. Everything else is `Leave`. +2. **first-run re-enables a self-disabled plist** — accepted. `Disabled` must be absent or + `false`; a `true` or non-boolean value is `Leave`. A launchd override is untouched by a + plist rewrite, so it needs no special case. `Leave` does not write the origin marker; + `Absent` does (unchanged from the current `Ok(false)` branch). +3. **ECMAScript line terminators** — accepted. `line_value` splits on `\n`, `\r`, U+2028 and + U+2029. Shared fixture adds CR / U+2028 / U+2029 terminated `launchctl print` output, a + program value cut at U+2028, a program value that starts after a terminator, and a path value + cut at CR (58 cases). +4. Inline main-thread setters confirmed (`tauri-runtime-wry` 2.11.4). + +Final state table (supersedes the one above): + +| State | Checked | Label | Clickable | Action | +|---|---|---|---|---| +| On | yes | Start at Login | yes | disable | +| Off | no | Start at Login | yes | enable for this copy | +| DisabledBySystem | no | Start at Login (disabled by macOS) | yes | open Login Items settings | +| OtherCopy | no | Start at Login (another OpenCodex copy) | no | — | +| MissingCopy | no | Start at Login (registered app is missing) | no | — | +| Unknown | no | Start at Login (status unknown) | no | — | + + +## Audit round 3 (GPT-6.1-Sol high): FAIL → 4 folded, 1 rebutted + +1. **migration drops keys** — folded. `Rewrite` requires exactly the three keys auto-launch + writes (`Label`, `ProgramArguments`, `RunAtLoad`); `Program`, `Disabled`, `KeepAlive` + or any other key is `Leave`. +2. **cross-process check-then-act** — rebutted as residual. Every mutation re-probes + immediately before acting and the window is the few milliseconds between that probe and the + plugin's file write; the only actor that can interleave is another OpenCodex copy of the + same user acting in that window. The `Off` → enable write is the existing plugin path. + Cross-process locking or a custom plist writer is disproportionate to #6972 and would replace + plugin-owned code; recorded as a known limit in the PR. +3. **lossy UTF-8** — folded: `String::from_utf8_lossy` like Node's `encoding: "utf8"`. +4. **reader joins** — folded: readers send over channels and are collected with + `recv_timeout` inside the same 750 ms deadline; a blocked reader is abandoned. +5. **presentation interleaving** — folded: every presentation is applied in one + main-thread closure. A click increments a pending counter and restores/disables on the main + thread; refresh results are dropped while a click is pending; the click job's own result + decrements the counter and applies. + +## Code review round 1 (GPT-6.1-Sol high, commit 4c5fb7734d): FAIL → folded + +1. **migration accepted a plist replaced during its own read** — folded: `migration` now + captures the plist identity, its resolved target and target identity before `plutil`, and + re-reads them plus the registered executable's resolution afterwards; any change is + `Leave`. Regression: `migration_rewrites_only_this_copys_own_enabled_entry` replacement + case. The reviewer accepted the cross-process rebuttal for the window after a stable final + probe. +2. **structure doc said two actionable states** — folded: three (verified disables, absent + enables, launchd-disabled opens Login Items settings). diff --git a/devlog/_plan/261012_6972_tray_start_at_login/010_publish.md b/devlog/_plan/261012_6972_tray_start_at_login/010_publish.md new file mode 100644 index 00000000000..dbe1a98a6a3 --- /dev/null +++ b/devlog/_plan/261012_6972_tray_start_at_login/010_publish.md @@ -0,0 +1,18 @@ +# 010 — Publish #6972 + +Previous D (wp1): implementation at a0b181577f passed the focused gates (receipt +`.codexclaw/evidence/.../test-receipt.json`) and the GPT-6.1-Sol code review round 2 PASS. +Direction unchanged: publish for the merge-management thread; never merge here. + +Steps: + +1. `git fetch origin dev`; rebase onto `origin/dev`; if it moved, rerun `.tmp/check.sh`. +2. Push `codex/6972-tray-start-at-login-launchd-261012` to `origin` (maintainer branch). +3. Open a PR to `dev` from `.github/PULL_REQUEST_TEMPLATE.md` (Summary, Verification with the + independent review results, Checklist), `Closes #6972` (close manually after landing on + `dev`). No `gui/` change, so no screenshot. +4. `link_pull_request`; `goat_thread_send` to mcp:284c76b4-6e51-464d-acf4-ff040b9dc299 with PR + number, head SHA and verification summary. +5. Observe exact-head CI; fix author-owned failures; report status. + +Out of scope: merging, closing the issue before `dev` lands, touching live launchd/service state. diff --git a/docs-site/src/content/docs/guides/desktop-app.md b/docs-site/src/content/docs/guides/desktop-app.md index b16551e1e5e..05ace2e004e 100644 --- a/docs-site/src/content/docs/guides/desktop-app.md +++ b/docs-site/src/content/docs/guides/desktop-app.md @@ -104,9 +104,14 @@ Turn on **Start at Login** in the OpenCodex menu instead of installing a backgro service. Startup safety credits the app only when its login registration is verified; if it cannot be verified, follow the Desktop guidance in status. The Startup page then labels the Desktop row **Running · Start at Login not verified**. -The menu checkbox only shows that the login item file exists. If it is already checked, -make sure macOS allows OpenCodex to open at login under System Settings → General → -Login Items & Extensions. While the app runs the proxy, the Startup page keeps the +On macOS the menu's **Start at Login** item is checked only when Startup safety would +verify the same registration. Otherwise the item is unchecked and its label says why: +**disabled by macOS** opens System Settings → General → Login Items & Extensions, where +you allow OpenCodex to open at login; **another OpenCodex copy** means the login item +belongs to a different copy of the app, so turn it off from that copy; **registered app +is missing** means the login item names an app that no longer exists, so delete +`~/Library/LaunchAgents/OpenCodex.plist` and turn the item on again; **status unknown** +means the registration could not be read. While the app runs the proxy, the Startup page keeps the background service and launcher shim installers disabled; to use them instead, quit OpenCodex and run `ocx service install` or `ocx codex-shim install` in a terminal. diff --git a/docs-site/src/content/docs/guides/macos-menu-bar.md b/docs-site/src/content/docs/guides/macos-menu-bar.md index 38af14941bb..c5da2bd6ce3 100644 --- a/docs-site/src/content/docs/guides/macos-menu-bar.md +++ b/docs-site/src/content/docs/guides/macos-menu-bar.md @@ -13,7 +13,7 @@ Download `OpenCodex--macos.dmg` from the [latest release](https://githu Release builds of `OpenCodex.app` are signed with a Developer ID, use the hardened runtime, and are notarized by Apple with the ticket stapled to the app. On first launch, macOS normally asks only for the standard confirmation for an app downloaded from the internet. If macOS still blocks it, open **System Settings → Privacy & Security** and choose **Open Anyway** for OpenCodex. Apps you build yourself are ad-hoc signed; see [Build from source](#build-from-source). -The app shows its startup progress in a window when you open it. It enables **Start at Login** once on first launch; you can turn that off from the tray menu. Later launches from the login item start with the window hidden while the tray remains available. +The app shows its startup progress in a window when you open it. It enables **Start at Login** once on first launch; you can turn that off from the tray menu. The tray item is checked only when this copy of the app is registered and macOS has not disabled the login item; otherwise its label says why (see [Desktop app](/guides/desktop-app/)). Later launches from the login item start with the window hidden while the tray remains available. ## Menu bar and usage panel diff --git a/docs-site/src/content/docs/ko/guides/desktop-app.md b/docs-site/src/content/docs/ko/guides/desktop-app.md index 6c0078f6543..7645930f3e4 100644 --- a/docs-site/src/content/docs/ko/guides/desktop-app.md +++ b/docs-site/src/content/docs/ko/guides/desktop-app.md @@ -80,9 +80,13 @@ macOS와 Linux에서 데스크톱 앱이 번들 프록시를 실행하는 것으 등록이 확인되어야 재시작 후 보호가 적용됩니다. 등록을 확인할 수 없으면 status에 표시되는 데스크톱 안내를 따르세요. 실행 관계 확인은 소유권을 생성하지 않습니다. 이때 시작 안전 페이지의 데스크톱 앱 행에는 **실행 중 · 로그인 시 시작 미확인**이 표시됩니다. -메뉴의 체크 표시는 로그인 항목 파일이 있다는 것만 나타냅니다. 이미 체크되어 있다면 -macOS의 시스템 설정 → 일반 → 로그인 항목 및 확장 프로그램에서 OpenCodex가 로그인 시 -열리도록 허용되어 있는지 확인하세요. 앱이 프록시를 실행하는 동안 시작 안전 페이지는 +macOS에서 메뉴의 **Start at Login** 항목은 시작 안전 페이지도 같은 등록을 확인할 때만 +체크됩니다. 그렇지 않으면 체크가 해제된 채 이유가 라벨에 붙습니다. **disabled by macOS**는 +시스템 설정 → 일반 → 로그인 항목 및 확장 프로그램을 열며, 거기서 OpenCodex가 로그인 시 +열리도록 허용하면 됩니다. **another OpenCodex copy**는 로그인 항목이 다른 앱 사본의 것이라는 +뜻이니 그 사본에서 끄세요. **registered app is missing**은 로그인 항목이 더 이상 없는 앱을 +가리킨다는 뜻이니 `~/Library/LaunchAgents/OpenCodex.plist`를 지우고 항목을 다시 켜세요. +**status unknown**은 등록을 읽지 못했다는 뜻입니다. 앱이 프록시를 실행하는 동안 시작 안전 페이지는 백그라운드 서비스와 런처 shim 설치를 막아 둡니다. 이를 대신 쓰려면 OpenCodex를 종료하고 터미널에서 `ocx service install` 또는 `ocx codex-shim install`을 실행하세요. diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index 244335dcb8c..1cecf7b1f02 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -87,7 +87,7 @@ "codex-sidecar-credit-policy.test.ts": "codex-integration", "codex-credits-after-limit-main.test.ts": "codex-integration", "service-runtime-preflight.test.ts": "service", "service-desktop-runtime-preflight.test.ts": "service", - "service-desktop-startup-health.test.ts": "service", "service-desktop-startup-linux.test.ts": "service", "service-desktop-startup.test.ts": "service", + "service-desktop-login-registration.test.ts": "service", "service-desktop-startup-health.test.ts": "service", "service-desktop-startup-linux.test.ts": "service", "service-desktop-startup.test.ts": "service", "startup-health-packaged-probe.test.ts": "server", "discovered-native-models.test.ts": "codex-integration", "responses-hosted-image-delivery.test.ts": "responses", diff --git a/src/service/desktop-startup.ts b/src/service/desktop-startup.ts index a567269103d..bb25a38e880 100644 --- a/src/service/desktop-startup.ts +++ b/src/service/desktop-startup.ts @@ -121,6 +121,18 @@ function macLoginRegistration(deps: DesktopStartupDeps) { type DesktopLoginFacts = Pick; +/** + * Start at Login credit for one Desktop app copy. The tray's Start at Login item ports this + * judgment (desktop/src-tauri/src/login_registration.rs); both run the cases in + * tests/fixtures/desktop-login-registration-cases.json. Throws when evidence is unreadable. + */ +export function macLoginFactsFor(app: string, deps: DesktopStartupDeps): DesktopLoginFacts { + const login = macLoginRegistration(deps); + return login !== null && login.app === app + ? { loginEnabled: login.loginEnabled, ...(login.loginRegistration ? { loginRegistration: login.loginRegistration } : {}) } + : { loginEnabled: false }; +} + function diagnoseSupervisedStartup(deps: DesktopStartupDeps, ownerRevision: number, loginFor: (app: string) => DesktopLoginFacts): DesktopStartupDiagnostic | undefined { const evidence = inspectDesktopSupervision(deps); if (evidence.kind !== "desktop") return undefined; @@ -144,12 +156,7 @@ export function diagnoseMacDesktopStartup(deps: DesktopStartupDeps = {}): Deskto if ((deps.platform ?? process.platform) !== "darwin") return undefined; const ownership = deps.ownership ?? resolveServiceOwnership; const owner = ownership(); - if (owner.kind === "none") return diagnoseSupervisedStartup(deps, owner.revision, app => { - const login = macLoginRegistration(deps); - return login !== null && login.app === app - ? { loginEnabled: login.loginEnabled, ...(login.loginRegistration ? { loginRegistration: login.loginRegistration } : {}) } - : { loginEnabled: false }; - }); + if (owner.kind === "none") return diagnoseSupervisedStartup(deps, owner.revision, app => macLoginFactsFor(app, deps)); if (owner.kind !== "owned" || owner.ownership.owner !== "desktop") return undefined; const facts: Omit = { owned: true, loginEnabled: false, running: false }; const execute = deps.run ?? run; diff --git a/structure/INDEX.md b/structure/INDEX.md index 589579e8e54..d0be33e4d97 100644 --- a/structure/INDEX.md +++ b/structure/INDEX.md @@ -99,6 +99,7 @@ Background service, docs, release, and design discipline. | --- | --- | | [`desktop-shell.md`](desktop-shell.md) | Tauri desktop shell, proxy attachment and sidecar lifecycle, tray controls, bootstrap navigation, and desktop companion presence. | | [`desktop-terminal-command.md`](desktop-terminal-command.md) | Desktop-owned ocx command on PATH: record and journal, POSIX shim and shell blocks, Windows user Path, private-path ACL policy, and the local Terminal command page. | +| [`desktop-start-at-login.md`](desktop-start-at-login.md) | macOS tray Start at Login judgment shared with Startup safety, its parity fixture, click rules, and the guarded launch-origin migration. | | [`ops/service-and-sidecars.md`](ops/service-and-sidecars.md) | Service install/repair, platform launchers, tray, and sidecar processes. | | [`ops/service-health-recovery.md`](ops/service-health-recovery.md) | Opt-in sentinel that turns a hung but listening managed service proxy into an exit its manager restarts. | | [`ops/plugins.md`](ops/plugins.md) | Plugin loading from OPENCODEX_HOME/plugins and the upstream rewrite slot plugins attach to. | @@ -122,6 +123,9 @@ A source area can be described by more than one doc, because these docs are orga | `desktop/src-tauri/src/cli_command_posix.rs` | [`desktop-terminal-command.md`](desktop-terminal-command.md) | | `desktop/src-tauri/src/cli_command_record.rs` | [`desktop-terminal-command.md`](desktop-terminal-command.md) | | `desktop/src-tauri/src/cli_command_windows.rs` | [`desktop-terminal-command.md`](desktop-terminal-command.md) | +| `desktop/src-tauri/src/first_run.rs` | [`desktop-start-at-login.md`](desktop-start-at-login.md) | +| `desktop/src-tauri/src/login_registration.rs` | [`desktop-start-at-login.md`](desktop-start-at-login.md) | +| `desktop/src-tauri/src/tray_login.rs` | [`desktop-start-at-login.md`](desktop-start-at-login.md) | | `desktop/ui/cli.html` | [`desktop-terminal-command.md`](desktop-terminal-command.md) | | `desktop/ui/cli.js` | [`desktop-terminal-command.md`](desktop-terminal-command.md) | | `docs-site/` | [`ops/docs-and-release.md`](ops/docs-and-release.md) | @@ -184,6 +188,7 @@ A source area can be described by more than one doc, because these docs are orga | `src/service-manager-probe.ts` | [`ops/service-and-sidecars.md`](ops/service-and-sidecars.md) | | `src/service.ts` | [`runtime.md`](runtime.md)
[`ops/docs-and-release.md`](ops/docs-and-release.md) | | `src/service/` | [`runtime.md`](runtime.md) | +| `src/service/desktop-startup.ts` | [`desktop-start-at-login.md`](desktop-start-at-login.md) | | `src/service/health-sentinel-launch.ts` | [`ops/service-health-recovery.md`](ops/service-health-recovery.md) | | `src/service/health-sentinel-main.ts` | [`ops/service-health-recovery.md`](ops/service-health-recovery.md) | | `src/service/health-sentinel-policy.ts` | [`ops/service-health-recovery.md`](ops/service-health-recovery.md) | diff --git a/structure/desktop-shell.md b/structure/desktop-shell.md index 8e9b9538b5e..085e53c5e4b 100644 --- a/structure/desktop-shell.md +++ b/structure/desktop-shell.md @@ -408,6 +408,9 @@ plugin's registration name and Task Manager explicit-enable semantics. A Windows launch-origin migration marker revisits older registrations once, only when already enabled; entries disabled in the tray or Task Manager stay disabled. +The macOS tray item and the macOS side of this migration judge the login item the way Startup +safety does; see [Desktop Start at Login](desktop-start-at-login.md). + The WidgetKit extension in `app/` needs three things that Xcode's app-extension target would supply on its own, and SwiftPM has no such target: `@main` on `OpenCodexWidgetBundle`, the `-e _NSExtensionMain` linker entry, and diff --git a/structure/desktop-start-at-login.md b/structure/desktop-start-at-login.md new file mode 100644 index 00000000000..8fc9e7d8032 --- /dev/null +++ b/structure/desktop-start-at-login.md @@ -0,0 +1,34 @@ +# Desktop Start at Login + +How the macOS tray's **Start at Login** item decides what to show and what a click may change, +and how the once-per-installation launch-origin migration avoids another copy's login item. The +surrounding first-run default and tray are described in [Desktop shell](desktop-shell.md); the +plan and audit history is in `devlog/_plan/261012_6972_tray_start_at_login/`. + +On macOS the tray's Start at Login item does not read the plugin's `is_enabled`, which is +only "the LaunchAgent plist exists". `desktop/src-tauri/src/login_registration.rs` ports +`macLoginRegistration` and `macLoginFactsFor` from `src/service/desktop-startup.ts` step for +step — plist shape, `launchctl print-disabled` override, registered app equal to this running +copy, loaded `program`/`path` or the exact service-not-found answer with an unchanged +re-read — so the item is checked exactly when Startup safety would credit this copy (#6972). +The cases in `tests/fixtures/desktop-login-registration-cases.json` run against both +implementations (`login_registration::tests` and +`tests/service/service-desktop-login-registration.test.ts`); a change to either judgment +must change the fixture and pass both. Every other state is unchecked with its reason in the +label. Three states are actionable: verified turns Start at Login off, absent registers this +copy, and a launchd-disabled job of this copy opens Login Items settings, because rewriting the +plist cannot clear launchd's override. +Another copy's registration, a registration naming a missing app and unreadable evidence leave +the item disabled; the tray never rewrites them. `desktop/src-tauri/src/tray_login.rs` probes on +blocking workers under one mutex (at build, on hover at most every two seconds, every 60 +seconds, and around a click, re-probing before it acts) and applies presentation in one +main-thread closure. muda flips a check item before the click reaches the app, so the click +handler restores the shown state and disables the item until its own answer; refreshes landing +meanwhile are dropped. The probe and the action are not atomic across processes: another copy +rewriting the plist in the milliseconds between them is a known limit. + +The macOS launch-origin migration in `desktop/src-tauri/src/first_run.rs` is narrower than on +other platforms. The marker sits in a config directory every copy shares, and the plugin +rewrites the whole plist, so `login_registration::migration` allows the rewrite only for this +copy's entry carrying exactly the three keys the plugin writes and not replaced while it was +read. Anything else is left untouched and unmarked. diff --git a/structure/manifest.json b/structure/manifest.json index c23541bae70..b9cc8f4565b 100644 --- a/structure/manifest.json +++ b/structure/manifest.json @@ -560,6 +560,18 @@ "desktop/ui/cli.js" ] }, + { + "path": "desktop-start-at-login.md", + "tier": 6, + "title": "Desktop Start at Login", + "scope": "macOS tray Start at Login judgment shared with Startup safety, its parity fixture, click rules, and the guarded launch-origin migration.", + "documents": [ + "desktop/src-tauri/src/first_run.rs", + "desktop/src-tauri/src/login_registration.rs", + "desktop/src-tauri/src/tray_login.rs", + "src/service/desktop-startup.ts" + ] + }, { "path": "ops/service-and-sidecars.md", "tier": 6, diff --git a/tests/fixtures/desktop-login-registration-cases.json b/tests/fixtures/desktop-login-registration-cases.json new file mode 100644 index 00000000000..9c232168a7f --- /dev/null +++ b/tests/fixtures/desktop-login-registration-cases.json @@ -0,0 +1,1136 @@ +{ + "//": "Shared by desktop/src-tauri/src/login_registration.rs and tests/service/service-desktop-login-registration.test.ts (#6972). Each case is one macOS Start at Login registration; expect.loginEnabled is what Startup safety reports for this app, expect.tray is what the tray shows. Only tray \"on\" is checked, and it must coincide with loginEnabled. Placeholders: {APP} this copy, {OTHER_APP} another existing copy, {MISSING_APP} a copy that does not exist, {PLIST} ~/Library/LaunchAgents/OpenCodex.plist, {OTHER_PLIST} another existing plist, {UID} 501. plist null means the LaunchAgent file is absent; disabled null means print-disabled fails; loaded is launchctl print: stdout, or an exit status with stderr (and optional signal), or a failure such as a timeout.", + "cases": [ + { + "name": "loaded from this copy and this plist", + "plist": { + "Label": "OpenCodex", + "RunAtLoad": true, + "ProgramArguments": [ + "{APP}", + "--autostart" + ] + }, + "disabled": "disabled services = {\n\t\"com.apple.example\" => disabled\n\t\"OpenCodex\" => enabled\n}", + "loaded": { + "stdout": "gui/{UID}/OpenCodex = {\n\tactive count = 1\n\tpath = {PLIST}\n\tstate = running\n\tprogram = {APP}\n}" + }, + "expect": { + "tray": "on", + "loginEnabled": true, + "loginRegistration": "loaded" + } + }, + { + "name": "valid registration not loaded yet", + "plist": { + "Label": "OpenCodex", + "RunAtLoad": true, + "ProgramArguments": [ + "{APP}", + "--autostart" + ] + }, + "disabled": "disabled services = {\n\t\"com.apple.example\" => disabled\n\t\"OpenCodex\" => enabled\n}", + "loaded": { + "status": 113, + "stderr": "Could not find service \"OpenCodex\" in domain for user gui: {UID}" + }, + "expect": { + "tray": "on", + "loginEnabled": true, + "loginRegistration": "registered-not-loaded" + } + }, + { + "name": "valid registration not loaded yet, Bad request form", + "plist": { + "Label": "OpenCodex", + "RunAtLoad": true, + "ProgramArguments": [ + "{APP}", + "--autostart" + ] + }, + "disabled": "disabled services = {\n\t\"com.apple.example\" => disabled\n\t\"OpenCodex\" => enabled\n}", + "loaded": { + "status": 113, + "stderr": "Bad request.\nCould not find service \"OpenCodex\" in domain for user gui: {UID}\n" + }, + "expect": { + "tray": "on", + "loginEnabled": true, + "loginRegistration": "registered-not-loaded" + } + }, + { + "name": "no override for OpenCodex", + "plist": { + "Label": "OpenCodex", + "RunAtLoad": true, + "ProgramArguments": [ + "{APP}", + "--autostart" + ] + }, + "disabled": "disabled services = {\n\t\"com.apple.example\" => disabled\n}", + "loaded": { + "stdout": "gui/{UID}/OpenCodex = {\n\tactive count = 1\n\tpath = {PLIST}\n\tstate = running\n\tprogram = {APP}\n}" + }, + "expect": { + "tray": "on", + "loginEnabled": true, + "loginRegistration": "loaded" + } + }, + { + "name": "empty override list", + "plist": { + "Label": "OpenCodex", + "RunAtLoad": true, + "ProgramArguments": [ + "{APP}", + "--autostart" + ] + }, + "disabled": "disabled services = {\n}", + "loaded": { + "stdout": "gui/{UID}/OpenCodex = {\n\tactive count = 1\n\tpath = {PLIST}\n\tstate = running\n\tprogram = {APP}\n}" + }, + "expect": { + "tray": "on", + "loginEnabled": true, + "loginRegistration": "loaded" + } + }, + { + "name": "override false", + "plist": { + "Label": "OpenCodex", + "RunAtLoad": true, + "ProgramArguments": [ + "{APP}", + "--autostart" + ] + }, + "disabled": "disabled services = {\n\t\"OpenCodex\" => false\n}", + "loaded": { + "stdout": "gui/{UID}/OpenCodex = {\n\tactive count = 1\n\tpath = {PLIST}\n\tstate = running\n\tprogram = {APP}\n}" + }, + "expect": { + "tray": "on", + "loginEnabled": true, + "loginRegistration": "loaded" + } + }, + { + "name": "single-line override list", + "plist": { + "Label": "OpenCodex", + "RunAtLoad": true, + "ProgramArguments": [ + "{APP}", + "--autostart" + ] + }, + "disabled": "disabled services = { \"OpenCodex\" => enabled }", + "loaded": { + "stdout": "gui/{UID}/OpenCodex = {\n\tactive count = 1\n\tpath = {PLIST}\n\tstate = running\n\tprogram = {APP}\n}" + }, + "expect": { + "tray": "on", + "loginEnabled": true, + "loginRegistration": "loaded" + } + }, + { + "name": "override followed by a comma", + "plist": { + "Label": "OpenCodex", + "RunAtLoad": true, + "ProgramArguments": [ + "{APP}", + "--autostart" + ] + }, + "disabled": "disabled services = { \"OpenCodex\" => enabled, \"x\" => disabled }", + "loaded": { + "stdout": "gui/{UID}/OpenCodex = {\n\tactive count = 1\n\tpath = {PLIST}\n\tstate = running\n\tprogram = {APP}\n}" + }, + "expect": { + "tray": "on", + "loginEnabled": true, + "loginRegistration": "loaded" + } + }, + { + "name": "override with CRLF line ending", + "plist": { + "Label": "OpenCodex", + "RunAtLoad": true, + "ProgramArguments": [ + "{APP}", + "--autostart" + ] + }, + "disabled": "disabled services = {\r\n\t\"OpenCodex\" => enabled\r\n}", + "loaded": { + "stdout": "gui/{UID}/OpenCodex = {\n\tactive count = 1\n\tpath = {PLIST}\n\tstate = running\n\tprogram = {APP}\n}" + }, + "expect": { + "tray": "on", + "loginEnabled": true, + "loginRegistration": "loaded" + } + }, + { + "name": "override with trailing spaces before newline", + "plist": { + "Label": "OpenCodex", + "RunAtLoad": true, + "ProgramArguments": [ + "{APP}", + "--autostart" + ] + }, + "disabled": "disabled services = {\n\t\"OpenCodex\" => enabled \n}", + "loaded": { + "stdout": "gui/{UID}/OpenCodex = {\n\tactive count = 1\n\tpath = {PLIST}\n\tstate = running\n\tprogram = {APP}\n}" + }, + "expect": { + "tray": "on", + "loginEnabled": true, + "loginRegistration": "loaded" + } + }, + { + "name": "plist absent", + "plist": null, + "disabled": "disabled services = {\n\t\"com.apple.example\" => disabled\n\t\"OpenCodex\" => enabled\n}", + "loaded": { + "stdout": "gui/{UID}/OpenCodex = {\n\tactive count = 1\n\tpath = {PLIST}\n\tstate = running\n\tprogram = {APP}\n}" + }, + "expect": { + "tray": "off", + "loginEnabled": false + } + }, + { + "name": "plist absent while launchd keeps a disabled override", + "plist": null, + "disabled": "disabled services = {\n\t\"OpenCodex\" => disabled\n}", + "loaded": { + "stdout": "gui/{UID}/OpenCodex = {\n\tactive count = 1\n\tpath = {PLIST}\n\tstate = running\n\tprogram = {APP}\n}" + }, + "expect": { + "tray": "off", + "loginEnabled": false + } + }, + { + "name": "launchd disables this copy", + "plist": { + "Label": "OpenCodex", + "RunAtLoad": true, + "ProgramArguments": [ + "{APP}", + "--autostart" + ] + }, + "disabled": "disabled services = {\n\t\"OpenCodex\" => disabled\n}", + "loaded": { + "stdout": "gui/{UID}/OpenCodex = {\n\tactive count = 1\n\tpath = {PLIST}\n\tstate = running\n\tprogram = {APP}\n}" + }, + "expect": { + "tray": "disabled-by-system", + "loginEnabled": false + } + }, + { + "name": "launchd disables this copy with true", + "plist": { + "Label": "OpenCodex", + "RunAtLoad": true, + "ProgramArguments": [ + "{APP}", + "--autostart" + ] + }, + "disabled": "disabled services = {\n\t\"OpenCodex\" => true\n}", + "loaded": { + "stdout": "gui/{UID}/OpenCodex = {\n\tactive count = 1\n\tpath = {PLIST}\n\tstate = running\n\tprogram = {APP}\n}" + }, + "expect": { + "tray": "disabled-by-system", + "loginEnabled": false + } + }, + { + "name": "plist Disabled true without override", + "plist": { + "Label": "OpenCodex", + "RunAtLoad": true, + "ProgramArguments": [ + "{APP}", + "--autostart" + ], + "Disabled": true + }, + "disabled": "disabled services = {\n}", + "loaded": { + "stdout": "gui/{UID}/OpenCodex = {\n\tactive count = 1\n\tpath = {PLIST}\n\tstate = running\n\tprogram = {APP}\n}" + }, + "expect": { + "tray": "unknown", + "loginEnabled": false + } + }, + { + "name": "plist Disabled false without override", + "plist": { + "Label": "OpenCodex", + "RunAtLoad": true, + "ProgramArguments": [ + "{APP}", + "--autostart" + ], + "Disabled": false + }, + "disabled": "disabled services = {\n}", + "loaded": { + "stdout": "gui/{UID}/OpenCodex = {\n\tactive count = 1\n\tpath = {PLIST}\n\tstate = running\n\tprogram = {APP}\n}" + }, + "expect": { + "tray": "on", + "loginEnabled": true, + "loginRegistration": "loaded" + } + }, + { + "name": "plist Disabled true with enabled override", + "plist": { + "Label": "OpenCodex", + "RunAtLoad": true, + "ProgramArguments": [ + "{APP}", + "--autostart" + ], + "Disabled": true + }, + "disabled": "disabled services = {\n\t\"com.apple.example\" => disabled\n\t\"OpenCodex\" => enabled\n}", + "loaded": { + "stdout": "gui/{UID}/OpenCodex = {\n\tactive count = 1\n\tpath = {PLIST}\n\tstate = running\n\tprogram = {APP}\n}" + }, + "expect": { + "tray": "on", + "loginEnabled": true, + "loginRegistration": "loaded" + } + }, + { + "name": "plist Disabled is not a boolean", + "plist": { + "Label": "OpenCodex", + "RunAtLoad": true, + "ProgramArguments": [ + "{APP}", + "--autostart" + ], + "Disabled": "yes" + }, + "disabled": "disabled services = {\n\t\"com.apple.example\" => disabled\n\t\"OpenCodex\" => enabled\n}", + "loaded": { + "stdout": "gui/{UID}/OpenCodex = {\n\tactive count = 1\n\tpath = {PLIST}\n\tstate = running\n\tprogram = {APP}\n}" + }, + "expect": { + "tray": "unknown", + "loginEnabled": false + } + }, + { + "name": "Program equal to the registered app", + "plist": { + "Label": "OpenCodex", + "RunAtLoad": true, + "ProgramArguments": [ + "{APP}", + "--autostart" + ], + "Program": "{APP}" + }, + "disabled": "disabled services = {\n\t\"com.apple.example\" => disabled\n\t\"OpenCodex\" => enabled\n}", + "loaded": { + "stdout": "gui/{UID}/OpenCodex = {\n\tactive count = 1\n\tpath = {PLIST}\n\tstate = running\n\tprogram = {APP}\n}" + }, + "expect": { + "tray": "on", + "loginEnabled": true, + "loginRegistration": "loaded" + } + }, + { + "name": "Program differs from the registered app", + "plist": { + "Label": "OpenCodex", + "RunAtLoad": true, + "ProgramArguments": [ + "{APP}", + "--autostart" + ], + "Program": "{OTHER_APP}" + }, + "disabled": "disabled services = {\n\t\"com.apple.example\" => disabled\n\t\"OpenCodex\" => enabled\n}", + "loaded": { + "stdout": "gui/{UID}/OpenCodex = {\n\tactive count = 1\n\tpath = {PLIST}\n\tstate = running\n\tprogram = {APP}\n}" + }, + "expect": { + "tray": "unknown", + "loginEnabled": false + } + }, + { + "name": "another existing copy is registered", + "plist": { + "Label": "OpenCodex", + "RunAtLoad": true, + "ProgramArguments": [ + "{OTHER_APP}", + "--autostart" + ] + }, + "disabled": "disabled services = {\n\t\"com.apple.example\" => disabled\n\t\"OpenCodex\" => enabled\n}", + "loaded": { + "stdout": "gui/{UID}/OpenCodex = {\n\tactive count = 1\n\tpath = {PLIST}\n\tstate = running\n\tprogram = {APP}\n}" + }, + "expect": { + "tray": "other-copy", + "loginEnabled": false + } + }, + { + "name": "another copy is registered and launchd disables it", + "plist": { + "Label": "OpenCodex", + "RunAtLoad": true, + "ProgramArguments": [ + "{OTHER_APP}", + "--autostart" + ] + }, + "disabled": "disabled services = {\n\t\"OpenCodex\" => disabled\n}", + "loaded": { + "stdout": "gui/{UID}/OpenCodex = {\n\tactive count = 1\n\tpath = {PLIST}\n\tstate = running\n\tprogram = {APP}\n}" + }, + "expect": { + "tray": "other-copy", + "loginEnabled": false + } + }, + { + "name": "loaded job runs another copy", + "plist": { + "Label": "OpenCodex", + "RunAtLoad": true, + "ProgramArguments": [ + "{APP}", + "--autostart" + ] + }, + "disabled": "disabled services = {\n\t\"com.apple.example\" => disabled\n\t\"OpenCodex\" => enabled\n}", + "loaded": { + "stdout": "gui/{UID}/OpenCodex = {\n\tpath = {PLIST}\n\tprogram = {OTHER_APP}\n}" + }, + "expect": { + "tray": "other-copy", + "loginEnabled": false + } + }, + { + "name": "loaded job runs a copy that no longer exists", + "plist": { + "Label": "OpenCodex", + "RunAtLoad": true, + "ProgramArguments": [ + "{APP}", + "--autostart" + ] + }, + "disabled": "disabled services = {\n\t\"com.apple.example\" => disabled\n\t\"OpenCodex\" => enabled\n}", + "loaded": { + "stdout": "gui/{UID}/OpenCodex = {\n\tpath = {PLIST}\n\tprogram = {MISSING_APP}\n}" + }, + "expect": { + "tray": "other-copy", + "loginEnabled": false + } + }, + { + "name": "loaded job came from another plist", + "plist": { + "Label": "OpenCodex", + "RunAtLoad": true, + "ProgramArguments": [ + "{APP}", + "--autostart" + ] + }, + "disabled": "disabled services = {\n\t\"com.apple.example\" => disabled\n\t\"OpenCodex\" => enabled\n}", + "loaded": { + "stdout": "gui/{UID}/OpenCodex = {\n\tpath = {OTHER_PLIST}\n\tprogram = {APP}\n}" + }, + "expect": { + "tray": "unknown", + "loginEnabled": false + } + }, + { + "name": "loaded job without a program line", + "plist": { + "Label": "OpenCodex", + "RunAtLoad": true, + "ProgramArguments": [ + "{APP}", + "--autostart" + ] + }, + "disabled": "disabled services = {\n\t\"com.apple.example\" => disabled\n\t\"OpenCodex\" => enabled\n}", + "loaded": { + "stdout": "gui/{UID}/OpenCodex = {\n\tpath = {PLIST}\n}" + }, + "expect": { + "tray": "unknown", + "loginEnabled": false + } + }, + { + "name": "loaded job without a path line", + "plist": { + "Label": "OpenCodex", + "RunAtLoad": true, + "ProgramArguments": [ + "{APP}", + "--autostart" + ] + }, + "disabled": "disabled services = {\n\t\"com.apple.example\" => disabled\n\t\"OpenCodex\" => enabled\n}", + "loaded": { + "stdout": "gui/{UID}/OpenCodex = {\n\tprogram = {APP}\n}" + }, + "expect": { + "tray": "unknown", + "loginEnabled": false + } + }, + { + "name": "registered app no longer exists", + "plist": { + "Label": "OpenCodex", + "RunAtLoad": true, + "ProgramArguments": [ + "{MISSING_APP}", + "--autostart" + ] + }, + "disabled": "disabled services = {\n\t\"com.apple.example\" => disabled\n\t\"OpenCodex\" => enabled\n}", + "loaded": { + "stdout": "gui/{UID}/OpenCodex = {\n\tactive count = 1\n\tpath = {PLIST}\n\tstate = running\n\tprogram = {APP}\n}" + }, + "expect": { + "tray": "missing-copy", + "loginEnabled": false + } + }, + { + "name": "this copy has no bundled ocx", + "plist": { + "Label": "OpenCodex", + "RunAtLoad": true, + "ProgramArguments": [ + "{APP}", + "--autostart" + ] + }, + "disabled": "disabled services = {\n\t\"com.apple.example\" => disabled\n\t\"OpenCodex\" => enabled\n}", + "loaded": { + "stdout": "gui/{UID}/OpenCodex = {\n\tactive count = 1\n\tpath = {PLIST}\n\tstate = running\n\tprogram = {APP}\n}" + }, + "proxy": false, + "expect": { + "tray": "unknown", + "loginEnabled": false + } + }, + { + "name": "plist cannot be converted", + "plist": { + "Label": "OpenCodex", + "RunAtLoad": true, + "ProgramArguments": [ + "{APP}", + "--autostart" + ] + }, + "disabled": "disabled services = {\n\t\"com.apple.example\" => disabled\n\t\"OpenCodex\" => enabled\n}", + "loaded": { + "stdout": "gui/{UID}/OpenCodex = {\n\tactive count = 1\n\tpath = {PLIST}\n\tstate = running\n\tprogram = {APP}\n}" + }, + "plutilFails": true, + "expect": { + "tray": "unknown", + "loginEnabled": false + } + }, + { + "name": "registration lacks --autostart", + "plist": { + "Label": "OpenCodex", + "RunAtLoad": true, + "ProgramArguments": [ + "{APP}" + ] + }, + "disabled": "disabled services = {\n\t\"com.apple.example\" => disabled\n\t\"OpenCodex\" => enabled\n}", + "loaded": { + "stdout": "gui/{UID}/OpenCodex = {\n\tactive count = 1\n\tpath = {PLIST}\n\tstate = running\n\tprogram = {APP}\n}" + }, + "expect": { + "tray": "unknown", + "loginEnabled": false + } + }, + { + "name": "registration carries an extra argument", + "plist": { + "Label": "OpenCodex", + "RunAtLoad": true, + "ProgramArguments": [ + "{APP}", + "--autostart", + "--x" + ] + }, + "disabled": "disabled services = {\n\t\"com.apple.example\" => disabled\n\t\"OpenCodex\" => enabled\n}", + "loaded": { + "stdout": "gui/{UID}/OpenCodex = {\n\tactive count = 1\n\tpath = {PLIST}\n\tstate = running\n\tprogram = {APP}\n}" + }, + "expect": { + "tray": "unknown", + "loginEnabled": false + } + }, + { + "name": "registration label is not OpenCodex", + "plist": { + "Label": "com.example.OpenCodex", + "RunAtLoad": true, + "ProgramArguments": [ + "{APP}", + "--autostart" + ] + }, + "disabled": "disabled services = {\n\t\"com.apple.example\" => disabled\n\t\"OpenCodex\" => enabled\n}", + "loaded": { + "stdout": "gui/{UID}/OpenCodex = {\n\tactive count = 1\n\tpath = {PLIST}\n\tstate = running\n\tprogram = {APP}\n}" + }, + "expect": { + "tray": "unknown", + "loginEnabled": false + } + }, + { + "name": "RunAtLoad is not true", + "plist": { + "Label": "OpenCodex", + "RunAtLoad": false, + "ProgramArguments": [ + "{APP}", + "--autostart" + ] + }, + "disabled": "disabled services = {\n\t\"com.apple.example\" => disabled\n\t\"OpenCodex\" => enabled\n}", + "loaded": { + "stdout": "gui/{UID}/OpenCodex = {\n\tactive count = 1\n\tpath = {PLIST}\n\tstate = running\n\tprogram = {APP}\n}" + }, + "expect": { + "tray": "unknown", + "loginEnabled": false + } + }, + { + "name": "registered program is not the desktop binary", + "plist": { + "Label": "OpenCodex", + "RunAtLoad": true, + "ProgramArguments": [ + "/usr/bin/true", + "--autostart" + ] + }, + "disabled": "disabled services = {\n\t\"com.apple.example\" => disabled\n\t\"OpenCodex\" => enabled\n}", + "loaded": { + "stdout": "gui/{UID}/OpenCodex = {\n\tactive count = 1\n\tpath = {PLIST}\n\tstate = running\n\tprogram = {APP}\n}" + }, + "expect": { + "tray": "unknown", + "loginEnabled": false + } + }, + { + "name": "print-disabled fails", + "plist": { + "Label": "OpenCodex", + "RunAtLoad": true, + "ProgramArguments": [ + "{APP}", + "--autostart" + ] + }, + "disabled": null, + "loaded": { + "stdout": "gui/{UID}/OpenCodex = {\n\tactive count = 1\n\tpath = {PLIST}\n\tstate = running\n\tprogram = {APP}\n}" + }, + "expect": { + "tray": "unknown", + "loginEnabled": false + } + }, + { + "name": "print-disabled output is malformed", + "plist": { + "Label": "OpenCodex", + "RunAtLoad": true, + "ProgramArguments": [ + "{APP}", + "--autostart" + ] + }, + "disabled": "\"OpenCodex\" => enabled", + "loaded": { + "stdout": "gui/{UID}/OpenCodex = {\n\tactive count = 1\n\tpath = {PLIST}\n\tstate = running\n\tprogram = {APP}\n}" + }, + "expect": { + "tray": "unknown", + "loginEnabled": false + } + }, + { + "name": "print-disabled envelope is not closed", + "plist": { + "Label": "OpenCodex", + "RunAtLoad": true, + "ProgramArguments": [ + "{APP}", + "--autostart" + ] + }, + "disabled": "disabled services = {\n\t\"OpenCodex\" => enabled\n", + "loaded": { + "stdout": "gui/{UID}/OpenCodex = {\n\tactive count = 1\n\tpath = {PLIST}\n\tstate = running\n\tprogram = {APP}\n}" + }, + "expect": { + "tray": "unknown", + "loginEnabled": false + } + }, + { + "name": "OpenCodex mentioned twice", + "plist": { + "Label": "OpenCodex", + "RunAtLoad": true, + "ProgramArguments": [ + "{APP}", + "--autostart" + ] + }, + "disabled": "disabled services = {\n\t\"OpenCodex\" => enabled\n\t\"OpenCodex\" => disabled\n}", + "loaded": { + "stdout": "gui/{UID}/OpenCodex = {\n\tactive count = 1\n\tpath = {PLIST}\n\tstate = running\n\tprogram = {APP}\n}" + }, + "expect": { + "tray": "unknown", + "loginEnabled": false + } + }, + { + "name": "override value has a suffix", + "plist": { + "Label": "OpenCodex", + "RunAtLoad": true, + "ProgramArguments": [ + "{APP}", + "--autostart" + ] + }, + "disabled": "disabled services = {\n\t\"OpenCodex\" => enabledx\n}", + "loaded": { + "stdout": "gui/{UID}/OpenCodex = {\n\tactive count = 1\n\tpath = {PLIST}\n\tstate = running\n\tprogram = {APP}\n}" + }, + "expect": { + "tray": "unknown", + "loginEnabled": false + } + }, + { + "name": "override value is unrecognized", + "plist": { + "Label": "OpenCodex", + "RunAtLoad": true, + "ProgramArguments": [ + "{APP}", + "--autostart" + ] + }, + "disabled": "disabled services = {\n\t\"OpenCodex\" => maybe\n}", + "loaded": { + "stdout": "gui/{UID}/OpenCodex = {\n\tactive count = 1\n\tpath = {PLIST}\n\tstate = running\n\tprogram = {APP}\n}" + }, + "expect": { + "tray": "unknown", + "loginEnabled": false + } + }, + { + "name": "override value followed by text on the same line", + "plist": { + "Label": "OpenCodex", + "RunAtLoad": true, + "ProgramArguments": [ + "{APP}", + "--autostart" + ] + }, + "disabled": "disabled services = {\n\t\"OpenCodex\" => enabled later\n}", + "loaded": { + "stdout": "gui/{UID}/OpenCodex = {\n\tactive count = 1\n\tpath = {PLIST}\n\tstate = running\n\tprogram = {APP}\n}" + }, + "expect": { + "tray": "unknown", + "loginEnabled": false + } + }, + { + "name": "launchctl print fails for another reason", + "plist": { + "Label": "OpenCodex", + "RunAtLoad": true, + "ProgramArguments": [ + "{APP}", + "--autostart" + ] + }, + "disabled": "disabled services = {\n\t\"com.apple.example\" => disabled\n\t\"OpenCodex\" => enabled\n}", + "loaded": { + "status": 5, + "stderr": "Operation not permitted" + }, + "expect": { + "tray": "unknown", + "loginEnabled": false + } + }, + { + "name": "launchctl print times out", + "plist": { + "Label": "OpenCodex", + "RunAtLoad": true, + "ProgramArguments": [ + "{APP}", + "--autostart" + ] + }, + "disabled": "disabled services = {\n\t\"com.apple.example\" => disabled\n\t\"OpenCodex\" => enabled\n}", + "loaded": { + "failure": "timeout" + }, + "expect": { + "tray": "unknown", + "loginEnabled": false + } + }, + { + "name": "service-not-found result killed by a signal", + "plist": { + "Label": "OpenCodex", + "RunAtLoad": true, + "ProgramArguments": [ + "{APP}", + "--autostart" + ] + }, + "disabled": "disabled services = {\n\t\"com.apple.example\" => disabled\n\t\"OpenCodex\" => enabled\n}", + "loaded": { + "status": 113, + "stderr": "Could not find service \"OpenCodex\" in domain for user gui: {UID}", + "signal": "SIGKILL" + }, + "expect": { + "tray": "unknown", + "loginEnabled": false + } + }, + { + "name": "exit 113 with different stderr", + "plist": { + "Label": "OpenCodex", + "RunAtLoad": true, + "ProgramArguments": [ + "{APP}", + "--autostart" + ] + }, + "disabled": "disabled services = {\n\t\"com.apple.example\" => disabled\n\t\"OpenCodex\" => enabled\n}", + "loaded": { + "status": 113, + "stderr": "Could not find service \"OpenCodex\" in domain for user gui: 502" + }, + "expect": { + "tray": "unknown", + "loginEnabled": false + } + }, + { + "name": "exit 113 with extra stderr", + "plist": { + "Label": "OpenCodex", + "RunAtLoad": true, + "ProgramArguments": [ + "{APP}", + "--autostart" + ] + }, + "disabled": "disabled services = {\n\t\"com.apple.example\" => disabled\n\t\"OpenCodex\" => enabled\n}", + "loaded": { + "status": 113, + "stderr": "Could not find service \"OpenCodex\" in domain for user gui: {UID}\nextra" + }, + "expect": { + "tray": "unknown", + "loginEnabled": false + } + }, + { + "name": "override changes during the not-loaded re-read", + "plist": { + "Label": "OpenCodex", + "RunAtLoad": true, + "ProgramArguments": [ + "{APP}", + "--autostart" + ] + }, + "disabled": "disabled services = {\n\t\"com.apple.example\" => disabled\n\t\"OpenCodex\" => enabled\n}", + "loaded": { + "status": 113, + "stderr": "Could not find service \"OpenCodex\" in domain for user gui: {UID}" + }, + "disabledAfterFirstRead": "disabled services = {\n\t\"OpenCodex\" => disabled\n}", + "expect": { + "tray": "unknown", + "loginEnabled": false + } + }, + { + "name": "override text changes during the not-loaded re-read", + "plist": { + "Label": "OpenCodex", + "RunAtLoad": true, + "ProgramArguments": [ + "{APP}", + "--autostart" + ] + }, + "disabled": "disabled services = {\n\t\"com.apple.example\" => disabled\n\t\"OpenCodex\" => enabled\n}", + "loaded": { + "status": 113, + "stderr": "Could not find service \"OpenCodex\" in domain for user gui: {UID}" + }, + "disabledAfterFirstRead": "disabled services = {\n\t\"OpenCodex\" => enabled\n\t\"com.apple.new\" => disabled\n}", + "expect": { + "tray": "unknown", + "loginEnabled": false + } + }, + { + "name": "plist is rewritten while it is read", + "plist": { + "Label": "OpenCodex", + "RunAtLoad": true, + "ProgramArguments": [ + "{APP}", + "--autostart" + ] + }, + "disabled": "disabled services = {\n\t\"com.apple.example\" => disabled\n\t\"OpenCodex\" => enabled\n}", + "loaded": { + "stdout": "gui/{UID}/OpenCodex = {\n\tactive count = 1\n\tpath = {PLIST}\n\tstate = running\n\tprogram = {APP}\n}" + }, + "touchPlistOnRead": true, + "expect": { + "tray": "unknown", + "loginEnabled": false + } + }, + { + "name": "plist is a symlink and the job was loaded through it", + "plist": { + "Label": "OpenCodex", + "RunAtLoad": true, + "ProgramArguments": [ + "{APP}", + "--autostart" + ] + }, + "disabled": "disabled services = {\n\t\"com.apple.example\" => disabled\n\t\"OpenCodex\" => enabled\n}", + "loaded": { + "stdout": "gui/{UID}/OpenCodex = {\n\tactive count = 1\n\tpath = {PLIST}\n\tstate = running\n\tprogram = {APP}\n}" + }, + "plistLink": true, + "expect": { + "tray": "on", + "loginEnabled": true, + "loginRegistration": "loaded" + } + }, + { + "name": "plist is a symlink and not loaded yet", + "plist": { + "Label": "OpenCodex", + "RunAtLoad": true, + "ProgramArguments": [ + "{APP}", + "--autostart" + ] + }, + "disabled": "disabled services = {\n\t\"com.apple.example\" => disabled\n\t\"OpenCodex\" => enabled\n}", + "loaded": { + "status": 113, + "stderr": "Could not find service \"OpenCodex\" in domain for user gui: {UID}" + }, + "plistLink": true, + "expect": { + "tray": "on", + "loginEnabled": true, + "loginRegistration": "registered-not-loaded" + } + }, + { + "name": "launchctl print lines end in CR", + "plist": { + "Label": "OpenCodex", + "RunAtLoad": true, + "ProgramArguments": [ + "{APP}", + "--autostart" + ] + }, + "disabled": "disabled services = {\n\t\"com.apple.example\" => disabled\n\t\"OpenCodex\" => enabled\n}", + "loaded": { + "stdout": "gui/{UID}/OpenCodex = {\r\tpath = {PLIST}\r\tprogram = {APP}\r}" + }, + "expect": { + "tray": "on", + "loginEnabled": true, + "loginRegistration": "loaded" + } + }, + { + "name": "launchctl print lines end in U+2028", + "plist": { + "Label": "OpenCodex", + "RunAtLoad": true, + "ProgramArguments": [ + "{APP}", + "--autostart" + ] + }, + "disabled": "disabled services = {\n\t\"com.apple.example\" => disabled\n\t\"OpenCodex\" => enabled\n}", + "loaded": { + "stdout": "gui/{UID}/OpenCodex = {
\tpath = {PLIST}
\tprogram = {APP}
}" + }, + "expect": { + "tray": "on", + "loginEnabled": true, + "loginRegistration": "loaded" + } + }, + { + "name": "launchctl print lines end in U+2029", + "plist": { + "Label": "OpenCodex", + "RunAtLoad": true, + "ProgramArguments": [ + "{APP}", + "--autostart" + ] + }, + "disabled": "disabled services = {\n\t\"com.apple.example\" => disabled\n\t\"OpenCodex\" => enabled\n}", + "loaded": { + "stdout": "gui/{UID}/OpenCodex = {
\tpath = {PLIST}
\tprogram = {APP}
}" + }, + "expect": { + "tray": "on", + "loginEnabled": true, + "loginRegistration": "loaded" + } + }, + { + "name": "loaded program value stops at U+2028", + "plist": { + "Label": "OpenCodex", + "RunAtLoad": true, + "ProgramArguments": [ + "{APP}", + "--autostart" + ] + }, + "disabled": "disabled services = {\n\t\"com.apple.example\" => disabled\n\t\"OpenCodex\" => enabled\n}", + "loaded": { + "stdout": "gui/{UID}/OpenCodex = {\n\tpath = {PLIST}\n\tprogram = {APP}
tail\n}" + }, + "expect": { + "tray": "on", + "loginEnabled": true, + "loginRegistration": "loaded" + } + }, + { + "name": "loaded program value starts after a line terminator", + "plist": { + "Label": "OpenCodex", + "RunAtLoad": true, + "ProgramArguments": [ + "{APP}", + "--autostart" + ] + }, + "disabled": "disabled services = {\n\t\"com.apple.example\" => disabled\n\t\"OpenCodex\" => enabled\n}", + "loaded": { + "stdout": "gui/{UID}/OpenCodex = {\n\tpath = {PLIST}\n\tprogram = 
{APP}\n}" + }, + "expect": { + "tray": "unknown", + "loginEnabled": false + } + }, + { + "name": "loaded path value stops at CR", + "plist": { + "Label": "OpenCodex", + "RunAtLoad": true, + "ProgramArguments": [ + "{APP}", + "--autostart" + ] + }, + "disabled": "disabled services = {\n\t\"com.apple.example\" => disabled\n\t\"OpenCodex\" => enabled\n}", + "loaded": { + "stdout": "gui/{UID}/OpenCodex = {\n\tpath = {PLIST}\rtail\n\tprogram = {APP}\n}" + }, + "expect": { + "tray": "on", + "loginEnabled": true, + "loginRegistration": "loaded" + } + } + ] +} diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index f0baa213f6a..0da96d02b86 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -111,7 +111,7 @@ "codex-pool-credit-policy.test.ts": "codex-integration", "codex-sidecar-credit-policy.test.ts": "codex-integration", "service-runtime-preflight.test.ts": "service", "service-desktop-runtime-preflight.test.ts": "service", - "codex-credits-after-limit-main.test.ts": "codex-integration", "service-desktop-startup-health.test.ts": "service", "service-desktop-startup-linux.test.ts": "service", + "codex-credits-after-limit-main.test.ts": "codex-integration", "service-desktop-login-registration.test.ts": "service", "service-desktop-startup-health.test.ts": "service", "service-desktop-startup-linux.test.ts": "service", "service-desktop-startup.test.ts": "service", "startup-health-packaged-probe.test.ts": "server", "discovered-native-models.test.ts": "codex-integration", "responses-hosted-image-delivery.test.ts": "responses", "responses-hosted-image-display.test.ts": "responses", "responses-hosted-image-limits.test.ts": "responses", diff --git a/tests/service/service-desktop-login-registration.test.ts b/tests/service/service-desktop-login-registration.test.ts new file mode 100644 index 00000000000..a0b54f87afe --- /dev/null +++ b/tests/service/service-desktop-login-registration.test.ts @@ -0,0 +1,111 @@ +import { afterEach, expect, test } from "bun:test"; +import { chmodSync, mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { macLoginFactsFor } from "../../src/service/desktop-startup"; +import { fixturePath } from "../helpers/repo-root"; + +// One registration judgment, two implementations: Startup safety here and the Desktop tray's +// Start at Login item in desktop/src-tauri/src/login_registration.rs. Both suites run every case +// in this fixture, so neither side can drift without failing (#6972). +interface LoginCase { + name: string; + plist: Record | null; + plutilFails?: boolean; + proxy?: boolean; + plistLink?: boolean; + touchPlistOnRead?: boolean; + disabled: string | null; + disabledAfterFirstRead?: string; + loaded: { stdout: string } | { status: number; stderr: string; signal?: string } | { failure: string }; + expect: { tray: string; loginEnabled: boolean; loginRegistration?: "loaded" | "registered-not-loaded" }; +} + +const { cases } = JSON.parse(readFileSync(fixturePath("desktop-login-registration-cases.json"), "utf8")) as { cases: LoginCase[] }; +const roots: string[] = []; +afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); + +function fill(value: T, paths: Record): T { + if (typeof value === "string") return value.replace(/\{[A-Z_]+\}/g, key => paths[key.slice(1, -1)] ?? key) as T; + if (Array.isArray(value)) return value.map(item => fill(item, paths)) as T; + if (value && typeof value === "object") { + return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, fill(item, paths)])) as T; + } + return value; +} + +function judge(raw: LoginCase) { + const root = realpathSync(mkdtempSync(join(tmpdir(), "ocx-login-parity-"))); + roots.push(root); + const home = join(root, "home"); + const paths = { + APP: join(root, "OpenCodex.app", "Contents", "MacOS", "opencodex-desktop"), + OTHER_APP: join(root, "Other", "OpenCodex.app", "Contents", "MacOS", "opencodex-desktop"), + MISSING_APP: join(root, "Gone", "OpenCodex.app", "Contents", "MacOS", "opencodex-desktop"), + PLIST: join(home, "Library", "LaunchAgents", "OpenCodex.plist"), + OTHER_PLIST: join(home, "Other.plist"), + UID: "501", + }; + const c = fill(raw, paths); + const files = [paths.APP, paths.OTHER_APP, join(dirname(paths.OTHER_APP), "ocx"), paths.OTHER_PLIST]; + if (c.proxy !== false) files.push(join(dirname(paths.APP), "ocx")); + const realPlist = join(home, "Real.plist"); + if (c.plist !== null) files.push(c.plistLink ? realPlist : paths.PLIST); + for (const file of files) { + mkdirSync(dirname(file), { recursive: true }); + writeFileSync(file, "fixture"); + chmodSync(file, 0o700); + } + if (c.plist !== null && c.plistLink) { + mkdirSync(dirname(paths.PLIST), { recursive: true }); + symlinkSync(realPlist, paths.PLIST); + } + let disabledReads = 0; + const run = (command: string, args: string[]) => { + if (command === "/usr/bin/plutil") { + if (c.touchPlistOnRead) writeFileSync(paths.PLIST, "rewritten while it was read"); + if (c.plutilFails || c.plist === null) throw new Error("plutil failed"); + return JSON.stringify(c.plist); + } + if (command === "/bin/launchctl" && args[0] === "print-disabled") { + disabledReads++; + if (c.disabled === null) throw new Error("print-disabled failed"); + return disabledReads > 1 && c.disabledAfterFirstRead !== undefined ? c.disabledAfterFirstRead : c.disabled; + } + if (command === "/bin/launchctl" && args[0] === "print") { + if ("stdout" in c.loaded) return c.loaded.stdout; + if ("failure" in c.loaded) { + throw Object.assign(new Error(`launchctl ${c.loaded.failure}`), { code: "ETIMEDOUT", signal: "SIGTERM", status: null }); + } + throw Object.assign(new Error("launchctl print failed"), { + status: c.loaded.status, stderr: Buffer.from(c.loaded.stderr), signal: c.loaded.signal ?? null, + }); + } + throw new Error(`unexpected command: ${command}`); + }; + // diagnoseSupervisedStartup turns a thrown login probe into an uncredited registration. + try { + return macLoginFactsFor(realpathSync(paths.APP), { platform: "darwin", home, uid: 501, run }); + } catch { + return { loginEnabled: false }; + } +} + +// The judgment needs POSIX paths (the registered program must end in /Contents/MacOS/...) and +// symlinks, which Windows runners do not provide without privileges. +const parity = process.platform === "win32" ? test.skip : test; + +test("the shared fixture keeps the tray's checked state equal to Startup safety's credit", () => { + expect(cases.length).toBeGreaterThan(40); + for (const c of cases) { + expect({ name: c.name, checked: c.expect.tray === "on" }).toEqual({ name: c.name, checked: c.expect.loginEnabled }); + expect({ name: c.name, registration: c.expect.loginRegistration !== undefined }).toEqual({ name: c.name, registration: c.expect.loginEnabled }); + } +}); + +for (const c of cases) { + parity(`Startup safety login credit: ${c.name}`, () => { + const { loginEnabled, loginRegistration } = c.expect; + expect(judge(c)).toEqual({ loginEnabled, ...(loginRegistration ? { loginRegistration } : {}) }); + }); +} From b993e662c708c6f358f72fa56c2c191f30c4c874 Mon Sep 17 00:00:00 2001 From: lidge-jun Date: Tue, 13 Oct 2026 01:09:41 +0900 Subject: [PATCH 7/9] fix(link): read-only Home management, Home Codex settings and one CLI refusal on Remote Link Children (#6959) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Squashed from #6959 for the 2.84.0 release train. A Remote Link Child relayed inference to its Home, but its dashboard failed with "Dashboard data could not be loaded", its Codex `config.toml` kept its own OpenCodex-written settings, and its CLI answered with hub wording, `not_found` or "Hub does not support client usage". This PR gives the Child an authenticated, read-only view of its Home, applies the Home's OpenCodex-owned Codex settings when the Child syncs, and makes every management command either read the Home or refuse with one message. **Read-only management relay over the link** - `src/link/management-routes.ts` holds a shared, default-deny allowlist of Home management GET reads. A read is allowed only for a listed path, GET or HEAD, and a query without `refresh`, `reload`, `force` or `quota`. - The Home's hub-link listener serves those paths under `/v1/link-management/…`, after the existing link-key admission (only recorded link keys). `src/server/index/link-management.ts` builds a fresh loopback request with no caller headers, cookies, Origin or principal and returns only content headers with `no-store`. The prefix does not exist on the public listener. - The Child dashboard's shared plane becomes the Child listener's `/api/machine/hub-relay` base. The browser holds only the Child's own session (`/api/machine/hub-relay/opencodex-session` is answered locally), and `src/client/link-management-relay.ts` sends allowlisted reads through the existing connection-bound link relay, so the link key is attached only after the tunnel listener proves the link identity. Everything else gets `403 link_child_read_only` before a request exists. The Child's unauthenticated data-plane ingress still relays only the data-plane table. - The dashboard shows "Managed by Home over Remote Link ()", outside the page error boundary. The header quota strip now reads the Home's `/api/provider-quotas`. - The Home answers every link read as a passive read (`src/lib/passive-read.ts`): it shows what the Home already knows and does not make it act on providers, accounts, processes or files. Model discovery returns no routed models. Provider quota and Codex accounts come from the Home's caches without a probe or token refresh. The startup diagnostic, Codex version probes, bundled-catalog refresh and native-main ownership re-probe are not started. Credential-store hardening and malformed-file backups are skipped, and memory-budget eviction only drops caches registered as memory-only. `/api/injection-model` and `/api/sidecar-settings` therefore list native options plus the Home's current picks. **Home Codex settings (scope 1)** I checked which Codex keys OpenCodex writes. The injector reads four policy fields from OpenCodex config: `fastMode` (`[features] fast_mode`), `multiAgentMode` (the `features.multi_agent_v2` reconcile), `webSearchSidecar.enabled` (root `web_search`), and the effective managed sub-agent defaults (`syncCodexSubagentDefaults` + `injectionModel` + `injectionEffort`). Effort caps, the sub-agent roster and fallback, the injection prompt, sidecar and compaction routing are applied per request by the Home's proxy, so they already followed the Home. `src/link/codex-settings.ts` publishes those four on `/v1/link-management/codex-settings`. It is a separate authenticated read because the catalog bytes are written verbatim to `model_catalog_json`. Connect and `syncConnectedClient` overlay the strictly parsed document before injection. After the injection that applies it succeeds, a fresh read is recorded in `/link/home-codex-settings.json` (0600, keyed by link id, removed on disconnect). A failed read reuses the record, and an older Home keeps the Child's local values and says so. `ocx sync` and `ocx status` list which values came from the Home. The record's host name is also how a Home-initiated Child names its Home in messages. **CLI (scope 3)** `src/cli/link-child.ts` applies the same table to `runtimeRequest`, the account client, observation and `ocx debug`. Reads go to the Home over the bound relay. Everything else is refused with `linkChildManagementRefusal`: "This machine is a Remote Link Child managed by Home (link ). runs on the Home, not on this Child: use the Home's dashboard or run the command there. No request was sent." The text no longer mentions a relay transport. `ocx usage` reads the Home with Home wording, `ocx account list` reads the Home, `ocx status` describes a Remote Link Home (rebased onto #6955's local Child relay for `/v1/hub-state`), `ocx provider list|show` and `ocx config get` print a stderr note that the values are local and inactive, `ocx config set|unset|import` write locally with that note, and `ocx provider add|remove|set-default` are refused. Account commands classify providers from the Home's `/api/providers` projection, so Home-only providers are listed. Closes #6957 --- .../src/content/docs/guides/remote-link.md | 10 + gui/src/App.tsx | 8 + gui/src/api-targets.ts | 15 +- gui/src/i18n/de.ts | 2 + gui/src/i18n/en.ts | 2 + gui/src/i18n/fr.ts | 2 + gui/src/i18n/ja.ts | 2 + gui/src/i18n/ko.ts | 2 + gui/src/i18n/pt.ts | 2 + gui/src/i18n/ru.ts | 2 + gui/src/i18n/tr.ts | 2 + gui/src/i18n/vi.ts | 2 + gui/src/i18n/zh-TW.ts | 2 + gui/src/i18n/zh.ts | 2 + gui/tests/api-targets.test.ts | 18 + scripts/test-layout/layout.json | 4 +- src/cli/account-api.ts | 65 +++ src/cli/account.ts | 46 +- src/cli/catalog-command-result.ts | 3 +- src/cli/config-command.ts | 10 +- src/cli/debug.ts | 8 + src/cli/dispatch.ts | 19 +- src/cli/index.ts | 4 +- src/cli/link-child.ts | 104 ++++ src/cli/observe-stream.ts | 7 +- src/cli/observe.ts | 6 +- src/cli/provider.ts | 12 + src/cli/runtime-api.ts | 29 +- src/cli/status.ts | 74 ++- src/client/connect.ts | 43 +- src/client/hub-client.ts | 11 +- src/client/hub-state.ts | 30 +- src/client/link-home.ts | 248 +++++++++ src/client/link-management-relay.ts | 61 +++ src/client/link-relay.ts | 7 +- src/client/machine-api.ts | 21 +- src/client/machine-listener.ts | 33 +- src/codex/auth-api/account-list.ts | 19 +- src/codex/catalog/bundled.ts | 3 + src/codex/native-profile-startup.ts | 5 +- src/codex/runtime.ts | 4 +- src/config/load-degrade.ts | 3 +- src/config/paths.ts | 3 + src/config/salvage.ts | 4 +- src/lib/app-owned-memory-stores.ts | 4 + src/lib/app-owned-memory.ts | 11 + src/lib/passive-read.ts | 35 ++ src/link/codex-settings.ts | 155 ++++++ src/link/management-routes.ts | 89 ++++ src/providers/quota.ts | 5 +- src/providers/quota/report-cache.ts | 14 + src/server/index/link-management.ts | 58 +++ src/server/index/optional-listeners.ts | 6 +- src/server/index/serve-options.ts | 7 + .../management/agent-settings-routes.ts | 38 +- src/server/management/model-rows.ts | 4 +- src/server/management/shared.ts | 3 + src/server/startup-health-cache.ts | 4 + structure/gui-and-management-api.md | 14 +- structure/remote-link.md | 20 +- tests/cli/cli-link-child.test.ts | 185 +++++++ tests/clients/client-link-relay.test.ts | 3 +- tests/clients/client-machine-listener.test.ts | 71 ++- tests/clients/link-home-settings.test.ts | 195 +++++++ tests/fixtures/test-layout-expected.json | 2 + tests/server/link-management-relay.test.ts | 486 ++++++++++++++++++ 66 files changed, 2276 insertions(+), 92 deletions(-) create mode 100644 src/cli/link-child.ts create mode 100644 src/client/link-home.ts create mode 100644 src/client/link-management-relay.ts create mode 100644 src/lib/passive-read.ts create mode 100644 src/link/codex-settings.ts create mode 100644 src/link/management-routes.ts create mode 100644 src/server/index/link-management.ts create mode 100644 tests/cli/cli-link-child.test.ts create mode 100644 tests/clients/link-home-settings.test.ts create mode 100644 tests/server/link-management-relay.test.ts diff --git a/docs-site/src/content/docs/guides/remote-link.md b/docs-site/src/content/docs/guides/remote-link.md index 3a17e3610e9..e3dbbf673e3 100644 --- a/docs-site/src/content/docs/guides/remote-link.md +++ b/docs-site/src/content/docs/guides/remote-link.md @@ -65,6 +65,16 @@ A failed link does not silently switch to a local provider. The Child returns a retryable `503` without forwarding the link key or request when its tunnel is failed, stopped, or not supervised. It also keeps the key local when another process owns the tunnel port; free that port so the Child's SSH tunnel can bind it. +## What a Child shows and manages + +A Child runs on the Home's providers, accounts and settings, so the Home is where you change them. + +- **Dashboard.** The Child's dashboard shows the Home's state read-only and says "Managed by Home over Remote Link". Dashboard and Providers read the Home through the link, and the usage strip in the header shows the Home's quota. The model list is not shown on a Child, because building it runs provider discovery on the Home; Codex on the Child already offers the Home's models. For the same reason, the delegation and sidecar model pickers on a Child list the native models and the Home's current choices, not the Home's full routed list. Saving a change there is refused with a message that names the Home; make the change in the Home's dashboard. The Child's browser never receives a Home credential. +- **Codex settings.** `ocx sync` on the Child (and connecting it) applies the Codex settings OpenCodex manages on the Home: Fast mode, the multi-agent mode, the web search sidecar switch, and the native sub-agent defaults when the Home syncs them. `ocx sync` and `ocx status` list the values that came from the Home. The default reasoning effort and the effort levels shown in the Codex app's model picker are Codex app settings on each computer; OpenCodex does not manage them on any computer. +- **CLI.** Commands that only read, such as `ocx account list`, `ocx provider quota` and `ocx usage`, read the Home. They show what the Home has already observed: a read from a Child never makes the Home probe a provider, refresh a token or start a process, so quota and account details are as fresh as the Home's own activity keeps them. Reads that only exist to run provider discovery or a forced refresh, such as `ocx models live`, `ocx models preset show` and `ocx account list --quota`, are refused like changes. Commands that change configuration are refused with one message that names the Home and the link and says no request was sent. `ocx status` describes the Home and the link. `ocx provider list` and `ocx config get` still show this computer's own configuration, labelled as local and inactive while the computer is linked. `ocx config set`, `ocx config unset` and `ocx config import` still write locally, each with the same note, and `ocx provider add`, `remove` and `set-default` are refused because those providers are not used while linked. + +These reads need an OpenCodex version with the Remote Link management relay on both the Home and the Child. An older Home answers with a message asking you to upgrade it. + ## Remove a Child Select **Disconnect** for the Child and confirm the alias. The Home stops the tunnel, revokes that Child's link key, and removes the saved link record. diff --git a/gui/src/App.tsx b/gui/src/App.tsx index 1ffb929af2c..10becb34b20 100644 --- a/gui/src/App.tsx +++ b/gui/src/App.tsx @@ -569,6 +569,14 @@ export default function App() { onFocusOrphaned={focusAfterSwitcher} /> )} + {/* Outside the page boundary: a page that fails to render must not hide whose state this is. */} + {targetsSettled && targets.link && ( + + {targets.link.home + ? t("connection.linkManaged", { home: targets.link.home, linkId: targets.link.linkId }) + : t("connection.linkManagedUnnamed", { linkId: targets.link.linkId })} + + )} ; + if (typeof row.linkId !== "string" || !/^lnk_[0-9a-f]{16}$/.test(row.linkId)) return undefined; + const home = typeof row.home === "string" && row.home.trim() && row.home.length <= 255 ? row.home.trim() : null; + return { linkId: row.linkId, home }; +} + export function relayUrlForPath(shared: ApiTarget, path: string): string { if (shared.transport !== "relay" || (!path.startsWith("/api/") && path !== "/opencodex-session")) { throw new TypeError("path is not eligible for the fixed hub relay"); @@ -174,7 +186,8 @@ export function targetsFromMachineStatus(initialBase: string, status: MachineSta const shared = status.managementTransport === "relay" ? target("shared", `${trimBase(initialBase)}/api/machine/hub-relay`, sharedOrigin, "relay") : target("shared", sharedOrigin, sharedOrigin, "direct"); - return { connected: true, machine, shared, apiKeyId: status.apiKeyId, catalogSyncedAt: status.catalogSyncedAt }; + const link = linkFromStatus(status.link); + return { connected: true, machine, shared, apiKeyId: status.apiKeyId, catalogSyncedAt: status.catalogSyncedAt, ...(link ? { link } : {}) }; } export function apiBaseForPlane(plane: ApiPlane, targets: ApiTargets): string { diff --git a/gui/src/i18n/de.ts b/gui/src/i18n/de.ts index aadba5373ce..68b0360de00 100644 --- a/gui/src/i18n/de.ts +++ b/gui/src/i18n/de.ts @@ -3499,6 +3499,8 @@ export const de: Record = { "models.aliasStale": "veraltet", "connection.discovering": "Discovering local and shared targets…", "connection.machineUnavailable": "The local machine plane is unavailable. Shared requests were not redirected locally.", + "connection.linkManaged": "Verwaltet von Home {home} über Remote Link ({linkId}). Dieses Dashboard zeigt die Einstellungen und das Kontingent des Home nur lesend an; Änderungen erfolgen auf dem Home.", + "connection.linkManagedUnnamed": "Verwaltet von seinem Home über Remote Link ({linkId}). Dieses Dashboard zeigt die Einstellungen und das Kontingent des Home nur lesend an; Änderungen erfolgen auf dem Home.", "connection.disconnect": "Disconnect from hub", "connection.disconnectConfirm": "Disconnect this machine from the hub and restart it in standalone mode?", "connection.pairing.hub": "Hub", diff --git a/gui/src/i18n/en.ts b/gui/src/i18n/en.ts index d08ea3f72a1..08dca00a7d3 100644 --- a/gui/src/i18n/en.ts +++ b/gui/src/i18n/en.ts @@ -3579,6 +3579,8 @@ export const en = { "models.aliasStale": "stale", "connection.discovering": "Discovering local and shared targets…", "connection.machineUnavailable": "The local machine plane is unavailable. Shared requests were not redirected locally.", + "connection.linkManaged": "Managed by Home {home} over Remote Link ({linkId}). This dashboard shows the Home's settings and quota read-only; make changes on the Home.", + "connection.linkManagedUnnamed": "Managed by its Home over Remote Link ({linkId}). This dashboard shows the Home's settings and quota read-only; make changes on the Home.", "connection.disconnect": "Disconnect from hub", "connection.disconnectConfirm": "Disconnect this machine from the hub and restart it in standalone mode?", "connection.pairing.hub": "Hub", diff --git a/gui/src/i18n/fr.ts b/gui/src/i18n/fr.ts index a4d9eb8c87d..693b979549c 100644 --- a/gui/src/i18n/fr.ts +++ b/gui/src/i18n/fr.ts @@ -3488,6 +3488,8 @@ export const fr: Record = { "models.aliasStale": "obsolète", "connection.discovering": "Détection des cibles locale et partagée…", "connection.machineUnavailable": "Le plan machine local est indisponible. Les requêtes partagées n'ont pas été redirigées localement.", + "connection.linkManaged": "Géré par le Home {home} via Remote Link ({linkId}). Ce tableau de bord affiche les paramètres et le quota du Home en lecture seule ; effectuez les modifications sur le Home.", + "connection.linkManagedUnnamed": "Géré par son Home via Remote Link ({linkId}). Ce tableau de bord affiche les paramètres et le quota du Home en lecture seule ; effectuez les modifications sur le Home.", "connection.disconnect": "Déconnecter du hub", "connection.disconnectConfirm": "Déconnecter cette machine du hub et la redémarrer en mode autonome ?", "connection.pairing.hub": "Hub", diff --git a/gui/src/i18n/ja.ts b/gui/src/i18n/ja.ts index 242ce560b21..ffb91a9b2f7 100644 --- a/gui/src/i18n/ja.ts +++ b/gui/src/i18n/ja.ts @@ -3521,6 +3521,8 @@ export const ja: Record = { "models.aliasStale": "古い", "connection.discovering": "Discovering local and shared targets…", "connection.machineUnavailable": "The local machine plane is unavailable. Shared requests were not redirected locally.", + "connection.linkManaged": "Remote Link ({linkId}) 経由で Home {home} が管理しています。このダッシュボードは Home の設定とクォータを読み取り専用で表示します。変更は Home で行ってください。", + "connection.linkManagedUnnamed": "Remote Link ({linkId}) 経由で Home が管理しています。このダッシュボードは Home の設定とクォータを読み取り専用で表示します。変更は Home で行ってください。", "connection.disconnect": "Disconnect from hub", "connection.disconnectConfirm": "Disconnect this machine from the hub and restart it in standalone mode?", "connection.pairing.hub": "ハブ", diff --git a/gui/src/i18n/ko.ts b/gui/src/i18n/ko.ts index a846a96ae00..8b953c2125b 100644 --- a/gui/src/i18n/ko.ts +++ b/gui/src/i18n/ko.ts @@ -3521,6 +3521,8 @@ export const ko: Record = { "models.aliasStale": "오래됨", "connection.discovering": "로컬 및 공유 대상을 확인하는 중…", "connection.machineUnavailable": "로컬 머신 연결을 사용할 수 없습니다. 공유 요청을 로컬로 우회하지 않았습니다.", + "connection.linkManaged": "Remote Link({linkId})를 통해 Home {home}이(가) 관리합니다. 이 대시보드는 Home의 설정과 할당량을 읽기 전용으로 보여 줍니다. 변경은 Home에서 하세요.", + "connection.linkManagedUnnamed": "Remote Link({linkId})를 통해 Home이 관리합니다. 이 대시보드는 Home의 설정과 할당량을 읽기 전용으로 보여 줍니다. 변경은 Home에서 하세요.", "connection.disconnect": "허브 연결 해제", "connection.disconnectConfirm": "이 머신의 허브 연결을 해제하고 독립 실행 모드로 다시 시작할까요?", "connection.pairing.hub": "허브", diff --git a/gui/src/i18n/pt.ts b/gui/src/i18n/pt.ts index 29d76fd860a..22d8ef0df30 100644 --- a/gui/src/i18n/pt.ts +++ b/gui/src/i18n/pt.ts @@ -3540,6 +3540,8 @@ export const pt: Record = { "models.aliasStale": "obsoleto", "connection.discovering": "Descobrindo destinos locais e compartilhados…", "connection.machineUnavailable": "O plano da máquina local está indisponível. As requisições compartilhadas não foram redirecionadas para o local.", + "connection.linkManaged": "Gerenciado pelo Home {home} via Remote Link ({linkId}). Este painel mostra as configurações e a cota do Home somente para leitura; faça as alterações no Home.", + "connection.linkManagedUnnamed": "Gerenciado pelo seu Home via Remote Link ({linkId}). Este painel mostra as configurações e a cota do Home somente para leitura; faça as alterações no Home.", "connection.disconnect": "Desconectar do hub", "connection.disconnectConfirm": "Desconectar esta máquina do hub e reiniciá-la no modo standalone?", "connection.pairing.hub": "Hub", diff --git a/gui/src/i18n/ru.ts b/gui/src/i18n/ru.ts index 92891a8c5df..0762efdb1ce 100644 --- a/gui/src/i18n/ru.ts +++ b/gui/src/i18n/ru.ts @@ -3522,6 +3522,8 @@ export const ru: Record = { "models.aliasStale": "устарел", "connection.discovering": "Discovering local and shared targets…", "connection.machineUnavailable": "The local machine plane is unavailable. Shared requests were not redirected locally.", + "connection.linkManaged": "Управляется Home {home} через Remote Link ({linkId}). Эта панель показывает настройки и квоту Home только для чтения; вносите изменения на Home.", + "connection.linkManagedUnnamed": "Управляется вашим Home через Remote Link ({linkId}). Эта панель показывает настройки и квоту Home только для чтения; вносите изменения на Home.", "connection.disconnect": "Disconnect from hub", "connection.disconnectConfirm": "Disconnect this machine from the hub and restart it in standalone mode?", "connection.pairing.hub": "Хаб", diff --git a/gui/src/i18n/tr.ts b/gui/src/i18n/tr.ts index e9a667b53f9..f0a340c7ef9 100644 --- a/gui/src/i18n/tr.ts +++ b/gui/src/i18n/tr.ts @@ -3522,6 +3522,8 @@ export const tr: Record = { "models.aliasStale": "eski", "connection.discovering": "Discovering local and shared targets…", "connection.machineUnavailable": "The local machine plane is unavailable. Shared requests were not redirected locally.", + "connection.linkManaged": "Remote Link ({linkId}) üzerinden Home {home} tarafından yönetiliyor. Bu pano Home'un ayarlarını ve kotasını salt okunur gösterir; değişiklikleri Home üzerinde yapın.", + "connection.linkManagedUnnamed": "Remote Link ({linkId}) üzerinden Home tarafından yönetiliyor. Bu pano Home'un ayarlarını ve kotasını salt okunur gösterir; değişiklikleri Home üzerinde yapın.", "connection.disconnect": "Disconnect from hub", "connection.disconnectConfirm": "Disconnect this machine from the hub and restart it in standalone mode?", "connection.pairing.hub": "Merkez", diff --git a/gui/src/i18n/vi.ts b/gui/src/i18n/vi.ts index 2c51b8a5206..4d69cd01acb 100644 --- a/gui/src/i18n/vi.ts +++ b/gui/src/i18n/vi.ts @@ -3496,6 +3496,8 @@ export const vi: Record = { "models.aliasStale": "cũ (stale)", "connection.discovering": "Đang khám phá các mục tiêu cục bộ (local) và được chia sẻ…", "connection.machineUnavailable": "Mặt phẳng máy cục bộ (local machine plane) không khả dụng. Các yêu cầu chia sẻ đã không được chuyển hướng cục bộ.", + "connection.linkManaged": "Được quản lý bởi Home {home} qua Remote Link ({linkId}). Bảng điều khiển này hiển thị cài đặt và hạn mức của Home ở chế độ chỉ đọc; hãy thay đổi trên Home.", + "connection.linkManagedUnnamed": "Được quản lý bởi Home của nó qua Remote Link ({linkId}). Bảng điều khiển này hiển thị cài đặt và hạn mức của Home ở chế độ chỉ đọc; hãy thay đổi trên Home.", "connection.disconnect": "Ngắt kết nối khỏi hub", "connection.disconnectConfirm": "Ngắt kết nối máy này khỏi hub và khởi động lại ở chế độ độc lập (standalone mode)?", "connection.pairing.hub": "Hub", diff --git a/gui/src/i18n/zh-TW.ts b/gui/src/i18n/zh-TW.ts index 12c3442b30c..2096819f15f 100644 --- a/gui/src/i18n/zh-TW.ts +++ b/gui/src/i18n/zh-TW.ts @@ -3485,6 +3485,8 @@ export const zhTW: Record = { "models.aliasStale": "過期", "connection.discovering": "正在探索本機與共享目標…", "connection.machineUnavailable": "本機機器平面無法使用。共享請求未改用本機資料。", + "connection.linkManaged": "由 Home {home} 透過 Remote Link（{linkId}）管理。此儀表板以唯讀方式顯示 Home 的設定與配額；請在 Home 上進行變更。", + "connection.linkManagedUnnamed": "由其 Home 透過 Remote Link（{linkId}）管理。此儀表板以唯讀方式顯示 Home 的設定與配額；請在 Home 上進行變更。", "connection.disconnect": "中斷 Hub 連線", "connection.disconnectConfirm": "要中斷此機器與 Hub 的連線，並以獨立模式重新啟動嗎？", "connection.pairing.hub": "中樞", diff --git a/gui/src/i18n/zh.ts b/gui/src/i18n/zh.ts index c545376a2ed..26beec97dbd 100644 --- a/gui/src/i18n/zh.ts +++ b/gui/src/i18n/zh.ts @@ -3520,6 +3520,8 @@ export const zh: Record = { "models.aliasStale": "过期", "connection.discovering": "Discovering local and shared targets…", "connection.machineUnavailable": "The local machine plane is unavailable. Shared requests were not redirected locally.", + "connection.linkManaged": "由 Home {home} 通过 Remote Link（{linkId}）管理。此仪表板以只读方式显示 Home 的设置和配额；请在 Home 上进行更改。", + "connection.linkManagedUnnamed": "由其 Home 通过 Remote Link（{linkId}）管理。此仪表板以只读方式显示 Home 的设置和配额；请在 Home 上进行更改。", "connection.disconnect": "Disconnect from hub", "connection.disconnectConfirm": "Disconnect this machine from the hub and restart it in standalone mode?", "connection.pairing.hub": "中心", diff --git a/gui/tests/api-targets.test.ts b/gui/tests/api-targets.test.ts index 4ebefa54cd2..6973b3ec1aa 100644 --- a/gui/tests/api-targets.test.ts +++ b/gui/tests/api-targets.test.ts @@ -87,6 +87,24 @@ describe("two-plane API targets", () => { await expect(discoverApiTargets("")).rejects.toThrow("local machine plane unavailable"); }); + test("a Remote Link Child's relay targets this machine and carries the Home it is managed by", () => { + const machine = "http://localhost"; + const linked = targetsFromMachineStatus("", { + ...status("relay"), + sharedBase: `${machine}/api/machine/hub-relay`, + sharedServerOrigin: machine, + link: { linkId: `lnk_${"a".repeat(16)}`, home: "home-box", readOnly: true }, + }); + expect(linked.shared).toMatchObject({ baseUrl: "/api/machine/hub-relay", serverOrigin: machine, transport: "relay" }); + expect(linked.link).toEqual({ linkId: `lnk_${"a".repeat(16)}`, home: "home-box" }); + const malformed = targetsFromMachineStatus("", { + ...status("relay"), + link: { linkId: "not-a-link", home: "x", readOnly: true }, + }); + expect(malformed.link).toBeUndefined(); + expect(targetsFromMachineStatus("", status("relay")).link).toBeUndefined(); + }); + test("rejects a non-string catalog fetch timestamp", async () => { globalThis.fetch = (async () => new Response(JSON.stringify({ ...status("direct"), catalogSyncedAt: 42 }), { headers: { "content-type": "application/json" }, diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index 1cecf7b1f02..6761fb7e741 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -125,7 +125,7 @@ "abort-race.test.ts": "adapters", "account-import.test.ts": "server", "account-pool-management-api.test.ts": "server", - "link-listener-admission.test.ts": "server", + "link-listener-admission.test.ts": "server", "link-management-relay.test.ts": "server", "link-listener-lifecycle.test.ts": "server", "link-listener-mutation.test.ts": "server", "link-join-route.test.ts": "server", "core-link-boundary.test.ts": "lab", @@ -518,7 +518,7 @@ "client-hub-usage.test.ts": "clients", "client-injection-guard.test.ts": "codex-integration", "client-lifecycle-lock.test.ts": "clients", - "client-machine-listener.test.ts": "clients", + "client-machine-listener.test.ts": "clients", "link-home-settings.test.ts": "clients", "cli-link-child.test.ts": "cli", "client-runtime.test.ts": "clients", "cline-client.test.ts": "clients", "cline-pass-deepseek-v4-tool-replay.test.ts": "providers", diff --git a/src/cli/account-api.ts b/src/cli/account-api.ts index 2a73dc1d7f5..c4ad6cb4abe 100644 --- a/src/cli/account-api.ts +++ b/src/cli/account-api.ts @@ -137,6 +137,7 @@ export async function apiJson( body?: unknown, options: { signal?: AbortSignal; redirect?: RequestRedirect } = {}, ): Promise { + if (baseUrl === LINK_HOME_BASE) return linkHomeApiJson(deps, method, path); const fetchImpl = deps.fetchImpl ?? fetch; try { const res = await fetchImpl(`${baseUrl}${path}`, { @@ -163,9 +164,73 @@ export async function resolveBaseUrl(deps: AccountDeps): Promise if (deps.baseUrl) return deps.baseUrl; const live = await findLiveProxy(); if (!live) return null; + if (live.role === "client") { + const { readLinkChildContext } = await import("./link-child"); + if (readLinkChildContext()) return LINK_HOME_BASE; + } return `http://${probeHostname(live.hostname)}:${live.port}`; } +/** + * Remote Link Child (#6957): account reads come from the Home over the link, and every account + * change is refused with the Child refusal. The marker is never a URL and never reaches fetch. + */ +export const LINK_HOME_BASE = "ocx-link-home:"; + +async function linkHomeApiJson(deps: AccountDeps, method: string, path: string): Promise { + const { readLinkChildContext, linkChildReadable, linkChildRefusal, readLinkHomeJson } = await import("./link-child"); + const link = readLinkChildContext(); + if (!link) return { status: 503, json: { error: "The Remote Link connection changed while the command ran; retry." } }; + const pathname = new URL(path, "http://cli.invalid").pathname; + if (!linkChildReadable(method, path)) { + return { status: 403, json: { error: linkChildRefusal(link, `${method} ${pathname}`) } }; + } + try { + const { status, body } = await readLinkHomeJson(link, path, deps.fetchImpl ? { fetchImpl: deps.fetchImpl } : {}); + return { status, json: (body && typeof body === "object" ? body : { error: String(body ?? "") }) as Record }; + } catch (error) { + return { status: 503, json: { error: error instanceof Error ? error.message : String(error) } }; + } +} + +/** + * The config account commands classify providers against. On a Remote Link Child the local + * providers are inactive, so the Home's `/api/providers` projection (presence flags only) takes + * their place. Anywhere else the local config is returned. When the Home cannot be read the + * result is an error: classifying against the inactive local providers would answer for the + * wrong machine. + */ +export async function accountClassificationConfig( + deps: AccountDeps, + local: OcxConfig, +): Promise<{ config: OcxConfig } | { error: string }> { + if (deps.baseUrl) return { config: local }; + const { readLinkChildContext, readLinkHomeJson } = await import("./link-child"); + const link = readLinkChildContext(); + if (!link) return { config: local }; + const unavailable = (detail: string) => ({ error: `Could not read the provider list from ${link.homePhrase} over Remote Link (${link.linkId}): ${detail}.` }); + try { + const { status, body } = await readLinkHomeJson(link, "/api/providers", deps.fetchImpl ? { fetchImpl: deps.fetchImpl } : {}); + if (status !== 200) { + // Keep the Home's own sentence (for example the upgrade instruction an older Home yields). + const message = body && typeof body === "object" && typeof (body as { error?: unknown }).error === "string" + ? (body as { error: string }).error : null; + return unavailable(message ?? `the Home answered HTTP ${status}`); + } + if (!Array.isArray(body)) return unavailable("the Home returned an unreadable provider list"); + const providers: Record = {}; + for (const row of body) { + if (!row || typeof row !== "object" || typeof (row as { name?: unknown }).name !== "string") continue; + const { name, hasApiKey, ...rest } = row as Record & { name: string }; + // A placeholder marks presence for classification; no credential value crosses the link. + providers[name] = { ...rest, ...(hasApiKey === true ? { apiKey: "home-credential" } : {}) }; + } + return { config: { ...local, providers: providers as OcxConfig["providers"] } }; + } catch (error) { + return { error: error instanceof Error ? error.message : String(error) }; + } +} + export function proxyUnreachable(transportError?: string): number { console.error("Proxy not reachable. Start it with 'ocx start' or 'ocx ensure'."); // Naming the transport cause distinguishes "nothing is listening" from a refused diff --git a/src/cli/account.ts b/src/cli/account.ts index 1ff8b307ef7..09ddaf81c40 100644 --- a/src/cli/account.ts +++ b/src/cli/account.ts @@ -21,7 +21,7 @@ import { cmdRoutes, cmdStrategy, } from "./account-extended"; -import { apiError, apiJson, classifyAccount, fetchRows, proxyUnreachable, resolveBaseUrl, type AccountDeps, type AccountRow, type AccountType, type ApiResult } +import { accountClassificationConfig, apiError, apiJson, classifyAccount, fetchRows, proxyUnreachable, resolveBaseUrl, type AccountDeps, type AccountRow, type AccountType, type ApiResult } from "./account-api"; export { classifyAccount } from "./account-api"; @@ -199,7 +199,12 @@ async function cmdList(rest: string[], deps: AccountDeps): Promise { console.error(ACCOUNT_USAGE); return 1; } - const config = deps.loadConfigImpl?.() ?? loadConfig(); + const classification = await accountClassificationConfig(deps, deps.loadConfigImpl?.() ?? loadConfig()); + if ("error" in classification) { + console.error(`Error: ${classification.error}`); + return 1; + } + const config = classification.config; const baseUrl = await resolveBaseUrl(deps); if (!baseUrl) return proxyUnreachable(); @@ -289,7 +294,12 @@ async function cmdCurrent(rest: string[], deps: AccountDeps): Promise { console.error(ACCOUNT_USAGE); return 1; } - const config = deps.loadConfigImpl?.() ?? loadConfig(); + const classification = await accountClassificationConfig(deps, deps.loadConfigImpl?.() ?? loadConfig()); + if ("error" in classification) { + console.error(`Error: ${classification.error}`); + return 1; + } + const config = classification.config; const c = classifyAccount(config, name); if ("error" in c) { console.error(`Error: ${c.error}. Known candidates: ${candidateNames(config)}`); @@ -332,7 +342,20 @@ async function cmdUse(rest: string[], deps: AccountDeps): Promise { console.error(ACCOUNT_USAGE); return 1; } - const config = deps.loadConfigImpl?.() ?? loadConfig(); + // #6957: an account change on a link Child is refused before any Home read. + if (!deps.baseUrl) { + const refusal = (await import("./link-child")).linkChildProviderChangeRefusal("ocx account use"); + if (refusal) { + console.error(`Error: ${refusal}`); + return 1; + } + } + const classification = await accountClassificationConfig(deps, deps.loadConfigImpl?.() ?? loadConfig()); + if ("error" in classification) { + console.error(`Error: ${classification.error}`); + return 1; + } + const config = classification.config; const c = classifyAccount(config, name); if ("error" in c) { console.error(`Error: ${c.error}. Known candidates: ${candidateNames(config)}`); @@ -392,7 +415,20 @@ async function cmdClear(rest: string[], deps: AccountDeps): Promise { console.error(ACCOUNT_USAGE); return 1; } - const config = deps.loadConfigImpl?.() ?? loadConfig(); + // #6957: an account change on a link Child is refused before any Home read. + if (!deps.baseUrl) { + const refusal = (await import("./link-child")).linkChildProviderChangeRefusal("ocx account clear"); + if (refusal) { + console.error(`Error: ${refusal}`); + return 1; + } + } + const classification = await accountClassificationConfig(deps, deps.loadConfigImpl?.() ?? loadConfig()); + if ("error" in classification) { + console.error(`Error: ${classification.error}`); + return 1; + } + const config = classification.config; const c = classifyAccount(config, name); if ("error" in c) { console.error(`Error: ${c.error}. Known candidates: ${candidateNames(config)}`); diff --git a/src/cli/catalog-command-result.ts b/src/cli/catalog-command-result.ts index 5abaccf18c7..1bcd8f057d6 100644 --- a/src/cli/catalog-command-result.ts +++ b/src/cli/catalog-command-result.ts @@ -23,7 +23,8 @@ export async function runCatalogAction( if (error instanceof RuntimeApiError) { if (error.code) { console.error(`Error: ${error.code === "proxy_not_running" ? PROXY_NOT_RUNNING_MESSAGE - : "This listener has no management API. Run management commands on the connected Hub. No request was sent."}`); + : error.code === "link_child_management_unavailable" ? terminalSafeText(error.message) + : "This listener has no management API. Run management commands on the connected Hub. No request was sent."}`); return 1; } const code = own(error.body, "code") ?? own(own(error.body, "error"), "code"); diff --git a/src/cli/config-command.ts b/src/cli/config-command.ts index 9f3e1285efd..5c75a19e5a6 100644 --- a/src/cli/config-command.ts +++ b/src/cli/config-command.ts @@ -60,11 +60,14 @@ export function remoteHubConfigNote( // alone describes a machine that cannot read its hub, and `ocx status` is still the command // that has the facts — so the note points there in every case, connected or not. const connected = connection.state === "connected" && connection.token === "owned"; + const link = config.client.transport === "link"; const note = connection.state !== "connected" ? `this machine is configured as a client but its connection is ${connection.state}${connection.reason ? ` (${connection.reason})` : ""}; run ocx connect status` : connection.token !== "owned" - ? `this machine is configured as a client but its hub data-plane token is ${connection.token}; run ocx connect status` - : "provider credentials and model availability live on the hub; run ocx status"; + ? `this machine is configured as a client but its ${link ? "Remote Link key" : "hub data-plane token"} is ${connection.token}; run ocx connect status` + : link + ? "this machine is a Remote Link Child: provider credentials, model availability and the Codex settings OpenCodex writes come from its Home; the settings in this file are local and inactive while linked; run ocx status" + : "provider credentials and model availability live on the hub; run ocx status"; return { connected, origin: config.client.serverUrl, note }; } @@ -235,6 +238,7 @@ export async function handleConfigCommand(argv: string[]): Promise { const path = args.shift(); if (!path) throw new CliUsageError("config path is required", USAGE); rejectArgs(args, USAGE); + await (await import("./link-child")).printLinkChildLocalNotice("The values"); const diagnostics = readConfigDiagnostics(); if (diagnostics.source === "fallback" || diagnostics.error) { console.error(`Warning: Config at ${getConfigPath()} is invalid or unreadable; defaults are being shown for invalid settings. Run: ocx config validate. Inspect: ocx config show --source.`); @@ -246,6 +250,7 @@ export async function handleConfigCommand(argv: string[]): Promise { return; } if (action === "set" || action === "unset") { + await (await import("./link-child")).printLinkChildLocalWriteNotice(); const path = args.shift(); const raw = action === "set" ? args.shift() : undefined; if (!path || (action === "set" && raw === undefined)) throw new CliUsageError(action === "unset" ? "config path is required" : "config path and value are required", USAGE); @@ -316,6 +321,7 @@ export async function handleConfigCommand(argv: string[]): Promise { return; } if (action === "import") { + await (await import("./link-child")).printLinkChildLocalWriteNotice(); const path = args.shift(); const yes = takeFlag(args, "--yes"); if (!path) throw new CliUsageError("import path is required", USAGE); diff --git a/src/cli/debug.ts b/src/cli/debug.ts index 5c815fab6c4..88a7386447e 100644 --- a/src/cli/debug.ts +++ b/src/cli/debug.ts @@ -10,6 +10,14 @@ async function requireLiveProxy() { console.error("Proxy is not running. Start it with: ocx start"); process.exit(1); } + if (live.role === "client") { + // Debug capture is the serving proxy's; a connected client has no management API (#6957). + const { readLinkChildContext, linkChildRefusal } = await import("./link-child"); + const link = readLinkChildContext(); + console.error(link ? linkChildRefusal(link, "ocx debug") + : "This listener runs in the client role and has no management API. Run this command on its serving hub. No request was sent."); + process.exit(1); + } return live; } diff --git a/src/cli/dispatch.ts b/src/cli/dispatch.ts index 6b0d75dfa24..bb14a39e7ad 100644 --- a/src/cli/dispatch.ts +++ b/src/cli/dispatch.ts @@ -31,7 +31,7 @@ import { restoreNativeCodexAsync, type CodexNativeRestoreResult } from "../codex import { stripGrokConfig } from "../grok/inject"; import { handleRestartScopeAfterWrite, readRestartScope, type RestartScope } from "./restart-scope"; import { normalizeUpdateChannel, runGuiUpdateWorker } from "../update/job"; -import { isJsonOption, takeFlag, terminalSafeError } from "./runtime-api"; +import { isJsonOption, takeFlag, terminalSafeError, terminalSafeText } from "./runtime-api"; import { printStopSummary, type StopOutcome } from "./stop-report"; import { parseStopApproval, type StopApproval } from "./stop-approval"; import type { ResolveArgs } from "./resolve"; @@ -407,9 +407,22 @@ const commandRunners: Record = { try { const { syncConnectedClient } = await import("../client/connect"); const result = await syncConnectedClient({ restartCodex }); + const link = clientState.value.transport === "link"; console.log(result.stale - ? "Hub unavailable; retained and applied the last-known-good remote catalog (stale)." - : "Remote hub catalog synchronized."); + ? `${link ? "Home" : "Hub"} unavailable; retained and applied the last-known-good remote catalog (stale).` + : link ? "Remote Link Home catalog synchronized." : "Remote hub catalog synchronized."); + if (result.homeSettings) { + const { describeHomeCodexSettings } = await import("../link/codex-settings"); + const settings = result.homeSettings; + if (settings.source === "local") { + console.log(`Codex settings: kept this machine's local values (${terminalSafeText(settings.reason)}).`); + } else { + console.log(settings.source === "home" + ? `Codex settings applied from Home ${terminalSafeText(settings.document.home.name)}:` + : `Codex settings applied from the last recorded Home read (${terminalSafeText(settings.reason)}):`); + for (const line of describeHomeCodexSettings(settings.document.settings)) console.log(` ${terminalSafeText(line)}`); + } + } await handleConnectedSyncCatalogWrite(result, restartScope); // `process.exitCode` rather than a literal 0, for the same reason every other // runner does it (tests/cli/cli-transport-honesty.test.ts): the catalog-write helper diff --git a/src/cli/index.ts b/src/cli/index.ts index b44a9755b07..6884b00891e 100755 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -1803,7 +1803,9 @@ async function handleStatus() { if (status.json.hub) { for (const line of hubStatusLines(status.json.hub)) console.log(` ${line}`); } - console.log(` ${status.json.connection.transport === "link" ? "Remote Link Home" : "Remote hub"}: ${status.json.connection.state}${status.json.connection.serverUrl ? ` (${status.json.connection.serverUrl})` : ""}`); + console.log(status.json.remoteHub.link + ? ` Remote Link Home: ${status.json.connection.state} (${status.json.remoteHub.link.homePhrase}, ${status.json.remoteHub.link.linkId})` + : ` ${status.json.connection.transport === "link" ? "Remote Link Home" : "Remote hub"}: ${status.json.connection.state}${status.json.connection.serverUrl ? ` (${status.json.connection.serverUrl})` : ""}`); if (status.json.connection.state === "invalid" || status.json.connection.state === "mismatched") { console.log(` ⚠️ ${status.json.connection.reason}`); } diff --git a/src/cli/link-child.ts b/src/cli/link-child.ts new file mode 100644 index 00000000000..5f7d19603bc --- /dev/null +++ b/src/cli/link-child.ts @@ -0,0 +1,104 @@ +/** + * Management commands on a Remote Link Child (#6957). + * + * A link Child's listener serves no management API, and the Home owns every setting that decides + * routing. Each management command therefore does one of two things: an allowlisted read goes + * to the Home through the connection-bound link relay, and anything else is refused with the one + * message `linkChildManagementRefusal` builds, which names the Home and the link. + */ +import { readBoundedResponseBytes } from "../lib/bounded-body"; +import { readServiceApiTokenState } from "../lib/service-secrets"; +import { linkChildManagementRefusal, linkManagementReadAllowed } from "../link/management-routes"; +import { readClientConnectionState } from "../client/state"; +import { fetchLinkHomeManagement, linkHomePhrase, type LinkHomeDeps } from "../client/link-home"; +import type { OcxClientConnectionConfig } from "../types"; + +const RESPONSE_MAX_BYTES = 32 * 1024 * 1024; + +export interface LinkChildContext { + connection: OcxClientConnectionConfig; + linkId: string; + homePhrase: string; +} + +/** The link Child this CLI runs on, or null on a standalone, a hub or a hub client. */ +export function readLinkChildContext(read: typeof readClientConnectionState = readClientConnectionState): LinkChildContext | null { + let state: ReturnType; + try { state = read(); } catch { return null; } + if (state.kind !== "connected" || state.value.transport !== "link" || !state.value.link) return null; + return { connection: state.value, linkId: state.value.link.linkId, homePhrase: linkHomePhrase(state.value) }; +} + +export function linkChildRefusal(context: LinkChildContext, operation?: string): string { + return linkChildManagementRefusal(context.homePhrase, context.linkId, operation); +} + +/** + * The stderr label for a local read on a link Child (`ocx provider list`, `ocx config get`): + * the values are real, but they are this machine's and the Home's apply. + */ +export function linkChildLocalNotice(what: string): string | null { + const link = readLinkChildContext(); + if (!link) return null; + return `Note: this machine is a Remote Link Child managed by ${link.homePhrase} (link ${link.linkId}). ` + + `${what} below are this machine's local configuration. Its provider, model and routing settings are inactive while it is linked; the Home's apply. Run 'ocx status' for the Home's state.`; +} + +export async function printLinkChildLocalNotice(what: string): Promise { + const notice = linkChildLocalNotice(what); + if (notice) console.error(notice); +} + +/** `ocx config set|unset|import` on a link Child: the write is local and its routing parts inactive. */ +export async function printLinkChildLocalWriteNotice(): Promise { + const link = readLinkChildContext(); + if (!link) return; + console.error(`Note: this machine is a Remote Link Child managed by ${link.homePhrase} (link ${link.linkId}). ` + + "This writes this machine's local configuration; its provider, model and routing settings stay inactive while it is linked, and the Home's apply."); +} + +/** The refusal for a local provider change on a link Child, or null elsewhere. */ +export function linkChildProviderChangeRefusal(operation: string): string | null { + const link = readLinkChildContext(); + return link ? linkChildRefusal(link, operation) : null; +} + +/** Can this management request be answered by reading the Home? */ +export function linkChildReadable(method: string, path: string): boolean { + const url = new URL(path.startsWith("/") ? path : `/${path}`, "http://cli.invalid"); + return linkManagementReadAllowed(method, url.pathname, url.searchParams); +} + +/** One allowlisted read from the Home. Throws with an operator sentence when it cannot be sent. */ +export async function readLinkHomeJson( + context: LinkChildContext, + path: string, + deps: LinkHomeDeps = {}, +): Promise<{ status: number; body: unknown }> { + const token = readServiceApiTokenState(); + if (token.kind !== "present" || token.fingerprint !== context.connection.tokenFingerprint) { + throw new Error(`The enrolled link key for ${context.homePhrase} is missing or changed; re-link this machine from the Home.`); + } + let response: Response; + try { + response = await fetchLinkHomeManagement(context.connection, token.token, path.startsWith("/") ? path : `/${path}`, {}, deps); + } catch (error) { + const detail = error instanceof Error && error.name === "LinkRelayAuthenticationError" ? error.message : "the link tunnel did not answer"; + throw new Error(`Could not read ${context.homePhrase} over Remote Link (${context.linkId}): ${detail}.`); + } + const result = await readBoundedResponseBytes(response, { maxBytes: RESPONSE_MAX_BYTES, signal: AbortSignal.timeout(30_000) }); + if (result.oversized) throw new Error(`${context.homePhrase} returned a response larger than the CLI accepts.`); + const text = new TextDecoder().decode(result.bytes); + let body: unknown = null; + if (text) { + try { body = JSON.parse(text); } catch { body = text; } + } + // A Home that predates the relay answers its listener's generic unknown-endpoint 404. + const error = body && typeof body === "object" ? (body as Record).error : undefined; + const unknownEndpoint = error === "not_found" + || (!!error && typeof error === "object" && (error as Record).code === "not_found"); + if (response.status === 404 && unknownEndpoint) { + return { status: 503, body: { error: `${context.homePhrase} does not serve Remote Link management reads; upgrade the Home.` } }; + } + return { status: response.status, body }; +} diff --git a/src/cli/observe-stream.ts b/src/cli/observe-stream.ts index eb88f0a6688..bd32fbeee98 100644 --- a/src/cli/observe-stream.ts +++ b/src/cli/observe-stream.ts @@ -89,7 +89,12 @@ export async function withObserveStream( // Snapshot recovery comes from our observed target, never arbitrary transport text. if (context.kind === "snapshot") { if (!found) throw new ObservationError("Proxy is not running. Start it with: ocx start"); - if (found.role === "client") throw new ObservationError("This listener runs in the client role and has no management API. Run this command on its serving hub, or open the hub dashboard."); + if (found.role === "client") { + const { readLinkChildContext, linkChildRefusal } = await import("./link-child"); + const link = readLinkChildContext(); + throw new ObservationError(link ? linkChildRefusal(link) + : "This listener runs in the client role and has no management API. Run this command on its serving hub, or open the hub dashboard."); + } } return found; } }), signal); diff --git a/src/cli/observe.ts b/src/cli/observe.ts index 018e5f6da62..4cdc7727a41 100644 --- a/src/cli/observe.ts +++ b/src/cli/observe.ts @@ -210,8 +210,12 @@ async function usage(argv: string[], deps: RuntimeApiDeps): Promise { if (token.kind !== "present" || token.fingerprint !== connection.value.tokenFingerprint) { throw new Error("Client usage unavailable: the enrolled data key is missing or changed; repair the client connection"); } + // A Remote Link Child sends this read through the connection-bound link relay (#6957). + const link = connection.value.transport === "link" && connection.value.link; + const peerFetch = deps.fetchImpl + ?? (link ? (await import("../client/link-home")).linkBoundFetch(connection.value, token.token) : undefined); result = await fetchHubUsage(connection.value.serverUrl, token.token, new URLSearchParams(suffix), { - fetchImpl: deps.fetchImpl, timeoutMs: 60_000, + fetchImpl: peerFetch, timeoutMs: 60_000, ...(link ? { peer: "Home" as const } : {}), }); const current = readClientConnectionState(); const currentToken = readServiceApiTokenState(); diff --git a/src/cli/provider.ts b/src/cli/provider.ts index da3902f76d1..a013f2d915e 100644 --- a/src/cli/provider.ts +++ b/src/cli/provider.ts @@ -584,8 +584,19 @@ export async function handleProviderCommand(args: string[], deps: ProviderComman return; } + // #6957: a link Child's local providers are inactive; changing them is refused like any other + // provider change, which belongs to the Home. + if (sub === "add" || sub === "remove" || sub === "set-default") { + const refusal = (await import("./link-child")).linkChildProviderChangeRefusal(`ocx provider ${sub}`); + if (refusal) { + console.error(refusal); + process.exitCode = 1; + return; + } + } switch (sub) { case "list": + await (await import("./link-child")).printLinkChildLocalNotice("The providers"); handleList(subArgs); break; case "add": @@ -595,6 +606,7 @@ export async function handleProviderCommand(args: string[], deps: ProviderComman handleRemove(subArgs); break; case "show": + await (await import("./link-child")).printLinkChildLocalNotice("The provider settings"); handleShow(subArgs); break; case "set-default": diff --git a/src/cli/runtime-api.ts b/src/cli/runtime-api.ts index 9f955ec21b1..d9b0445226f 100644 --- a/src/cli/runtime-api.ts +++ b/src/cli/runtime-api.ts @@ -12,6 +12,7 @@ import { findLiveProxy, probeHostname, type LivenessIo, type LiveProxy } from "../server/proxy-liveness"; import { runningProxyUpdateHeaders } from "../oauth/login-cli"; import { redactSecretArgs } from "./secret-args"; +import type { LinkHomeDeps } from "../client/link-home"; export { redactSecretArgs }; export type CliStdin = NodeJS.ReadableStream & { isTTY?: boolean; readableEnded?: boolean }; @@ -28,6 +29,8 @@ export interface RuntimeApiDeps { stdinSignal?: AbortSignal; /** Optional proxy liveness probe injection for commands that check or fall back around live runtime state. */ findLiveProxy?: (io?: LivenessIo) => Promise; + /** Remote Link Child: how allowlisted reads reach the Home (#6957). */ + linkHome?: LinkHomeDeps; } export class CliUsageError extends Error { @@ -45,7 +48,7 @@ export class RuntimeApiError extends Error { readonly status: number, readonly body: unknown, /** Local discovery refusals occur before the management request is sent. */ - readonly code?: "proxy_not_running" | "client_role_management_unavailable", + readonly code?: "proxy_not_running" | "client_role_management_unavailable" | "link_child_management_unavailable", ) { super(message); this.name = "RuntimeApiError"; @@ -83,6 +86,10 @@ export async function runtimeBaseUrl(deps: RuntimeApiDeps = {}): Promise // this costs no extra request. Only the client role is refused: an absent role is a // standalone or hub proxy (or a legacy body that predates the field), and both serve /api/*. if (live.role === "client") { + // A Remote Link Child has no relay transport to point at; it names its Home instead (#6957). + const { readLinkChildContext, linkChildRefusal } = await import("./link-child"); + const link = readLinkChildContext(); + if (link) throw new RuntimeApiError(linkChildRefusal(link), 503, null, "link_child_management_unavailable"); throw new RuntimeApiError(clientRoleManagementRefusal(live.port), 503, null, "client_role_management_unavailable"); } return `http://${probeHostname(live.hostname)}:${live.port}`; @@ -180,7 +187,25 @@ export async function runtimeRequest( init: RequestInit = {}, deps: RuntimeApiDeps = {}, ): Promise { - const baseUrl = await runtimeBaseUrl(deps); + let baseUrl: string; + try { + baseUrl = await runtimeBaseUrl(deps); + } catch (error) { + if (!(error instanceof RuntimeApiError) || error.code !== "link_child_management_unavailable") throw error; + // #6957: on a Remote Link Child an allowlisted read is answered by the Home over the link; + // every other request is refused with the operation named, before anything is sent. + const method = (init.method ?? "GET").toUpperCase(); + const { readLinkChildContext, linkChildReadable, linkChildRefusal, readLinkHomeJson } = await import("./link-child"); + const link = readLinkChildContext(); + if (!link) throw error; + const pathname = new URL(path.startsWith("/") ? path : `/${path}`, "http://cli.invalid").pathname; + if (!linkChildReadable(method, path)) { + throw new RuntimeApiError(linkChildRefusal(link, `${method} ${pathname}`), 503, null, "link_child_management_unavailable"); + } + const { status, body } = await readLinkHomeJson(link, path, deps.linkHome); + if (status < 200 || status >= 300) throw new RuntimeApiError(responseMessage(body, status), status, body); + return body as T; + } const headers = runningProxyUpdateHeaders(); for (const [key, value] of new Headers(init.headers).entries()) headers.set(key, value); const fetchImpl = deps.fetchImpl ?? fetch; diff --git a/src/cli/status.ts b/src/cli/status.ts index 70ca7233be8..518b570c952 100644 --- a/src/cli/status.ts +++ b/src/cli/status.ts @@ -104,6 +104,16 @@ export type CliRemoteHubStatus = { truncated: boolean; /** The hub's own Claude Code toggle; a client cannot infer it from local config. */ claudeCodeEnabled: boolean | null; + /** + * Present on a Remote Link Child (#6957): the peer is a Home reached over the link, not a hub, + * and every sentence names it that way. `codexSettings` lists the Home's Codex settings the + * last sync applied, or why the Child's local values are in effect. + */ + link?: { + linkId: string; + homePhrase: string; + codexSettings: { source: "home"; fetchedAt: string; lines: string[] } | { source: "local"; reason: string }; + }; }; export type CliStatusJson = { @@ -529,6 +539,11 @@ export async function collectRemoteHubStatus( })) as typeof fetch; } } + // #6957: the Home's name, the link id and the Codex settings the last sync applied. + const linkState = readClientConnectionState(); + const linkConnection = connection.transport === "link" && linkState.kind === "connected" && linkState.value.transport === "link" && linkState.value.link + ? linkState.value : null; + const linkHome = linkConnection ? await import("../client/link-home") : null; const resolved = await resolveHubState({ owner, token: connection.transport === "link" ? null : bound.token, @@ -536,9 +551,22 @@ export async function collectRemoteHubStatus( ...(bound.withheldReason ? { withheldTokenReason: bound.withheldReason } : {}), ...(fetchImpl ? { fetchImpl } : {}), allowNetwork, + ...(connection.transport === "link" ? { peer: "Home" as const } : {}), ...(options.timeoutMs === undefined ? {} : { timeoutMs: options.timeoutMs }), ...(options.now === undefined ? {} : { now: options.now }), }); + let link: CliRemoteHubStatus["link"]; + if (linkConnection && linkHome) { + const { describeHomeCodexSettings } = await import("../link/codex-settings"); + const record = linkHome.readHomeCodexSettingsRecord(linkConnection.link!.linkId); + link = { + linkId: linkConnection.link!.linkId, + homePhrase: linkHome.linkHomePhrase(linkConnection), + codexSettings: record + ? { source: "home", fetchedAt: record.fetchedAt, lines: describeHomeCodexSettings(record.document.settings) } + : { source: "local", reason: "no Home settings recorded yet; run: ocx sync" }, + }; + } return { connected: true, // The hub's own advertised origin when it publishes one; otherwise the URL this client @@ -554,6 +582,7 @@ export async function collectRemoteHubStatus( subagentModels: resolved.state?.subagentModels ?? [], truncated: resolved.state?.truncated === true, claudeCodeEnabled: resolved.state ? resolved.state.claudeCode.enabled : null, + ...(link ? { link } : {}), }; } @@ -567,6 +596,16 @@ export async function collectRemoteHubStatus( */ export function remoteHubBannerLine(remoteHub: CliRemoteHubStatus, transport?: "hub" | "link"): string | null { if (!remoteHub.connected) return null; + if (remoteHub.link) { + const peer = `${remoteHub.link.homePhrase} over Remote Link (${remoteHub.link.linkId})`; + if (remoteHub.stateSource === "unavailable") { + return `⚠️ ${peer}: state unavailable (${remoteHub.reason ?? "unknown reason"}) — provider and login lines below are LOCAL and do not describe the Home.`; + } + const cached = remoteHub.stateSource === "cache" + ? ` — cached ${remoteHub.ageSeconds ?? "?"}s ago (${remoteHub.reason ?? "live read failed"})` + : ""; + return `🔗 State from ${peer}${cached}: provider credentials, logins and delegable models below are the Home's, not this machine's.`; + } if (transport === "link") { const origin = remoteHub.origin ?? "unknown"; return remoteHub.stateSource === "unavailable" @@ -588,31 +627,46 @@ export function remoteHubBannerLine(remoteHub: CliRemoteHubStatus, transport?: " * spawning the CLI. Indentation is the caller's. Empty when nothing true is known. */ export function remoteHubStatusLines(remoteHub: CliRemoteHubStatus, transport?: "hub" | "link"): string[] { - if (!remoteHub.connected || remoteHub.stateSource === "unavailable") return []; - const origin = remoteHub.origin ?? "hub"; + if (!remoteHub.connected) return []; + const settings = remoteHub.link ? linkCodexSettingsLines(remoteHub.link) : []; + if (remoteHub.stateSource === "unavailable") return settings; + const isLink = Boolean(remoteHub.link) || transport === "link"; + const peer = remoteHub.link?.homePhrase ?? `${isLink ? "Home" : "hub"} ${remoteHub.origin ?? (isLink ? "Home" : "hub")}`; + const subject = isLink ? "the Home" : "the hub"; const lines = [ - `OAuth logins (hub ${origin}):`, + `OAuth logins (${peer}):`, ...(remoteHub.oauth.length === 0 - ? [" (the hub reported no OAuth providers)"] + ? [` (${subject} reported no OAuth providers)`] : remoteHub.oauth.map(entry => ` ${entry.provider.padEnd(10)} ${entry.loggedIn ? "✓ logged in" : "✗ not logged in"}`)), - `Providers (hub ${origin}):`, + `Providers (${peer}):`, ...(remoteHub.providers.length === 0 - ? [" (the hub reported no providers)"] + ? [` (${subject} reported no providers)`] : remoteHub.providers.map(provider => { // `authMode` is what keeps "no API key" from reading as "not configured": an `oauth` // provider legitimately has no key and is still fully usable. const credential = provider.hasCredential ? "credential stored" : `no API key (authMode ${provider.authMode ?? "key"})`; return ` ${provider.name.padEnd(10)} ${provider.adapter} — ${credential}${provider.disabled ? ", disabled" : ""}`; })), - `Delegable models (hub ${origin}): ${remoteHub.subagentModels.length === 0 ? "none" : remoteHub.subagentModels.join(", ")}`, + `Delegable models (${peer}): ${remoteHub.subagentModels.length === 0 ? "none" : remoteHub.subagentModels.join(", ")}`, ]; - if (remoteHub.hubVersion) lines.push(`Hub version: ${remoteHub.hubVersion}`); + if (remoteHub.hubVersion) lines.push(`${remoteHub.link ? "Home" : "Hub"} version: ${remoteHub.hubVersion}`); // The hub told us its own lists are a prefix. Saying nothing here would make this report claim // completeness it does not have — the same shape of confident-and-wrong answer as #4236. if (remoteHub.truncated) { - lines.push(" (the hub truncated this state to fit its response caps; some rows are not listed)"); + lines.push(` (${subject} truncated this state to fit its response caps; some rows are not listed)`); + } + return [...lines, ...settings]; +} + +/** Which Codex settings the last sync took from the Home (#6957). */ +export function linkCodexSettingsLines(link: NonNullable): string[] { + if (link.codexSettings.source === "local") { + return [`Codex settings: this machine's local values (${link.codexSettings.reason})`]; } - return lines; + return [ + `Codex settings from ${link.homePhrase} (applied ${link.codexSettings.fetchedAt}):`, + ...link.codexSettings.lines.map(line => ` ${line}`), + ]; } export function selectListenTarget( diff --git a/src/client/connect.ts b/src/client/connect.ts index 7aa3e65e005..2d86ca5ae2f 100644 --- a/src/client/connect.ts +++ b/src/client/connect.ts @@ -74,6 +74,16 @@ import { assertNoClientDisconnectPending, assertClientConnectionUnchanged, sameClientConnectionOwner, } from "./state"; import { assertClientCatalogCompatible, type CatalogCompatibilityDeps } from "./catalog-compatibility"; +import { overlayHomeCodexSettings } from "../link/codex-settings"; +import { + clearHomeCodexSettingsRecord, + fetchHomeCodexSettings, + resolveHomeCodexSettings, + recordHomeCodexSettings, + writeHomeCodexSettingsRecord, + type HomeCodexSettingsSource, + type LinkHomeDeps, +} from "./link-home"; import { hubStateCachePath } from "./hub-state"; import { ClientLinkStateError, clearClientLinkState, readClientLinkState } from "./link-state"; import { isLinkPort } from "../link/ports"; @@ -110,6 +120,8 @@ export interface ClientConnectDeps { now?: () => Date; lifecycleLockDeps?: ClientLifecycleLockDeps; catalogCompatibility?: CatalogCompatibilityDeps; + /** Link mode: how the Home's Codex settings are read (#6957). */ + linkHome?: LinkHomeDeps; } export interface RotateClientOptions { @@ -687,7 +699,16 @@ export async function connectClient( const config = earlyConfig ?? loadConfig(); const target = routingTarget(serverUrl, linkMode ? config.port : undefined, config); - const injectConfig = { ...config, syncResumeHistory: false }; + // A link Child writes the Home's OpenCodex-owned Codex settings, not its own (#6957). A read + // that fails here keeps the local values; the next `ocx sync` applies the Home's. + // Read only: the record is written after the connection commits, so a rollback leaves none. + const homeSettings = linkMode && linkMetadata && options.selectedClients.includes("codex") + ? await fetchHomeCodexSettings({ transport: "link", link: linkMetadata, apiKeyId }, admissionToken, deps.linkHome) + : undefined; + const injectConfig = { + ...(homeSettings?.kind === "fresh" ? overlayHomeCodexSettings(config, homeSettings.document.settings) : config), + syncResumeHistory: false, + }; const preflight = await injectCodexConfig(config.port, injectConfig, { validateOnly: true, routingTarget: target, @@ -734,6 +755,12 @@ export async function connectClient( commitClientConnection(connection); committed = true; }), deps.lifecycleLockDeps); + // Recorded only when the injection that applied it ran (not with --no-sync). + if (homeSettings?.kind === "fresh" && linkMetadata && injectionCommitted) { + try { + writeHomeCodexSettingsRecord({ linkId: linkMetadata.linkId, fetchedAt: now, document: homeSettings.document }); + } catch { /* the next sync records it; the applied config is already committed */ } + } return connection; } catch (error) { const rollbackFailures: string[] = []; @@ -780,7 +807,7 @@ export async function connectClient( export async function syncConnectedClient( _options: { restartCodex?: boolean } = {}, deps: ClientConnectDeps = {}, -): Promise<{ catalogWritten: boolean; cacheSynced: boolean; injected: boolean; stale: boolean }> { +): Promise<{ catalogWritten: boolean; cacheSynced: boolean; injected: boolean; stale: boolean; homeSettings?: HomeCodexSettingsSource }> { if (siblingOfLivePort() !== null) throw new Error(siblingSkipMessage()); const initial = withClientLifecycleSync(() => withConfigMutationLockSync(() => { assertNoClientDisconnectPending(); @@ -825,21 +852,28 @@ export async function syncConnectedClient( if (next.pendingOperation || token.kind !== "present" || token.fingerprint !== next.tokenFingerprint) throw new Error("client_token_changed"); }; let injected = false; + let homeSettings: HomeCodexSettingsSource | undefined; if (next.selectedClients.includes("codex")) { - const config = loadConfig(); + let config = loadConfig(); + // #6957: a link Child applies the Home's OpenCodex-owned Codex settings with the catalog. + if (isLinkConnection(next)) { + homeSettings = await resolveHomeCodexSettings(next, initial.token.token, deps.linkHome); + if (homeSettings.source !== "local") config = overlayHomeCodexSettings(config, homeSettings.document.settings); + } const result = await injectCodexConfig(config.port, { ...config, syncResumeHistory: false }, { routingTarget: routingTarget(next.serverUrl, isLinkConnection(next) ? config.port : undefined, config), catalogPath: DEFAULT_CATALOG_PATH, journalOwner: { kind: "client", apiKeyId: next.apiKeyId }, beforeClientWrite, }); if (!result.success || result.status === "skipped") throw new Error(result.message); injected = true; + if (homeSettings && next.link) recordHomeCodexSettings(next.link.linkId, homeSettings); } const cacheSynced = withClientLifecycleSync(() => { beforeClientWrite(); // Cache invalidation acquires K itself (N -> K -> C); never call it while C is held. return invalidateCodexModelsCache({ allowWhenDesiredDisabled: true }); }, deps.lifecycleLockDeps); - return { catalogWritten: downloaded !== undefined, cacheSynced, injected, stale }; + return { catalogWritten: downloaded !== undefined, cacheSynced, injected, stale, ...(homeSettings ? { homeSettings } : {}) }; } /** @@ -1035,6 +1069,7 @@ export async function disconnectClient( if (!(error instanceof ClientLinkStateError)) throw error; } if (sidecarLinkId === connection.link.linkId) clearClientLinkState(connection.link.linkId); + try { clearHomeCodexSettingsRecord(); } catch { /* derived data; never fails a disconnect */ } } removeHubStateCache(); requireDesktopResult(finishRemoteDesktopCleanup(held, receipt.owner)); diff --git a/src/client/hub-client.ts b/src/client/hub-client.ts index 0c97c6adca7..efdf1649139 100644 --- a/src/client/hub-client.ts +++ b/src/client/hub-client.ts @@ -485,7 +485,7 @@ export async function fetchHubUsage( serverUrl: string, admissionToken: string, query: URLSearchParams, - options: { timeoutMs?: number; fetchImpl?: typeof fetch } = {}, + options: { timeoutMs?: number; fetchImpl?: typeof fetch; peer?: "hub" | "Home" } = {}, ): Promise { const origin = normalizeHubOrigin(serverUrl); if (!isHubTransportPermitted(origin)) { @@ -498,9 +498,12 @@ export async function fetchHubUsage( }, options.timeoutMs); if (!response.ok) { try { await response.body?.cancel(); } catch { /* best effort */ } - const message = response.status === 404 ? "Hub does not support client usage; upgrade the hub" - : response.status === 401 || response.status === 403 ? "Hub rejected this client's usage credential" - : `Hub usage request failed (${response.status})`; + // A Remote Link Child reads the same route from its Home (#6957). + const peer = options.peer ?? "hub"; + const Peer = peer === "Home" ? "Home" : "Hub"; + const message = response.status === 404 ? `${Peer} does not support client usage; upgrade the ${peer}` + : response.status === 401 || response.status === 403 ? `${Peer} rejected this ${peer === "Home" ? "Child's link key" : "client's usage credential"}` + : `${Peer} usage request failed (${response.status})`; throw new HubClientError(`hub_usage_http_${response.status}`, message, response.status); } if (!jsonCompatibleContentType(response)) { diff --git a/src/client/hub-state.ts b/src/client/hub-state.ts index fb63d2adca0..9dee25ced4a 100644 --- a/src/client/hub-state.ts +++ b/src/client/hub-state.ts @@ -124,39 +124,43 @@ export function writeCachedHubState(owner: HubStateOwner, state: HubStateDTO, fe * banner reading `state unavailable (hub_state_http_507)` sends the reader looking for a client * bug when the hub has in fact answered and said something. */ -export function hubStateFailureReason(error: unknown): string { +export function hubStateFailureReason(error: unknown, peer: "hub" | "Home" = "hub"): string { + // A Remote Link Child reads the same route from its Home (#6957); the sentences name the peer + // the operator actually configured. + const holder = peer === "Home" ? "this Child's link key" : "this client's data key"; if (error instanceof HubClientError) { switch (error.code) { case "hub_state_not_a_hub": return "the remote host does not publish hub state"; case "hub_state_unsupported": - return "this hub is too old to report its state; upgrade the hub"; + return `this ${peer} is too old to report its state; upgrade the ${peer}`; case "hub_state_unauthorized": - return "the hub rejected this client's data key"; + return `the ${peer} rejected ${holder}`; case "hub_state_schema_invalid": case "hub_state_invalid": - return "the hub returned an unreadable hub-state document"; + return `the ${peer} returned an unreadable hub-state document`; case "hub_state_content_type_invalid": // Usually a captive portal, a TLS-terminating proxy or an error page in front of the // hub: the request reached SOMETHING, and that something is not the hub's API. - return "the hub's state response was not JSON"; + return `the ${peer}'s state response was not JSON`; case "body_too_large": - return "the hub's state response exceeded the allowed size"; + return `the ${peer}'s state response exceeded the allowed size`; case "unreachable": - return "the hub is unreachable"; + return `the ${peer} is unreachable`; case "redirect_refused": - return "the hub redirected the state request"; + return `the ${peer} redirected the state request`; default: { const status = error.code.startsWith("hub_state_http_") ? error.code.slice("hub_state_http_".length) : null; return status && /^\d+$/.test(status) - ? `the hub answered HTTP ${status} to the state request` + ? `the ${peer} answered HTTP ${status} to the state request` : error.code; } } } - return "the hub state could not be read"; + if (error instanceof Error && error.name === "LinkRelayAuthenticationError") return error.message; + return `the ${peer} state could not be read`; } export interface ResolveHubStateOptions { @@ -178,6 +182,8 @@ export interface ResolveHubStateOptions { allowNetwork?: boolean; /** False skips the cache write, for read-only callers. */ persist?: boolean; + /** Which peer the sentences name: a hub, or a Remote Link Child's Home. */ + peer?: "hub" | "Home"; } function withAge( @@ -218,8 +224,8 @@ export async function resolveHubState(options: ResolveHubStateOptions): Promise< } catch (error) { // Deliberately no local-state fallback here. A stale cache is still the HUB's state; the // client's own providers and logins are not, at any age. - if (error instanceof HubClientError && error.code === "hub_state_not_a_hub") return withAge("unavailable", null, undefined, now, hubStateFailureReason(error)); - return fromCache(hubStateFailureReason(error)); + if (error instanceof HubClientError && error.code === "hub_state_not_a_hub") return withAge("unavailable", null, undefined, now, hubStateFailureReason(error, options.peer)); + return fromCache(hubStateFailureReason(error, options.peer)); } const fetchedAt = new Date(now).toISOString(); if (options.persist !== false) writeCachedHubState(options.owner, state, fetchedAt); diff --git a/src/client/link-home.ts b/src/client/link-home.ts new file mode 100644 index 00000000000..a015bc98408 --- /dev/null +++ b/src/client/link-home.ts @@ -0,0 +1,248 @@ +/** + * A Remote Link Child's view of its Home (#6957): the Home's name for refusal and status text, + * the Home's OpenCodex-owned Codex settings, and authenticated read-only management reads. + * + * Every read goes through the connection-bound link relay (`fetchBoundLinkRelay`): the link key + * is sent only after the listener on the tunnel port proves it holds the same link identity, so a + * process that took the tunnel port never sees the key. + */ +import { chmodSync, lstatSync, mkdirSync, readFileSync, unlinkSync } from "node:fs"; +import { join } from "node:path"; +import { atomicWriteFile, isMissingPathError } from "../config/atomic-write"; +import { readBoundedResponseBytes } from "../lib/bounded-body"; +import { assertNotRealHomeUnderTest } from "../lib/test-home-guard"; +import { hardenSecretDir } from "../lib/windows-secret-acl"; +import { + parseHomeCodexSettingsDocument, + type HomeCodexSettingsDocument, +} from "../link/codex-settings"; +import { + LINK_CODEX_SETTINGS_PATH, + LINK_MANAGEMENT_PREFIX, + linkManagementReadAllowed, +} from "../link/management-routes"; +import { linkDir } from "../link/paths"; +import type { OcxClientConnectionConfig } from "../types"; +import { fetchBoundLinkRelay } from "./link-relay-transport"; +import { readClientLinkState, type ClientLinkState } from "./link-state"; + +const RECORD_MAX_BYTES = 64 * 1024; +const SETTINGS_MAX_BYTES = 64 * 1024; +const READ_TIMEOUT_MS = 10_000; + +export interface LinkHomeDeps { + /** Trusted complete-send test seam; production uses the connection-bound transport. */ + fetchImpl?: typeof fetch; + timeoutMs?: number; +} + +/** The connection fields a Home read needs; connect calls these before its state is committed. */ +export type LinkHomeConnection = Pick; + +export interface HomeCodexSettingsRecord { + linkId: string; + fetchedAt: string; + document: HomeCodexSettingsDocument; +} + +export function homeCodexSettingsPath(): string { + return join(linkDir(), "home-codex-settings.json"); +} + +function linkTarget(connection: LinkHomeConnection, admissionKey: string) { + if (connection.transport !== "link" || !connection.link) throw new Error("not a Remote Link Child"); + return { + tunnelPort: connection.link.tunnelPort, + linkId: connection.link.linkId, + apiKeyId: connection.apiKeyId, + admissionKey, + }; +} + +/** The last applied Home settings for this link, or null when absent, malformed or another link's. */ +export function readHomeCodexSettingsRecord(linkId: string, path = homeCodexSettingsPath()): HomeCodexSettingsRecord | null { + try { + const stat = lstatSync(path); + if (!stat.isFile() || stat.size > RECORD_MAX_BYTES) return null; + const raw = JSON.parse(readFileSync(path, "utf8")) as unknown; + if (!raw || typeof raw !== "object" || Array.isArray(raw)) return null; + const record = raw as Record; + if (record.linkId !== linkId || typeof record.fetchedAt !== "string" || Number.isNaN(Date.parse(record.fetchedAt))) return null; + const document = parseHomeCodexSettingsDocument(record.document); + return document ? { linkId, fetchedAt: record.fetchedAt, document } : null; + } catch { + return null; + } +} + +export function writeHomeCodexSettingsRecord(record: HomeCodexSettingsRecord, path = homeCodexSettingsPath()): void { + const dir = linkDir(); + assertNotRealHomeUnderTest(join(dir, "..")); + mkdirSync(dir, { recursive: true, mode: 0o700 }); + if (process.platform === "win32") hardenSecretDir(dir, { required: true }); + else chmodSync(dir, 0o700); + atomicWriteFile(path, `${JSON.stringify(record, null, 2)}\n`); + if (process.platform !== "win32") chmodSync(path, 0o600); +} + +export function clearHomeCodexSettingsRecord(path = homeCodexSettingsPath()): void { + try { unlinkSync(path); } catch (error) { if (!isMissingPathError(error)) throw error; } +} + +/** + * The Home's name as this Child knows it: the SSH alias of a Child-initiated link, else the + * host name the Home published with its Codex settings, else null ("its Home"). + */ +export function linkHomeName( + connection: OcxClientConnectionConfig, + readSidecar: () => ClientLinkState | null = readClientLinkState, +): string | null { + const linkId = connection.link?.linkId; + if (!linkId) return null; + try { + const sidecar = readSidecar(); + if (sidecar && sidecar.linkId === linkId) return sidecar.alias; + } catch { /* an unreadable sidecar names nothing */ } + return readHomeCodexSettingsRecord(linkId)?.document.home.name ?? null; +} + +/** "Home home-box" or "its Home": the phrase every link Child message uses. */ +export function linkHomePhrase( + connection: OcxClientConnectionConfig, + readSidecar?: () => ClientLinkState | null, +): string { + const name = linkHomeName(connection, readSidecar); + return name ? `Home ${name}` : "its Home"; +} + +/** + * A `fetch` for the existing data-plane reads (`/v1/hub-state`, `/v1/usage`) that sends each + * request through the connection-bound link relay, so the link key reaches only the Home. + */ +export function linkBoundFetch(connection: LinkHomeConnection, admissionKey: string): typeof fetch { + const target = linkTarget(connection, admissionKey); + return ((input: RequestInfo | URL, init?: RequestInit) => { + // A Request keeps its method, headers and signal unless init overrides them. + const request = input instanceof Request ? input : null; + const url = request ? request.url : String(input); + return fetchBoundLinkRelay(target, url, { + ...(request ? { method: request.method, headers: request.headers, signal: request.signal } : {}), + ...(init ?? {}), + redirect: "manual", + }); + }) as typeof fetch; +} + +/** + * One authenticated, read-only management read on the Home. `path` is a Home management path + * with an optional query (`/api/providers`); anything outside the shared allowlist throws before + * a request is built. + */ +export async function fetchLinkHomeManagement( + connection: LinkHomeConnection, + admissionKey: string, + path: string, + init: { method?: "GET" | "HEAD"; signal?: AbortSignal } = {}, + deps: LinkHomeDeps = {}, +): Promise { + const method = init.method ?? "GET"; + const local = new URL(path, "http://link.invalid"); + if (local.origin !== "http://link.invalid" || !linkManagementReadAllowed(method, local.pathname, local.searchParams)) { + throw new Error("path is not readable over the Remote Link"); + } + const target = linkTarget(connection, admissionKey); + const destination = `http://127.0.0.1:${target.tunnelPort}${LINK_MANAGEMENT_PREFIX}${local.pathname}${local.search}`; + const signal = init.signal + ? AbortSignal.any([init.signal, AbortSignal.timeout(deps.timeoutMs ?? READ_TIMEOUT_MS)]) + : AbortSignal.timeout(deps.timeoutMs ?? READ_TIMEOUT_MS); + const request: RequestInit = { + method, + headers: { accept: "application/json", authorization: `Bearer ${admissionKey}` }, + redirect: "manual", + signal, + }; + return deps.fetchImpl ? deps.fetchImpl(destination, request) : fetchBoundLinkRelay(target, destination, request); +} + +export type HomeCodexSettingsFetch = + | { kind: "fresh"; document: HomeCodexSettingsDocument } + | { kind: "unsupported" } + | { kind: "failed"; reason: string }; + +export async function fetchHomeCodexSettings( + connection: LinkHomeConnection, + admissionKey: string, + deps: LinkHomeDeps = {}, +): Promise { + let response: Response; + try { + const target = linkTarget(connection, admissionKey); + const destination = `http://127.0.0.1:${target.tunnelPort}${LINK_CODEX_SETTINGS_PATH}`; + const request: RequestInit = { + method: "GET", + headers: { accept: "application/json", authorization: `Bearer ${admissionKey}` }, + redirect: "manual", + signal: AbortSignal.timeout(deps.timeoutMs ?? READ_TIMEOUT_MS), + }; + response = deps.fetchImpl ? await deps.fetchImpl(destination, request) : await fetchBoundLinkRelay(target, destination, request); + } catch (error) { + return { kind: "failed", reason: error instanceof Error && error.name === "LinkRelayAuthenticationError" ? error.message : "the Home is unreachable" }; + } + if (response.status === 404) { + try { await response.body?.cancel(); } catch { /* best effort */ } + return { kind: "unsupported" }; + } + if (!response.ok) { + try { await response.body?.cancel(); } catch { /* best effort */ } + return { kind: "failed", reason: `the Home answered HTTP ${response.status}` }; + } + try { + const result = await readBoundedResponseBytes(response, { maxBytes: SETTINGS_MAX_BYTES, signal: AbortSignal.timeout(deps.timeoutMs ?? READ_TIMEOUT_MS) }); + if (result.oversized) return { kind: "failed", reason: "the Home's settings exceeded the allowed size" }; + const document = parseHomeCodexSettingsDocument(JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(result.bytes))); + return document ? { kind: "fresh", document } : { kind: "failed", reason: "the Home returned an unreadable settings document" }; + } catch { + return { kind: "failed", reason: "the Home returned an unreadable settings document" }; + } +} + +export type HomeCodexSettingsSource = + | { source: "home"; document: HomeCodexSettingsDocument; fetchedAt: string } + | { source: "cached"; document: HomeCodexSettingsDocument; fetchedAt: string; reason: string } + | { source: "local"; reason: string }; + +/** + * The settings a sync applies. A failed read reuses the last record for this link so a brief + * outage never flips the Child back to its local values; a Home that predates the route leaves + * the Child's local values in place and says so. A fresh read is NOT recorded here: the caller + * records it with `recordHomeCodexSettings` only after the injection that applies it succeeds, + * because `ocx status` reads the record as "applied". + */ +export async function resolveHomeCodexSettings( + connection: LinkHomeConnection, + admissionKey: string, + deps: LinkHomeDeps & { now?: () => Date } = {}, +): Promise { + const linkId = connection.link?.linkId; + if (!linkId) return { source: "local", reason: "not a Remote Link Child" }; + const fetched = await fetchHomeCodexSettings(connection, admissionKey, deps); + if (fetched.kind === "fresh") { + return { source: "home", document: fetched.document, fetchedAt: (deps.now ?? (() => new Date()))().toISOString() }; + } + if (fetched.kind === "unsupported") { + clearHomeCodexSettingsRecord(); + return { source: "local", reason: "the Home does not publish Codex settings; upgrade the Home" }; + } + const cached = readHomeCodexSettingsRecord(linkId); + return cached + ? { source: "cached", document: cached.document, fetchedAt: cached.fetchedAt, reason: fetched.reason } + : { source: "local", reason: fetched.reason }; +} + +/** Record settings a successful injection applied. Best effort: the config is already written. */ +export function recordHomeCodexSettings(linkId: string, settings: HomeCodexSettingsSource): void { + if (settings.source !== "home") return; + try { + writeHomeCodexSettingsRecord({ linkId, fetchedAt: settings.fetchedAt, document: settings.document }); + } catch { /* the next sync records it */ } +} diff --git a/src/client/link-management-relay.ts b/src/client/link-management-relay.ts new file mode 100644 index 00000000000..25207803bed --- /dev/null +++ b/src/client/link-management-relay.ts @@ -0,0 +1,61 @@ +/** + * The Child end of the Remote Link management relay (#6957). + * + * The Child's dashboard reaches its Home's management reads at `/api/machine/hub-relay/api/*`, + * the same base a hub client's dashboard uses. The browser authenticates to the Child with the + * Child's own dashboard session; this relay drops every browser header, builds a fresh GET for + * the Home's `/v1/link-management` prefix and sends it through the connection-bound link relay, + * which attaches the link key only after the tunnel listener proves the link identity. Anything + * outside the shared read allowlist is refused here, before a request exists. + */ +import { HUB_RELAY_RESPONSE_BODY_MAX_BYTES } from "./hub-relay"; +import type { LinkIngress } from "./link-ingress"; +import { relayLinkDataRequest } from "./link-relay"; +import { + LINK_MANAGEMENT_PREFIX, + linkChildManagementRefusal, + linkManagementReadAllowed, + linkManagementRouteAllowed, +} from "../link/management-routes"; + +export const LINK_CHILD_READ_ONLY = "link_child_read_only"; + +export function linkChildRefusalResponse(homePhrase: string, linkId: string, operation: string): Response { + return Response.json({ + error: linkChildManagementRefusal(homePhrase, linkId, operation), + code: LINK_CHILD_READ_ONLY, + linkId, + }, { status: 403, headers: { "cache-control": "no-store" } }); +} + +export async function relayLinkManagementRead( + req: Request, + suffix: string, + ingress: LinkIngress, + homePhrase: string, +): Promise { + const linkId = ingress.linkId ?? "unknown"; + const query = suffix.indexOf("?"); + const pathname = query === -1 ? suffix : suffix.slice(0, query); + const search = query === -1 ? "" : suffix.slice(query); + if (!linkManagementReadAllowed(req.method, pathname, new URLSearchParams(search.slice(1)))) { + return linkChildRefusalResponse(homePhrase, linkId, `${req.method.toUpperCase()} ${pathname}`); + } + const admissionKey = ingress.linkKey(); + if (!admissionKey) return Response.json({ error: "link_credential_unavailable" }, { status: 503 }); + const outbound = new Request(`http://127.0.0.1${LINK_MANAGEMENT_PREFIX}${pathname}${search}`, { + method: req.method, + headers: { accept: "application/json" }, + signal: req.signal, + }); + return relayLinkDataRequest(outbound, { + tunnelPort: ingress.tunnelPort, + admissionKey, + linkId: ingress.linkId, + apiKeyId: ingress.apiKeyId, + }, { + ...ingress.relay, + bodyLimitBytes: HUB_RELAY_RESPONSE_BODY_MAX_BYTES, + routeAllowed: linkManagementRouteAllowed, + }); +} diff --git a/src/client/link-relay.ts b/src/client/link-relay.ts index 127d2e80e45..f0d8942f161 100644 --- a/src/client/link-relay.ts +++ b/src/client/link-relay.ts @@ -69,6 +69,11 @@ export interface LinkRelayDeps { /** The longest a request waits for a reconnecting tunnel, from its first wait. */ holdMs?: number; now?: () => number; + /** + * The route table this relay enforces. Defaults to the data-plane table; only the Child's + * authenticated management relay passes the read-only management table (#6957). + */ + routeAllowed?: (url: URL, req: Request) => boolean; } export const LINK_RELAY_RETRY_AFTER_SECONDS = 1; @@ -305,7 +310,7 @@ export async function relayLinkDataRequest( deps: LinkRelayDeps = {}, ): Promise { const url = new URL(req.url); - if (!linkRouteAllowed(url, req)) return jsonError(404, "not_found"); + if (!(deps.routeAllowed ?? linkRouteAllowed)(url, req)) return jsonError(404, "not_found"); let destination: string; try { destination = linkRelayDestination(url, target); } catch { return jsonError(404, "not_found"); } const validation = validateHubRelayRequestHeaders(linkFramingHeaders(req.headers)); diff --git a/src/client/machine-api.ts b/src/client/machine-api.ts index 4faae68bfbb..3413eb0be8f 100644 --- a/src/client/machine-api.ts +++ b/src/client/machine-api.ts @@ -3,6 +3,7 @@ import { diagnoseCodexShim, installCodexShim, uninstallCodexShim } from "../code import { readManagementJsonBody } from "../server/management/body"; import type { OcxClientConnectionConfig } from "../types"; import { disconnectClient, syncConnectedClient } from "./connect"; +import { linkHomeName } from "./link-home"; import { isLinkConnection } from "./state"; export type HubReachability = "unknown" | "online" | "offline" | "unauthorized"; @@ -19,6 +20,8 @@ export interface MachineStatusV1 { connectedAt: string; catalogSyncedAt?: string; hubReachability: HubReachability; + /** Present only on a Remote Link Child: the dashboard shows "managed by Home ". */ + link?: { linkId: string; home: string | null; readOnly: true }; } export interface MachineApiDeps { @@ -27,6 +30,8 @@ export interface MachineApiDeps { scheduleStandaloneRecycle: (disconnectedTokenFingerprint: string) => void; hubReachability?: () => HubReachability; setHubReachability?: (value: HubReachability) => void; + /** Link mode: the Home's name as this listener knows it (sidecar alias or recorded host name). */ + linkHomeName?: () => string | null; } const defaultDeps: MachineApiDeps = { @@ -52,24 +57,26 @@ async function jsonBody(req: Request): Promise { function statusPayload(req: Request, state: OcxClientConnectionConfig, deps: MachineApiDeps): MachineStatusV1 { const machineBase = new URL(req.url).origin; // A link Child's management URL is the tunnel, whose hub-link ingress serves no /api/* and no - // session bootstrap. Its dashboard reads the Child's own listener instead (`/api/link/status`). + // session bootstrap. Its dashboard therefore uses this listener's relay base: the browser holds + // only this listener's session, and the listener relays allowlisted reads to the Home with the + // link key (#6957, src/client/link-management-relay.ts). const linkMode = isLinkConnection(state); + const relay = linkMode || state.managementTransport === "relay"; return { mode: "client", connected: true, machineBase, - sharedBase: linkMode - ? machineBase - : state.managementTransport === "relay" - ? `${machineBase}/api/machine/hub-relay` - : state.managementUrl, + sharedBase: relay ? `${machineBase}/api/machine/hub-relay` : state.managementUrl, sharedServerOrigin: linkMode ? machineBase : state.managementUrl, - managementTransport: state.managementTransport, + managementTransport: relay ? "relay" : state.managementTransport, apiKeyId: state.apiKeyId, protocolVersion: state.protocolVersion, connectedAt: state.connectedAt, ...(state.catalogSyncedAt ? { catalogSyncedAt: state.catalogSyncedAt } : {}), hubReachability: deps.hubReachability?.() ?? "unknown", + ...(linkMode && state.link + ? { link: { linkId: state.link.linkId, home: deps.linkHomeName ? deps.linkHomeName() : linkHomeName(state), readOnly: true as const } } + : {}), }; } diff --git a/src/client/machine-listener.ts b/src/client/machine-listener.ts index ffce1086f9d..fcbc63d941d 100644 --- a/src/client/machine-listener.ts +++ b/src/client/machine-listener.ts @@ -23,6 +23,8 @@ import { projectClientLinkChild, type ClientLinkSidecarRead } from "./link-statu import type { ClientLinkSupervisorStatus } from "./link-tunnel"; import { packageVersion } from "../lib/package-version"; import { linkRouteAllowed } from "../link/routes"; +import { linkHomeName, linkHomePhrase } from "./link-home"; +import { relayLinkManagementRead } from "./link-management-relay"; const VERSION = packageVersion("0.0.0"); const GUI_SPA_PATHS = new Set([ @@ -100,6 +102,8 @@ export function startMachineListener( if (linkMode && !connection.link) throw new Error("link machine listener requires link transport metadata"); const managementAuth = deps.managementAuthState ?? initializeManagementAuthState(config); let hubReachability: HubReachability = "unknown"; + const readSidecar = deps.readSidecar ?? (() => readClientLinkState()); + const sidecarOrNull = () => { const read = readSidecarSafely(readSidecar); return read === "invalid" ? null : read; }; const machineApiDeps: MachineApiDeps = { sync: deps.machineApi?.sync ?? syncConnectedClient, disconnect: deps.machineApi?.disconnect ?? disconnectClient, @@ -108,8 +112,11 @@ export function startMachineListener( }), hubReachability: deps.machineApi?.hubReachability ?? (() => hubReachability), setHubReachability: deps.machineApi?.setHubReachability ?? (value => { hubReachability = value; }), + ...(linkMode ? { linkHomeName: () => linkHomeName(connection, sidecarOrNull) } : {}), }; - const relayEnabled = !linkMode && connection.managementTransport === "relay"; + // A link Child relays its dashboard's management reads to the Home over the link (#6957); + // a hub client relays over its management transport when that is `relay`. + const relayEnabled = linkMode || connection.managementTransport === "relay"; // Link mode only. Everything is resolved once here, so a relayed request reads no file and // builds no policy: the key is cached, the loopback policy is fixed at bind like the listener. const inboundBodyLimit = resolveInboundBodyLimitBytes(config.maxInboundBodyBytes); @@ -123,8 +130,6 @@ export function startMachineListener( relay: { fetchImpl: deps.fetchImpl, bodyLimitBytes: inboundBodyLimit, tunnel: deps.linkTunnel, ...deps.linkRelay }, } : null; - const readSidecar = deps.readSidecar ?? (() => readClientLinkState()); - return (deps.serve ?? (options => Bun.serve(options)))({ port: port ?? config.port ?? 10100, hostname: "127.0.0.1", @@ -147,9 +152,29 @@ export function startMachineListener( } if (url.pathname.startsWith("/api/machine/hub-relay/")) { if (!relayEnabled) return json404(req); + const prefix = "/api/machine/hub-relay"; + if (linkIngress) { + const suffix = `${url.pathname.slice(prefix.length)}${url.search}`; + // The dashboard's shared plane bootstraps here. On a link Child that plane is this + // listener, so it receives this listener's own session; no Home credential exists. + if (suffix === "/opencodex-session") { + if (req.method !== "GET") return json404(req); + const session = issueGuiSession(req, config, managementAuth, { trustedTailscaleIngress: false }); + return session ? serveSessionBootstrap(session) : Response.json({ error: "session unavailable" }, { status: 503 }); + } + const authError = requireManagementAuth(req, managementAuth, config); + if (authError) return authError; + if (managementPrincipal(req, managementAuth, config) !== "gui-session") { + return Response.json({ error: "opencodex machine GUI session required" }, { status: 401 }); + } + const response = await relayLinkManagementRead(req, suffix, linkIngress, linkHomePhrase(connection, sidecarOrNull)); + if (response.status === 401) hubReachability = "unauthorized"; + else if (response.status >= 500) hubReachability = "offline"; + else if (response.status !== 403) hubReachability = "online"; + return response; + } const authError = requireMachineAuth(req, managementAuth, config); if (authError) return authError; - const prefix = "/api/machine/hub-relay"; const suffix = `${url.pathname.slice(prefix.length)}${url.search}`; const response = await relayHubManagementRequest(req, suffix, { managementUrl: connection.managementUrl, diff --git a/src/codex/auth-api/account-list.ts b/src/codex/auth-api/account-list.ts index 5af3075b762..267a39d1219 100644 --- a/src/codex/auth-api/account-list.ts +++ b/src/codex/auth-api/account-list.ts @@ -33,6 +33,7 @@ import { fetchMainAccountInfoAttempt, EMPTY_MAIN_ACCOUNT_INFO, mainResetCreditsF import { fetchPoolAccountQuota, PoolQuotaProbeBusyError, POOL_QUOTA_REFRESH_CONCURRENCY } from "./pool-quota-probe"; import type { PoolQuotaResult } from "./pool-quota-probe"; import { getRuntimeConfig, configuredPoolAccount, mapWithConcurrency } from "./runtime-config"; +import { isPassiveRead } from "../../lib/passive-read"; export function quotaForPlan | StoredAccountQuota | null>( quota: T, @@ -274,13 +275,25 @@ export async function listCodexAuthAccountsSnapshot( pruneCodexCredits([MAIN_CODEX_ACCOUNT_ID, ...poolAccounts.map(account => account.id)]); // One redaction decision for the whole snapshot, read once from the operator's config (#3859). const maskEmails = emailMaskingEnabled(runtimeConfig); - const mainResult = await fetchMainAccountInfoAttempt(forceRefresh, 1, undefined, false, - forceRefresh, false, runtimeConfig); + // A passive read (a Remote Link Child's, #6959) reports what this machine already observed: no + // token refresh, no WHAM probe, and so none of the plan or credential writes they make. + const passive = isPassiveRead(); + const mainResult: Awaited> = passive + ? { info: getMainAccountInfoCache() ?? EMPTY_MAIN_ACCOUNT_INFO, credentialChecked: false, hasCredential: false, + identityGeneration: captureMainAccountIdentityGeneration() } + : await fetchMainAccountInfoAttempt(forceRefresh, 1, undefined, false, forceRefresh, false, runtimeConfig); const refreshedPool = await mapWithConcurrency(poolAccounts, POOL_QUOTA_REFRESH_CONCURRENCY, async account => { const cred = getCodexAccountCredential(account.id); let quotaResult: PoolQuotaResult; if (!cred) { quotaResult = { quota: null, needsReauth: true }; + } else if (passive) { + quotaResult = { + quota: getAccountQuota(account.id), + needsReauth: false, + credentialGeneration: readCodexAccountRecord(account.id)?.generation, + quotaProbeSkipped: true, + }; } else { try { // Credits are process-local while pool quota is hydrated from disk, so right after a @@ -410,6 +423,8 @@ export async function listCodexAuthAccountsSnapshot( }; // Project the refusal and DTO in the same owned synchronous observation. A same-account grant // replacement during earlier pool probes must not inherit the previous grant's refusal. + // A passive read opens no claim database (its lock file is hardened on open). + if (passive) return projectSnapshot(); const projectionLease = tryAcquireNativeMainProfileClaim(); if (!projectionLease) return projectSnapshot(); try { diff --git a/src/codex/catalog/bundled.ts b/src/codex/catalog/bundled.ts index 795858214fb..5ffb7590ca8 100644 --- a/src/codex/catalog/bundled.ts +++ b/src/codex/catalog/bundled.ts @@ -28,6 +28,7 @@ import { import type { NormalizedComboConfig } from "../../combos/types"; import { providerDestinationResolvedError } from "../../lib/destination-policy"; import { redactSecretString } from "../../lib/redact"; +import { isPassiveRead } from "../../lib/passive-read"; import upstreamModelsSnapshot from "../data/upstream-models.json"; import { decodeOverlayState } from "../shim-state-file"; @@ -377,6 +378,8 @@ export function loadBundledCodexCatalogAsync( if (useCache && bundledCatalogFlight?.inputs === inputs && bundledCatalogFlight.epoch === epoch) { return view(bundledCatalogFlight.promise); } + // A passive read (a Remote Link Child's, #6959) runs no Codex binary and persists no runtime. + if (isPassiveRead()) return Promise.resolve(null); const controller = new AbortController(); const flight = { inputs, epoch, controller, promise: Promise.resolve(null) }; if (useCache && !signal && isCurrent === catalogAlwaysCurrent) { diff --git a/src/codex/native-profile-startup.ts b/src/codex/native-profile-startup.ts index 8d332787782..076425b44ed 100644 --- a/src/codex/native-profile-startup.ts +++ b/src/codex/native-profile-startup.ts @@ -20,6 +20,7 @@ import { withNativeMainExclusiveClaim } from "./native-main-claim"; import { scrubNativeMainAuthTempResidues } from "./native-main-auth-temp"; import { NATIVE_STAGE_SWEEP_INTERVAL_MS } from "./native-profile-stage-store"; import type { NativeCodexOwnership } from "../integrations/native/ownership-preflight"; +import { isPassiveRead } from "../lib/passive-read"; export type NativeMainStartupGateSnapshot = | { status: "ready"; homeId: string | null } @@ -701,7 +702,9 @@ export async function releaseNativeMainStartupLifecycle(server: object): Promise export function isNativeMainTrafficBlocked(): boolean { const reason = activeServiceOwnershipBlockReason(); - if (reason !== null && reprobeServiceOwnership(reason)) { + // A passive read (a Remote Link Child's, #6959) reports the gate as it stands: re-inspecting + // service ownership can activate the native-main lifecycle and its recovery on this machine. + if (reason !== null && !isPassiveRead() && reprobeServiceOwnership(reason)) { // The host became ownable after boot (#2108): the fence lifts here rather than // waiting for the restart the reporter had to perform by hand. return activeServiceOwnershipBlockReason() !== null || snapshot.status === "blocked"; diff --git a/src/codex/runtime.ts b/src/codex/runtime.ts index 32b9fb65a0e..227acf2529c 100644 --- a/src/codex/runtime.ts +++ b/src/codex/runtime.ts @@ -8,6 +8,7 @@ import { resolveCodexHomeDir } from "./home"; import { decodeOverlayState } from "./shim-state-file"; import { redactSecretString, redactUserPath } from "../lib/redact"; import { SUBPROCESS_KILL_GRACE_MS, waitForSubprocessExit } from "../lib/bounded-subprocess"; +import { isPassiveRead } from "../lib/passive-read"; export type CodexRuntimeSource = | "environment" @@ -984,7 +985,8 @@ export function getCodexRuntimeSnapshot(): ResolveCodexRuntimeResult { const cacheKey = resolveCacheKey({}); const memo = resolveCache?.key === cacheKey ? resolveCache : null; if (memo && Date.now() - memo.at < RESOLVE_CACHE_MS) return cloneAndDeepFreeze(memo.value); - resolveCodexRuntimeAsync().catch(() => { /* the next read retries */ }); + // A passive read (a Remote Link Child's, #6959) starts no version probes on this machine. + if (!isPassiveRead()) resolveCodexRuntimeAsync().catch(() => { /* the next read retries */ }); if (memo) return cloneAndDeepFreeze(memo.value); return resolveCodexRuntime({ discoverAlternatives: false, probeVersion: false }); } diff --git a/src/config/load-degrade.ts b/src/config/load-degrade.ts index 03d342451ef..d994d7cedfd 100644 --- a/src/config/load-degrade.ts +++ b/src/config/load-degrade.ts @@ -16,6 +16,7 @@ import { } from "./provider-validation"; import { UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD } from "../codex/upstream-host-health"; import { hardenSecretPath } from "../lib/windows-secret-acl"; +import { isPassiveRead } from "../lib/passive-read"; import { redactSecretString } from "../lib/redact"; import { isValidProviderName } from "./provider-name"; import { MODEL_ALIAS_PATTERN } from "../providers/default-aliases"; @@ -49,7 +50,7 @@ import { import { hasWarnedInheritedFastWireConflict, markWarnedInheritedFastWireConflict } from "./warn-memo"; export function hardenExistingSecret(path: string): void { - if (existsSync(path)) { + if (existsSync(path) && !isPassiveRead()) { try { chmodSync(path, 0o600); } catch { /* best-effort */ } if (process.platform === "win32") { hardenSecretPath(path, { required: false }); diff --git a/src/config/paths.ts b/src/config/paths.ts index dff11480487..e241d026959 100644 --- a/src/config/paths.ts +++ b/src/config/paths.ts @@ -3,6 +3,7 @@ import { homedir } from "node:os"; import { join, resolve } from "node:path"; import { flushWindowsSecretAclReapsBeforeRemoval, hardenSecretDirAsync, windowsSecretAclApplies } from "../lib/windows-secret-acl"; import { assertNotRealHomeUnderTest } from "../lib/test-home-guard"; +import { isPassiveRead } from "../lib/passive-read"; /** * Expand a leading `~` in user-supplied paths without interpreting shell @@ -29,6 +30,8 @@ export function getConfigPath(): string { } export function hardenConfigDir(): void { + // A passive read (a Remote Link Child's, #6959) repairs nothing on this machine. + if (isPassiveRead()) return; const dir = getConfigDir(); // The guard runs before any mutation: refusing after chmod/ACL would already // have changed the protected directory used by the test-home boundary. diff --git a/src/config/salvage.ts b/src/config/salvage.ts index 254e4e19f62..624e67ed58a 100644 --- a/src/config/salvage.ts +++ b/src/config/salvage.ts @@ -3,6 +3,7 @@ import { join } from "node:path"; import * as z from "zod/v4"; import { CODEX_ACCOUNT_NAMESPACE_COMBO_ALIAS_COLLISION_ERROR } from "../codex/account-namespace-match"; import { redactSecretString } from "../lib/redact"; +import { isPassiveRead } from "../lib/passive-read"; import { hasWarnedConfigFallback, markWarnedConfigFallback } from "./warn-memo"; import { configSchema } from "./schema/config-schema"; import type { OcxConfig } from "../types"; @@ -232,7 +233,8 @@ export function warnAndBackupInvalidConfig(configPath: string, error: unknown): } export function backupInvalidConfig(configPath: string): string | null { - if (!existsSync(configPath)) return null; + // A passive read leaves a malformed file for the Home's own load to back up (#6959). + if (!existsSync(configPath) || isPassiveRead()) return null; const backupPath = `${configPath}.invalid-${new Date().toISOString().replace(/[:.]/g, "-")}`; try { copyFileSync(configPath, backupPath); diff --git a/src/lib/app-owned-memory-stores.ts b/src/lib/app-owned-memory-stores.ts index 70f7215f901..ac0516d6738 100644 --- a/src/lib/app-owned-memory-stores.ts +++ b/src/lib/app-owned-memory-stores.ts @@ -169,12 +169,16 @@ export const APP_OWNED_RETAINED_STORE_REGISTRATIONS = [ category: "caches", snapshot: usageSummaryRetainedStoreSnapshot, evictOldest: evictOldestUsageSummaryForBudget, + // Deletes one in-memory summary entry. + passiveEvictable: true, }, { id: "usage_snapshot", category: "caches", snapshot: usageSnapshotRetainedStoreSnapshot, evictOldest: evictOldestUsageSnapshot, + // Drops the retained ledger snapshot or one unpinned aggregate, both in memory. + passiveEvictable: true, }, { id: "cursor_blobs", diff --git a/src/lib/app-owned-memory.ts b/src/lib/app-owned-memory.ts index a1a3a4b9668..c48da92789f 100644 --- a/src/lib/app-owned-memory.ts +++ b/src/lib/app-owned-memory.ts @@ -1,3 +1,5 @@ +import { isPassiveRead } from "./passive-read"; + export type AppOwnedRetainedCategory = "logs" | "caches" | "blobs" | "continuation"; export type AppOwnedObservedCategory = "translator" | "serialized_tails"; @@ -14,6 +16,11 @@ export interface RetainedStoreRegistration { category: AppOwnedRetainedCategory; snapshot(): RetainedStoreSnapshot; evictOldest(): number; + /** + * Eviction only releases memory: no spill, snapshot or other write. Only such stores are + * demoted inside a passive read (#6959); the rest wait for this process's own next enforcement. + */ + passiveEvictable?: true; } export interface ObservedBufferRegistration { @@ -168,11 +175,15 @@ function nextCandidate( snapshots: ReadonlyMap, ineligible: Set, ): RetainedStoreRegistration | undefined { + // A passive read (a Remote Link Child's, #6959) demotes only stores whose eviction is memory-only: + // a continuation spills to disk and a replay cache rewrites its snapshot. + const passive = isPassiveRead(); for (const category of CATEGORY_ORDER) { let candidate: RetainedStoreRegistration | undefined; let candidateAt = Number.POSITIVE_INFINITY; for (const registration of retainedStores.values()) { if (registration.category !== category || ineligible.has(registration.id)) continue; + if (passive && registration.passiveEvictable !== true) continue; const snapshot = snapshots.get(registration.id); if (!snapshot || snapshot.evictableBytes <= 0) continue; const oldestAt = snapshot.oldestAt; diff --git a/src/lib/passive-read.ts b/src/lib/passive-read.ts new file mode 100644 index 00000000000..98966b9c34a --- /dev/null +++ b/src/lib/passive-read.ts @@ -0,0 +1,35 @@ +/** + * A passive management read: it shows what this process already knows and does not make it act on + * providers, accounts, processes or files (#6959). + * + * The Remote Link management relay runs every Child read inside this scope + * (`src/server/index/link-management.ts`), so a Child can show its Home's state without changing + * it or making it act. Inside the scope: + * + * - model discovery yields no routed models (`fetchAllModels`, the management model rows): it would + * reach provider upstreams, refresh Codex entitlements and persist a pending provider's initial + * model selection; + * - provider quota and the Codex account list are served from what the Home has observed, with no + * probe, token refresh, native-main claim or publication into routing evidence; + * - the startup-health diagnostic, the Codex runtime version probes and the bundled-catalog refresh + * are not started, so no process runs and no runtime record is written; + * - a native-main fence raised for unknown service ownership is reported, not re-inspected; + * - config-directory and credential-file hardening and malformed-file backups are skipped; + * - memory-budget enforcement demotes only stores registered as `passiveEvictable` (eviction that + * only releases memory: the usage caches), never a continuation spill or a replay snapshot. + * + * The Home's own dashboard and traffic keep those caches current. A passive read may still join + * work the Home already started; that work keeps the Home's scope and its writes are the Home's. + */ +import { AsyncLocalStorage } from "node:async_hooks"; + +const passive = new AsyncLocalStorage(); + +/** Run a management read passively, including everything it awaits or schedules. */ +export function runPassiveRead(run: () => Promise): Promise { + return passive.run(true, run); +} + +export function isPassiveRead(): boolean { + return passive.getStore() === true; +} diff --git a/src/link/codex-settings.ts b/src/link/codex-settings.ts new file mode 100644 index 00000000000..8639e71cca8 --- /dev/null +++ b/src/link/codex-settings.ts @@ -0,0 +1,155 @@ +/** + * The Codex settings a Remote Link Home decides for its Children (#6957). + * + * A link Child relays every model request to its Home, so everything the Home's proxy applies + * per request already follows the Home: effort caps, the sub-agent roster and fallback, the + * injection prompt, sidecar and compaction routing, and the catalog itself (served by + * `/v1/catalog`). What does NOT follow is the part of the Home's OpenCodex config that the + * injector writes into the Child's own Codex `config.toml`. This module carries exactly that + * part, which is every OpenCodex config field the Codex injection plan reads that is a policy + * decision rather than a fact about the Child's own listener: + * + * - `fastMode` -> `[features] fast_mode` + * - `multiAgentMode` -> the native `features.multi_agent_v2` reconcile (v1 mode) + * - `webSearchSidecar.enabled` -> the root `web_search` switch + * - `syncCodexSubagentDefaults` + `injectionModel` + `injectionEffort` -> the managed + * `[agents]` defaults + * + * Root `model_reasoning_effort` and the `desktop.*` picker keys are Codex app settings that + * OpenCodex never writes on any machine, so they are deliberately not part of this document. + * Listener facts (port, hostname, Desktop authless and client compaction routing forms, resume + * history) stay the Child's own because they describe how the Child's Codex reaches the Child. + * + * It rides on its own authenticated link read rather than inside the catalog because the + * catalog bytes are written verbatim to `model_catalog_json`, which Codex parses itself. + */ +import { hostname } from "node:os"; +import type { OcxConfig } from "../types"; + +export const HOME_CODEX_SETTINGS_VERSION = 1; +const MULTI_AGENT_MODES = ["v1", "default", "v2"] as const; +type MultiAgentMode = (typeof MULTI_AGENT_MODES)[number]; + +export interface HomeCodexSettings { + /** null: the Home leaves `fast_mode` alone. */ + fastMode: boolean | null; + /** null: the Home has no explicit mode. */ + multiAgentMode: MultiAgentMode | null; + webSearchSidecarEnabled: boolean; + subagentDefaults: { model: string; reasoningEffort: string | null } | null; +} + +export interface HomeCodexSettingsDocument { + version: typeof HOME_CODEX_SETTINGS_VERSION; + home: { name: string; version: string }; + settings: HomeCodexSettings; +} + +const NAME_MAX = 255; +const VALUE_MAX = 256; + +function safeLabel(value: string, max: number): string | null { + const trimmed = value.trim(); + if (!trimmed || trimmed.length > max) return null; + for (const character of trimmed) { + const code = character.codePointAt(0) ?? 0; + if (code < 0x20 || code === 0x7f || (code >= 0x202a && code <= 0x202e) || (code >= 0x2066 && code <= 0x2069)) return null; + } + return trimmed; +} + +export function projectHomeCodexSettings(config: OcxConfig, version: string): HomeCodexSettingsDocument { + const mode = config.multiAgentMode; + // Same predicate as `subagentDefaultSyncEffective` (src/config/diagnostics.ts), inlined so + // this module stays inside the src/link import boundary. + const sync = config.syncCodexSubagentDefaults === true && Boolean(config.injectionModel?.trim()); + const effort = config.injectionEffort?.trim(); + return { + version: HOME_CODEX_SETTINGS_VERSION, + home: { name: safeLabel(hostname(), NAME_MAX) ?? "Home", version }, + settings: { + fastMode: typeof config.fastMode === "boolean" ? config.fastMode : null, + multiAgentMode: mode && (MULTI_AGENT_MODES as readonly string[]).includes(mode) ? mode : null, + webSearchSidecarEnabled: config.webSearchSidecar?.enabled !== false, + subagentDefaults: sync + ? { model: config.injectionModel!.trim(), reasoningEffort: effort ? effort : null } + : null, + }, + }; +} + +function exactKeys(value: unknown, keys: readonly string[]): value is Record { + if (!value || typeof value !== "object" || Array.isArray(value)) return false; + const actual = Object.keys(value); + return actual.length === keys.length && actual.every(key => keys.includes(key)); +} + +/** Strict parse: an unknown field, a wrong type or an unsafe label rejects the whole document. */ +export function parseHomeCodexSettingsDocument(value: unknown): HomeCodexSettingsDocument | null { + if (!exactKeys(value, ["version", "home", "settings"]) || value.version !== HOME_CODEX_SETTINGS_VERSION) return null; + const home = value.home; + if (!exactKeys(home, ["name", "version"]) || typeof home.name !== "string" || typeof home.version !== "string") return null; + const name = safeLabel(home.name, NAME_MAX); + const homeVersion = safeLabel(home.version, 64); + if (!name || !homeVersion) return null; + const s = value.settings; + if (!exactKeys(s, ["fastMode", "multiAgentMode", "webSearchSidecarEnabled", "subagentDefaults"])) return null; + if (s.fastMode !== null && typeof s.fastMode !== "boolean") return null; + if (s.multiAgentMode !== null && !(MULTI_AGENT_MODES as readonly unknown[]).includes(s.multiAgentMode)) return null; + if (typeof s.webSearchSidecarEnabled !== "boolean") return null; + let subagentDefaults: HomeCodexSettings["subagentDefaults"] = null; + if (s.subagentDefaults !== null) { + const d = s.subagentDefaults; + if (!exactKeys(d, ["model", "reasoningEffort"]) || typeof d.model !== "string") return null; + const model = safeLabel(d.model, VALUE_MAX); + if (!model) return null; + let reasoningEffort: string | null = null; + if (d.reasoningEffort !== null) { + if (typeof d.reasoningEffort !== "string" || !/^[a-z][a-z0-9_-]{0,31}$/.test(d.reasoningEffort)) return null; + reasoningEffort = d.reasoningEffort; + } + subagentDefaults = { model, reasoningEffort }; + } + return { + version: HOME_CODEX_SETTINGS_VERSION, + home: { name, version: homeVersion }, + settings: { + fastMode: s.fastMode as boolean | null, + multiAgentMode: s.multiAgentMode as MultiAgentMode | null, + webSearchSidecarEnabled: s.webSearchSidecarEnabled, + subagentDefaults, + }, + }; +} + +/** The Child's config as the Codex injector should see it: local listener facts, Home policy. */ +export function overlayHomeCodexSettings(config: OcxConfig, settings: HomeCodexSettings): OcxConfig { + const next: OcxConfig = { + ...config, + webSearchSidecar: { ...(config.webSearchSidecar ?? {}), enabled: settings.webSearchSidecarEnabled }, + }; + if (settings.fastMode === null) delete next.fastMode; + else next.fastMode = settings.fastMode; + if (settings.multiAgentMode === null) delete next.multiAgentMode; + else next.multiAgentMode = settings.multiAgentMode; + if (settings.subagentDefaults) { + next.injectionModel = settings.subagentDefaults.model; + next.syncCodexSubagentDefaults = true; + if (settings.subagentDefaults.reasoningEffort) next.injectionEffort = settings.subagentDefaults.reasoningEffort; + else delete next.injectionEffort; + } else { + next.syncCodexSubagentDefaults = false; + } + return next; +} + +/** One line per applied value, for `ocx sync`, `ocx status` and the dashboard. */ +export function describeHomeCodexSettings(settings: HomeCodexSettings): string[] { + const d = settings.subagentDefaults; + return [ + `fast mode: ${settings.fastMode === null ? "unset" : settings.fastMode ? "on" : "off"}`, + `multi-agent mode: ${settings.multiAgentMode ?? "default"}`, + `web search sidecar: ${settings.webSearchSidecarEnabled ? "on" : "off"}`, + `native sub-agent defaults: ${d ? `${d.model}${d.reasoningEffort ? ` (${d.reasoningEffort})` : ""}` : "not synced"}`, + ]; +} diff --git a/src/link/management-routes.ts b/src/link/management-routes.ts new file mode 100644 index 00000000000..3ac88deaedd --- /dev/null +++ b/src/link/management-routes.ts @@ -0,0 +1,89 @@ +/** + * The read-only management surface a Remote Link Child may reach on its Home (#6957). + * + * Default deny: a path is readable only when it is listed here, the method is GET or HEAD, and + * the query does not ask the Home to refresh upstream state. Everything else is refused on both + * ends: the Child's listener and CLI refuse before sending, and the Home's link listener answers + * 404 for any other path under the prefix. No route here returns a credential value; provider + * and account projections report presence and masked identities only. + * + * A link read must not change the Home. The Home runs these reads passively + * (`src/lib/passive-read.ts`): model discovery, which reaches provider upstreams and persists the + * initial model selection of a pending provider, config writes and diagnostics do not run (#6959). + * `tests/server/link-management-relay.test.ts` reads every path here and asserts that the Home + * config and its upstreams are untouched. + */ +export const LINK_MANAGEMENT_PREFIX = "/v1/link-management"; +/** The Home's OpenCodex-owned Codex settings a Child applies when it syncs. */ +export const LINK_CODEX_SETTINGS_PATH = `${LINK_MANAGEMENT_PREFIX}/codex-settings`; + +export const LINK_MANAGEMENT_READ_PATHS: readonly string[] = [ + "/api/system/health", + "/api/startup-health", + "/api/providers", + "/api/settings", + "/api/usage", + "/api/effort-caps", + "/api/injection-model", + "/api/shadow-call-settings", + "/api/sidecar-settings", + "/api/v2", + "/api/provider-quotas", + "/api/codex-auth/active", + "/api/codex-auth/accounts", + "/api/oauth/accounts", + "/api/providers/keys", + "/api/oauth/providers", + "/api/catalog", + "/api/system/memory", + "/api/provider-presets", + "/api/provider-context-caps", +]; +const READ_PATHS = new Set(LINK_MANAGEMENT_READ_PATHS); +/** + * Query keys that make a read trigger upstream work on the Home: forced refreshes and the + * per-account quota probes `?quota=1` starts on the OAuth-account and key-pool reads. + * `/api/models`, `/api/model-presets`, `/api/subagent-models` and `/api/subagent-model-fallback` + * are absent for the same reason: they run live provider discovery and may persist initial model + * selection. The preset preview exists only to decide a write a Child cannot make. + */ +const REFRESH_KEYS = ["refresh", "reload", "force", "quota"]; + +/** Is `method path?query` (a Home management path, without the link prefix) readable over the link? */ +export function linkManagementReadAllowed(method: string, pathname: string, search: URLSearchParams | string = ""): boolean { + const upper = method.toUpperCase(); + if (upper !== "GET" && upper !== "HEAD") return false; + if (!READ_PATHS.has(pathname)) return false; + const query = typeof search === "string" ? new URLSearchParams(search) : search; + return !REFRESH_KEYS.some(key => query.has(key)); +} + +/** The Home management URL a prefixed link request names, or null when it names none. */ +export function linkManagementTarget(url: URL): URL | null { + if (!url.pathname.startsWith(`${LINK_MANAGEMENT_PREFIX}/api/`)) return null; + const target = new URL(url.href); + target.pathname = url.pathname.slice(LINK_MANAGEMENT_PREFIX.length); + return target; +} + +/** Home side: may the hub-link listener dispatch this request to the link management handler? */ +export function linkManagementRouteAllowed(url: URL, req: Request): boolean { + if (req.headers.has("upgrade")) return false; + if (url.pathname === LINK_CODEX_SETTINGS_PATH) return req.method === "GET" && url.search === ""; + const target = linkManagementTarget(url); + return target !== null && linkManagementReadAllowed(req.method, target.pathname, target.searchParams); +} + +export function isLinkManagementPath(pathname: string): boolean { + return pathname === LINK_MANAGEMENT_PREFIX || pathname.startsWith(`${LINK_MANAGEMENT_PREFIX}/`); +} + +/** + * The one refusal every management surface on a link Child uses: the dashboard relay, the CLI, + * and the Child's listener. It names the Home and the link and never suggests a relay transport. + * `homePhrase` is "Home " or "its Home" when the Child does not know the Home's name. + */ +export function linkChildManagementRefusal(homePhrase: string, linkId: string, operation = "This management command"): string { + return `This machine is a Remote Link Child managed by ${homePhrase} (link ${linkId}). ` + + `${operation} runs on the Home, not on this Child: use the Home's dashboard or run the command there. No request was sent.`; +} diff --git a/src/providers/quota.ts b/src/providers/quota.ts index a6403fbfdcd..9a5cb7ed944 100644 --- a/src/providers/quota.ts +++ b/src/providers/quota.ts @@ -15,6 +15,7 @@ import { } from "./kiro-usage"; import { mapQuotaRoster, readProviderApiKeyQuotas, type ProviderApiKeyQuota } from "./quota-key-accounts"; import type { OcxConfig, OcxProviderConfig } from "../types"; +import { isPassiveRead } from "../lib/passive-read"; import type { ProviderQuota, QuotaFailureCode } from "./quota-types"; import { accountReportCurrent, @@ -23,7 +24,7 @@ import { captureProviderReportEpochs, providerReportEpochChanged, cacheKeyWithAggregationState, getProviderQuotaReportCache, - hasCodexPoolProvider, + hasCodexPoolProvider, heldProviderQuotaReports, inflight, invalidationEpoch, isBuiltInChatGptForwardProvider, @@ -229,6 +230,8 @@ export function flushProviderQuotaObservationsForTests(): Promise { } export async function fetchProviderQuotaReports(config: OcxConfig, forceRefresh = false): Promise { + // A passive read (a Remote Link Child's, #6959) shows the held reports and starts nothing. + if (isPassiveRead()) return heldProviderQuotaReports(Object.keys(config.providers)); // A Pool report's cache signature and provider fetch must share one account snapshot. // Preserve force semantics when deciding whether that snapshot refreshes upstream data. const prefetchedCodexSnapshot = hasCodexPoolProvider(config) diff --git a/src/providers/quota/report-cache.ts b/src/providers/quota/report-cache.ts index 8329e10f63e..98ab3ec640f 100644 --- a/src/providers/quota/report-cache.ts +++ b/src/providers/quota/report-cache.ts @@ -344,3 +344,17 @@ export function isProviderQuotaReportCurrent(value: ProviderQuotaReport): boolea return (generation === undefined || isMainAccountIdentityGenerationLive(generation)) && (accountReportCurrent.get(value)?.() ?? true); } + +/** + * The reports this process already holds for the configured providers, for a passive read (#6959): + * no probe, token refresh, account snapshot or publication into routing evidence. The Home's own + * dashboard and traffic keep the cache current. + */ +export function heldProviderQuotaReports(configuredProviders: readonly string[]): ProviderQuotaResponse { + const held = getProviderQuotaReportCache()?.response; + const configured = new Set(configuredProviders); + return { + generatedAt: held?.generatedAt ?? Date.now(), + reports: (held?.reports ?? []).filter(item => configured.has(item.provider) && isProviderQuotaReportCurrent(item)), + }; +} diff --git a/src/server/index/link-management.ts b/src/server/index/link-management.ts new file mode 100644 index 00000000000..2dece895c61 --- /dev/null +++ b/src/server/index/link-management.ts @@ -0,0 +1,58 @@ +/** + * The Home end of the Remote Link management relay (#6957). + * + * Reached only on the hub-link listener, after the request has passed its link-key admission + * (`resolveApiAuth` restricted to recorded link keys) and the shared route table in + * `src/link/management-routes.ts`. The management handler sees a fresh request built here: no + * caller header, cookie, Origin or credential crosses from the link, and the principal is + * absent, so no consent- or capability-gated branch can be reached. Only allowlisted reads run. + * They run as passive reads (`src/lib/passive-read.ts`): no discovery, config write or diagnostic (#6959). + */ +import type { OcxConfig } from "../../types"; +import { handleManagementAPI, type ManagementApiDeps } from "../management-api"; +import type { ManagementSessionControl } from "../management-auth"; +import { runPassiveRead } from "../../lib/passive-read"; +import { LINK_CODEX_SETTINGS_PATH, linkManagementRouteAllowed, linkManagementTarget } from "../../link/management-routes"; +import { projectHomeCodexSettings } from "../../link/codex-settings"; +import { packageVersion } from "../../lib/package-version"; + +const VERSION = packageVersion("0.0.0"); +const KEPT_RESPONSE_HEADERS = ["content-type", "content-language", "etag", "last-modified"]; + +function notFound(): Response { + return Response.json({ error: "not_found" }, { status: 404, headers: { "cache-control": "no-store" } }); +} + +export async function handleLinkManagementRequest( + req: Request, + url: URL, + config: OcxConfig, + deps: ManagementApiDeps, + sessionControl?: ManagementSessionControl, +): Promise { + if (!linkManagementRouteAllowed(url, req)) return notFound(); + if (url.pathname === LINK_CODEX_SETTINGS_PATH) { + return Response.json(projectHomeCodexSettings(config, VERSION), { headers: { "cache-control": "no-store" } }); + } + const target = linkManagementTarget(url); + if (!target) return notFound(); + const local = new URL(`${target.pathname}${target.search}`, "http://127.0.0.1"); + // The handlers branch on GET; an admitted HEAD runs as GET and its body is dropped below. + const inner = new Request(local.href, { + method: "GET", + headers: { accept: "application/json", host: "127.0.0.1" }, + }); + const response = await runPassiveRead(() => + handleManagementAPI(inner, local, config, deps, undefined, sessionControl, { trustedLoopback: false })); + if (!response) return notFound(); + const headers = new Headers({ "cache-control": "no-store" }); + for (const name of KEPT_RESPONSE_HEADERS) { + const value = response.headers.get(name); + if (value) headers.set(name, value); + } + if (req.method === "HEAD") { + try { await response.body?.cancel(); } catch { /* best effort */ } + return new Response(null, { status: response.status, headers }); + } + return new Response(response.body, { status: response.status, headers }); +} diff --git a/src/server/index/optional-listeners.ts b/src/server/index/optional-listeners.ts index 7fbf5b14470..7a3b85069ea 100644 --- a/src/server/index/optional-listeners.ts +++ b/src/server/index/optional-listeners.ts @@ -14,6 +14,7 @@ import { type LinkListenerStatus, } from "./link-listener"; import { createLinkSupervisor, type LinkSupervisor } from "../../link/supervisor"; +import { linkManagementRouteAllowed } from "../../link/management-routes"; import { saveConfigPreservingClaudeCode } from "../../config/live-reconcile"; import { reconcileLiveStateStores } from "../../lib/state-store-registrations"; export { LINK_INGRESS_HOSTNAME } from "./link-listener"; @@ -77,7 +78,10 @@ export function createOptionalListenerSet(linkDeps: LinkListenerDeps = {}): O if (claudeIntercept.ownsListener(server)) return "claude-intercept"; return undefined; }, - linkRouteAllowed, + // The Home's link listener admits the data plane plus the read-only management relay + // (#6957). The Child's own ingress keeps the data-only table, so a local caller on the Child + // cannot reach the Home's management reads without the Child's dashboard session. + linkRouteAllowed: (url, req) => linkRouteAllowed(url, req) || linkManagementRouteAllowed(url, req), linkAdmissionKeyIds: () => linkListener.linkAdmissionKeyIds(), onAuthenticatedCatalog: listener => linkListener.onAuthenticatedCatalog(listener), notifyAuthenticatedCatalog: apiKeyId => linkListener.notifyAuthenticatedCatalog(apiKeyId), diff --git a/src/server/index/serve-options.ts b/src/server/index/serve-options.ts index f8a2f32ed99..bd94665772a 100644 --- a/src/server/index/serve-options.ts +++ b/src/server/index/serve-options.ts @@ -1,5 +1,6 @@ import type { Server, ServerWebSocket } from "bun"; import type { StartServerDeps } from "./startup-warnings"; +import { isLinkManagementPath } from "../../link/management-routes"; import { GUI_PAIRING_EXCHANGE_BODY_LIMIT, REMOTE_WORKSPACE_PAIRING_BODY_LIMIT, @@ -373,6 +374,12 @@ export function createServeOptions(ctx: ServeOptionsContext) { if (requestUrl.pathname === "/v1/catalog" && requestUrl.search === "" && (req.method === "GET" || req.method === "HEAD") && admission.kind === "configured") onAuthenticatedCatalog(admission.keyId); + // #6957: the Child's read-only view of this Home. Admitted by the link key above and by + // the shared table in src/link/management-routes.ts; never reachable on another ingress. + if (isLinkManagementPath(requestUrl.pathname)) { + const { handleLinkManagementRequest } = await import("./link-management"); + return withCors(await handleLinkManagementRequest(req, requestUrl, config, requestManagementApiDeps, managementSessionControl), req, policy); + } } const url = requestUrl; const admissionOptions = { linkIngress: policy.linkIngress?.allowedKeyIds }; diff --git a/src/server/management/agent-settings-routes.ts b/src/server/management/agent-settings-routes.ts index 2f23ba14a11..d42d821fde6 100644 --- a/src/server/management/agent-settings-routes.ts +++ b/src/server/management/agent-settings-routes.ts @@ -1,6 +1,7 @@ import { ensureManagementClaudeIntercept, interceptStartRefusal, interceptStatus } from "./claude-intercept-routes"; import { persistCommittedDesktopGateway } from "../../claude/desktop-gateway-state"; import { captureDesktopAppliedMarker, commitDesktopAppliedMarker } from "../../claude/desktop-applied-marker"; +import { isPassiveRead } from "../../lib/passive-read"; import { randomUUID } from "node:crypto"; import { readFileSync } from "node:fs"; import { dirname } from "node:path"; @@ -230,6 +231,32 @@ async function injectionModelOptions(config: OcxConfig, models: readonly Catalog return [...nativeModels, ...routedModels]; } +/** + * The GET /api/injection-model body. `routed` is the discovered routed catalog, or `null` in a + * passive read (a Remote Link Child's, #6959). That body lists the native + * options and the current pick, which is all a read-only Child shows. + */ +export async function injectionModelView(config: OcxConfig, routed: readonly CatalogModel[] | null) { + const { CODEX_REASONING_LEVELS } = await import("../../reasoning-effort"); + const available = await injectionModelOptions(config, routed ?? []); + const picked = config.injectionModel; + if (routed === null && picked && !available.some(option => option.namespaced === picked)) { + const slash = picked.indexOf("/"); + available.push(slash > 0 + ? { provider: picked.slice(0, slash), model: picked.slice(slash + 1), namespaced: picked } + : { provider: "openai", model: picked, namespaced: picked }); + } + return { + multiAgentGuidanceEnabled: multiAgentGuidanceEnabled(config), + syncCodexSubagentDefaults: subagentDefaultSyncEffective(config), + model: config.injectionModel ?? null, + effort: config.injectionEffort ?? null, + prompt: config.injectionPrompt ?? null, + efforts: CODEX_REASONING_LEVELS.map(l => l.effort), + available, + }; +} + export async function handleAgentSettingsRoutes(ctx: ManagementContext): Promise { const { req, url, config, deps, convergeCodexCatalog, syncClaudeAgentDefsBestEffort } = ctx; @@ -620,16 +647,7 @@ export async function handleAgentSettingsRoutes(ctx: ManagementContext): Promise // effort the prompt tells the agent to pass to spawn_agent. GET returns the current // picks + available models/efforts; PUT sets or clears them. if (url.pathname === "/api/injection-model" && req.method === "GET") { - const { CODEX_REASONING_LEVELS } = await import("../../reasoning-effort"); - return jsonResponse({ - multiAgentGuidanceEnabled: multiAgentGuidanceEnabled(config), - syncCodexSubagentDefaults: subagentDefaultSyncEffective(config), - model: config.injectionModel ?? null, - effort: config.injectionEffort ?? null, - prompt: config.injectionPrompt ?? null, - efforts: CODEX_REASONING_LEVELS.map(l => l.effort), - available: await injectionModelOptions(config, await fetchAllModels(config)), - }); + return jsonResponse(await injectionModelView(config, isPassiveRead() ? null : await fetchAllModels(config))); } // Read-only: sizes the described delegated work and proposes a model and effort from the // same options GET offers. The page applies an accepted proposal through the PUT below. diff --git a/src/server/management/model-rows.ts b/src/server/management/model-rows.ts index 66e99657ae3..b764009e364 100644 --- a/src/server/management/model-rows.ts +++ b/src/server/management/model-rows.ts @@ -11,6 +11,7 @@ import type { CatalogModel } from "../../codex/catalog"; import { antigravityFamilyDisabled } from "../../providers/antigravity-effort-families"; import { observeModelCacheRevision } from "../../codex/model-cache"; +import { isPassiveRead } from "../../lib/passive-read"; import { captureExportConfigAdmission, detachedConfigSnapshot, @@ -306,8 +307,9 @@ export async function listManagementModelRows( * that brings its own roster still sees exactly what a writer would, and the two cannot * disagree about the roster for any reason except the roster itself. */ + // A passive read (#6959) also skips the entitlement refresh, which reaches the Codex accounts. const models = options.models === undefined - ? (await Promise.all([ + ? isPassiveRead() ? [] : (await Promise.all([ fetchAllModels(config, options.providerContentRevisions, projection => { config = projection; }), ensureCodexEntitlementFreshness(config, { waitMs: options.entitlementWaitMs ?? 3_000, diff --git a/src/server/management/shared.ts b/src/server/management/shared.ts index f5a0621922c..965f5b8f12c 100644 --- a/src/server/management/shared.ts +++ b/src/server/management/shared.ts @@ -3,6 +3,7 @@ import { captureInitialSelectionBaseline, finalizeInitialModelSelection } from " import { captureModelDiscoveryBaseline, finalizeModelDiscovery } from "../../providers/new-model-policy-runtime"; import { CatalogGatherBusyError } from "../../codex/catalog/routed-gather"; import { initialModelSelectionPending } from "../../providers/initial-model-selection"; +import { isPassiveRead } from "../../lib/passive-read"; import { readFileSync } from "node:fs"; import type { CatalogModel } from "../../codex/catalog"; import { catalogModelSlug, invalidateCodexModelsCache, nativeModelRows, uniqueCatalogModelsForPublicList } from "../../codex/catalog"; @@ -291,6 +292,8 @@ export async function fetchAllModels( /** Management renders disabled rows using this detached state after inventory drift. */ publishProjection?: (projection: OcxConfig) => void, ): Promise { + // A Remote Link read: no upstream, no initial selection, no write (#6959). + if (isPassiveRead()) return []; const { gatherRoutedModels } = await import("../../codex/catalog"); const baseline = captureInitialSelectionBaseline(config); const discoveryBaseline = captureModelDiscoveryBaseline(config); diff --git a/src/server/startup-health-cache.ts b/src/server/startup-health-cache.ts index 1f769438a0b..095a6552085 100644 --- a/src/server/startup-health-cache.ts +++ b/src/server/startup-health-cache.ts @@ -9,6 +9,7 @@ import { selfLaunchArgv } from "../lib/self-launch-argv"; import { desktopStartupOwnership } from "../service/desktop-startup"; import type { OcxConfig } from "../types"; import { truncateRetainedUtf8 } from "../lib/admission"; +import { isPassiveRead } from "../lib/passive-read"; const CACHE_TTL_MS = 30_000; const PROBE_TIMEOUT_MS = startupHealthProbeBudgetMs(); @@ -69,6 +70,8 @@ export function getStartupHealthSnapshot( ): StartupHealth { const now = deps.now ?? Date.now; if (cached && now() - cached.timestamp < CACHE_TTL_MS) return cached.value; + // A passive read (a Remote Link Child's, #6959) starts no diagnostic process on this machine. + if (isPassiveRead()) return staleOrFallback(config); refreshInBackground(config, deps); return staleOrFallback(config); } @@ -189,6 +192,7 @@ export async function getCachedStartupHealth( ): Promise { const now = deps.now ?? Date.now; if (cached && now() - cached.timestamp < CACHE_TTL_MS) return cached.value; + if (isPassiveRead()) return staleOrFallback(config); refreshInBackground(config, deps); // An expired or empty read is an explicit protection check. Wait for the // isolated probe instead of presenting a synthetic failure while that probe diff --git a/structure/gui-and-management-api.md b/structure/gui-and-management-api.md index 218b6318e00..87118ed14dc 100644 --- a/structure/gui-and-management-api.md +++ b/structure/gui-and-management-api.md @@ -425,13 +425,13 @@ changes. The queue lives in `gui/src/use-model-visibility.ts`; the dashboard pag ## The client role owns no management plane A connected client machine runs `src/client/machine-listener.ts` instead of the standalone server. -It binds the address the standalone proxy would (`port ?? config.port ?? 10100`) and serves -`GET /healthz`, `/readyz`, the packaged GUI/SPA routes, and `/api/machine/*`. A link-transport client -also relays the link data routes to its Home and serves its own read-only `GET /api/link/status` -([Remote Link](remote-link.md#dashboard-admission)). Every other `/api/*` and `/v1/*` path is -refused before dispatch with a JSON 404 naming the method and path. There is no second management -port on such a machine: management rides the same listener a standalone or hub install runs, so a -connected client has no mutating `/api/*` management surface at all. +It binds the address the standalone proxy would (`port ?? config.port ?? 10100`) and serves `GET /healthz`, +`/readyz`, the packaged GUI/SPA routes, and `/api/machine/*`. A link-transport client also relays the link data +routes to its Home, serves its own read-only `GET /api/link/status` ([Remote Link](remote-link.md#dashboard-admission)), +and relays allowlisted management reads to the Home under `/api/machine/hub-relay/api/*` for a Child dashboard session +([Remote Link](remote-link.md#child-management-relay-and-home-codex-settings)). Every other `/api/*` and `/v1/*` path is +refused before dispatch with a JSON 404 naming the method and path. There is no second management port on such a machine: +management rides the same listener a standalone or hub install runs, so a connected client has no mutating `/api/*` surface. The discriminator is `role` on `/healthz` and `/readyz`. The machine listener reports `role: "client"`; the standalone and hub server omit the field. `src/server/proxy-liveness.ts` parses diff --git a/structure/remote-link.md b/structure/remote-link.md index b55a1fb32d7..6885112bfa6 100644 --- a/structure/remote-link.md +++ b/structure/remote-link.md @@ -12,7 +12,7 @@ Every remote `ocx` call goes through `remoteOcxArgv`, a quoted POSIX `sh -c` scr `src/link/tunnel-state.ts` is the tunnel lifecycle reducer: connecting, connected, reconnecting with capped jittered backoff, and failed for auth, host-key and forward errors or after five minutes without a connection, whether or not an attempt is in flight. Without a retry policy, failed is terminal. `CLIENT_TUNNEL_RETRY_POLICY` gives failed a `retryAt`: timeout and forward failures are retried after 60 seconds and auth failures after five minutes, so no more than 12 auth attempts reach the peer's sshd in an hour; a host-key failure is never retried. The Child's own tunnel runs with it, and `src/link/supervisor.ts` runs the Home's `-R` tunnels with `HOME_TUNNEL_RETRY_POLICY`, the same policy. A retry runs while the state still reads failed (`inFlight`), a ready event promotes it to connected, and an exit returns it to failed with the original `since` and the next `retryAt`, so a long outage keeps the one-a-minute cadence instead of restarting fast backoff. The Home's supervisor promotes a first attempt to connected when its link key authenticates a catalog request or the child stays alive for five seconds. It promotes a retry (reconnecting, or failed with `inFlight`) on the same catalog request or once the child outlives a 15-second retry grace, longer than the ten seconds ssh's `ConnectTimeout` allows for the TCP connect, handshake and key exchange, because an idle Child may send no catalog request at all. Liveness is a heuristic here, as it is for a first attempt: authentication and the remote forward request come after `ConnectTimeout`, so one that stalls past the grace reads connected until ssh exits, and the exit returns the link to the retry path. Promotion is checked before the outage limit, so a child that has outlived its grace by the time a tick runs, at the limit or late after a sleep, is promoted. At the outage limit it kills the child, except a retry still inside its grace, which carries on as the failed state's retry. A reload spawns a tunnel only for a fresh link or a due attempt, so it neither skips the backoff nor starts a second child while an exited one is still reporting. `reconnect(linkId)` is the deliberate exception: it queues with reload, link stop and shutdown, uses the updated record when a preceding reload finishes, stops that link's child, forgets its backoff or failed state and spawns a fresh attempt. After the child exits it rechecks the persisted record, listener port, generation and reload revision; removal or a changed tunnel cancels that restart. Automatic ticks wait until lifecycle work drains. Overlapping reconnects for one link share one restart, and every spawn bumps a per-link generation, so an exit still being classified or a stop still waiting for its child never overwrites the state of a newer child. `POST /api/link/{id}/reconnect` (the dashboard's **Retry** on a failed or reconnecting Home link, and `ocx link reconnect --link-id `) requires the link listener to be listening (`503 listener_unavailable` otherwise, as for apply), then calls it after `ensureStarted`, answers `{ linkId }`, and refuses a Child-initiated link with `409 link_not_home_initiated` because the Child owns that tunnel. DELETE holds the link against reconnects and automatic spawns throughout remote disconnect, key revocation and store removal. Successful removal reloads the supervisor before releasing the hold; a remote disconnect failure releases it before reloading to restore the retained tunnel. -`src/link/routes.ts` holds the one route and method table for linked traffic; the hub-link listener and the client relay both decide admission from it. +`src/link/routes.ts` holds the one route and method table for linked data-plane traffic; the hub-link listener and the client relay both decide admission from it. The read-only management table beside it is described in [Child management relay and Home Codex settings](#child-management-relay-and-home-codex-settings). `src/link/store.ts` persists link records in `/link/links.json` with private permissions. Records hold aliases, ports, confirmed host-key fingerprints and data-key ids, never keys. The file is a trust boundary: unknown fields, malformed values and duplicate ids are errors, and `hasLinks` reports false for a damaged file. A null host-key fingerprint is accepted only for a client-initiated link, because the hub never opens SSH to that client. Link create/delete, compensation, and listener-port persistence re-read and mutate the store under `withConfigMutationLockSync`. A bind revalidates both the stored port and the presence of a recorded link before activation; deleting the last link during bind closes that listener without publishing a port. @@ -68,7 +68,7 @@ Codex keeps the standalone loopback routing: `routingTarget` in `src/client/conn `src/client/link-ingress.ts` is the link-mode data plane of the machine listener. A `/v1/responses` WebSocket upgrade answers `426 upgrade_required`, which codex-rs maps to its HTTP fallback, and no upgrade is ever relayed. Every relayed route first passes the standalone loopback Host and Origin gate (`isAllowedRequestOrigin` in `src/server/auth-cors.ts`), so a rebinding or cross-site page gets `403 origin_rejected` and nothing is fetched upstream. `/readyz` is answered locally. The link key is read from the service token file once, when the listener starts, and held in memory; the file must hold the key whose fingerprint the connection committed. While it does not, relayed routes answer `503 link_credential_unavailable` without an upstream fetch, and the file is read again at most once a second; a valid key is never re-read. The key is never logged or returned. -`src/client/link-relay.ts` forwards exactly the `linkRouteAllowed` routes from `src/link/routes.ts` through the tunnel. It drops the caller's `Authorization`, Azure `api-key`, Anthropic-compatible `x-api-key`, Google `x-goog-api-key`, `x-opencodex-api-key`, `chatgpt-account-id` and `cookie` and sends the link key as `Authorization: Bearer`, the wire an `env_key` config sent; `GET /v1/usage` takes it as `x-opencodex-api-key`, the only header that route admits. The explicit provider forms matter because the Child and Home are separate credential owners: caller provider credentials never cross the tunnel merely because they are not Bearer tokens. The Home admits the link key and serves the Child with its own accounts. The request body is streamed chunk by chunk with the caller's `Content-Length` and a byte-counting cap at the inbound limit (`resolveInboundBodyLimitBytes`, 256 MiB by default); a larger declared or streamed body answers 413. A lone `Transfer-Encoding: chunked` without `Content-Length` is admitted as a standalone admits it, because the listener has already de-chunked the body; any other Transfer-Encoding, or one next to a `Content-Length`, answers 400. The Home's response headers may take up to 300 seconds, and a caller abort ends the wait sooner. SSE passes through chunk by chunk with caller-abort propagation and a 300-second idle limit, other response bodies stream under the same byte cap, and the relay answers 503 with Retry-After while the tunnel is down. The client supervisor is the relay's tunnel gate (`LinkTunnelGate`): only while the tunnel is connecting or reconnecting (including the start of the client runtime) does a relayed request wait, for at most 15 seconds (`LINK_RELAY_HOLD_MS`) from its first wait and with at most 64 requests waiting, before it is forwarded once; a connected tunnel costs one `pending()` call per request, and a failed one answers 503 at once. A forward whose connection was refused sent nothing, so while the tunnel reconnects it may wait again and be sent again inside the same 15 seconds, provided the streamed body was never read or cancelled; any other failure (a reset, a timeout, a failure after the body started) is never replayed. Both the Child's machine listener and the Home's hub-link listener bind with `idleTimeout: 255`, the public listener's limit, so a held or slow turn is not cut by Bun's 10-second default. Like a standalone data route, a relayed request then lifts its own idle timer (`server.timeout(req, 0)` in `src/client/link-ingress.ts`), so a quiet stretch longer than 255 seconds inside a long generation is not cut either; the relay's header deadline, SSE idle limit and caller abort bound the wait instead. Hub transport keeps the 4 MiB management-relay listener bound and its default idle limit. Link mode waits for the configured port without signalling its holder, then binds there or fails; `src/client/runtime.ts` passes the cached link key, tunnel status and tunnel gate through `bindClientListener` to every bind attempt. Link mode turns the management relay off and refuses key rotation and revocation, which belong to the hub. +`src/client/link-relay.ts` forwards exactly the `linkRouteAllowed` routes from `src/link/routes.ts` through the tunnel. It drops the caller's `Authorization`, Azure `api-key`, Anthropic-compatible `x-api-key`, Google `x-goog-api-key`, `x-opencodex-api-key`, `chatgpt-account-id` and `cookie` and sends the link key as `Authorization: Bearer`, the wire an `env_key` config sent; `GET /v1/usage` takes it as `x-opencodex-api-key`, the only header that route admits. The explicit provider forms matter because the Child and Home are separate credential owners: caller provider credentials never cross the tunnel merely because they are not Bearer tokens. The Home admits the link key and serves the Child with its own accounts. The request body is streamed chunk by chunk with the caller's `Content-Length` and a byte-counting cap at the inbound limit (`resolveInboundBodyLimitBytes`, 256 MiB by default); a larger declared or streamed body answers 413. A lone `Transfer-Encoding: chunked` without `Content-Length` is admitted as a standalone admits it, because the listener has already de-chunked the body; any other Transfer-Encoding, or one next to a `Content-Length`, answers 400. The Home's response headers may take up to 300 seconds, and a caller abort ends the wait sooner. SSE passes through chunk by chunk with caller-abort propagation and a 300-second idle limit, other response bodies stream under the same byte cap, and the relay answers 503 with Retry-After while the tunnel is down. The client supervisor is the relay's tunnel gate (`LinkTunnelGate`): only while the tunnel is connecting or reconnecting (including the start of the client runtime) does a relayed request wait, for at most 15 seconds (`LINK_RELAY_HOLD_MS`) from its first wait and with at most 64 requests waiting, before it is forwarded once; a connected tunnel costs one `pending()` call per request, and a failed one answers 503 at once. A forward whose connection was refused sent nothing, so while the tunnel reconnects it may wait again and be sent again inside the same 15 seconds, provided the streamed body was never read or cancelled; any other failure (a reset, a timeout, a failure after the body started) is never replayed. Both the Child's machine listener and the Home's hub-link listener bind with `idleTimeout: 255`, the public listener's limit, so a held or slow turn is not cut by Bun's 10-second default. Like a standalone data route, a relayed request then lifts its own idle timer (`server.timeout(req, 0)` in `src/client/link-ingress.ts`), so a quiet stretch longer than 255 seconds inside a long generation is not cut either; the relay's header deadline, SSE idle limit and caller abort bound the wait instead. Hub transport keeps the 4 MiB management-relay listener bound and its default idle limit. Link mode waits for the configured port without signalling its holder, then binds there or fails; `src/client/runtime.ts` passes the cached link key, tunnel status and tunnel gate through `bindClientListener` to every bind attempt. Link mode replaces the hub management relay with the read-only link management relay (see [Child management relay and Home Codex settings](#child-management-relay-and-home-codex-settings)) and refuses key rotation and revocation, which belong to the Home. > Decision record: [ADR-6032](decisions/ADR-6032-link-relay-credential-boundary.md) @@ -107,3 +107,19 @@ Before issuance, apply requires curl and effective standalone Child configuratio `src/cli/link-enrollment.ts` binds probe/confirm/apply to one identity-checked live Home, rejects redirects, and caps the entire flow including response-body reads and interactive confirmation at 360 seconds. Non-TTY add requires an exact SHA256 fingerprint; TTY requires explicit yes. Mismatch stops before host trust or key issuance. Strict response parsing and sanitized errors never print a link key; a timeout reports uncertain outcome. Coverage: `tests/server/link-enrollment.test.ts`, `tests/cli/cli-link-enrollment.test.ts`. Client-side status supplies initiation identity from `src/client/link-status.ts`: a matching valid sidecar establishes Child initiation and SSH alias, while missing sidecar plus absence of any Child marker establishes Home initiation. Unreadable/mismatched evidence, including a marker naming a different link, emits no synthetic direction. Link-mode `ocx status` preserves Hub state when Home is a Hub by asking the identity-checked local Child without a credential; the Child's connection-bound relay sends the key. A missing, unsafe, or mismatched CLI token file does not block this credential-free read; the saved connection owner must still match after Child discovery. It never sends that key directly to the saved tunnel URL. A bounded 404 body with `error.code: hub_state_not_a_hub` is role absence, not version skew, and suppresses old cached Hub projection. Other errors retain existing owner-bound cache fallback. Verified Child observations update the matching active link by ID even when the local store contains records; unrelated records and persisted compensation failures retain their own state. Home remains the transport label in either role. + +## Child management relay and Home Codex settings + +A link Child relays every model request to its Home, so everything the Home's proxy applies per request (effort caps, the sub-agent roster and fallback, the injection prompt, sidecar and compaction routing, the catalog) already follows the Home. Two things did not: the Codex settings the injector writes into the Child's own `config.toml`, and every management read. Both now cross the link, and nothing else does. + +`src/link/management-routes.ts` is the shared, default-deny table. A Home management path is readable over the link only when it is listed in `LINK_MANAGEMENT_READ_PATHS` (the reads the dashboard polls except the model list, the header quota strip, and the account, usage, catalog and preset reads the CLI makes; combos and routing profiles stay out until their projections are reviewed for decision-endpoint configuration), the method is GET or HEAD, and the query carries no `refresh`, `reload`, `force` or `quota` key, which would make the Home refresh credentials or probe upstream quota. `/api/models`, `/api/model-presets`, `/api/subagent-models` and `/api/subagent-model-fallback` are absent because they run live provider discovery and can persist an initial model selection; the Child's Codex picker already shows the Home's catalog, and the preset preview only decides a write a Child cannot make. `/api/injection-model` stays readable because the Child's dashboard and `ocx effort` show the Home's delegation model and effort from it, Every link read is a passive read (`runPassiveRead` in `src/lib/passive-read.ts`, entered by `src/server/index/link-management.ts`): it shows what the Home already knows and does not make the Home act on providers, accounts, processes or files. Inside it `fetchAllModels` and the management model rows return no routed models, so `/api/injection-model` and `/api/sidecar-settings` list native options plus the persisted picks; `fetchProviderQuotaReports` returns the reports the Home holds, without a probe, token refresh, account snapshot or publication into routing evidence; the Codex account list uses the cached main-account info and pool quota, without a token refresh, WHAM probe or native-main claim; the startup-health diagnostic, the Codex runtime version probes and the bundled-catalog refresh (which runs the Codex binary and persists the runtime) are not started, and a native-main fence raised for unknown service ownership is reported as it stands instead of re-inspected (a successful re-inspection activates the native-main lifecycle and its recovery); `hardenConfigDir`, `hardenExistingSecret` and `backupInvalidConfig` do nothing, so reading a credential store neither tightens its permissions nor backs up a malformed copy; and memory-budget enforcement triggered by a read (the usage caches publish and enforce) demotes only stores registered as `passiveEvictable` in `src/lib/app-owned-memory-stores.ts`, whose eviction only releases memory (today the two usage caches), because demoting other stores can spill a continuation to disk or rewrite a replay snapshot; the rest of the budget is enforced by the Home's own next enforcement. The Home's own dashboard and traffic keep those caches current. `tests/server/link-management-relay.test.ts` reads every allowlisted path, GET and HEAD, against a Home with a pending provider, a quota-probed provider, a canonical OpenAI provider, a pool account with an expired token, a malformed OAuth store and a native-main ownership fence. Inside a passive read it records calls to `fetch`; `child_process` `execFile`, `execFileSync`, `spawn`, `spawnSync`, `exec` and `execSync`; `Bun.spawn`, `Bun.spawnSync` and `Bun.write`; the `node:fs` writers `writeFileSync`, `appendFileSync`, `copyFileSync`, `renameSync`, `chmodSync`, `mkdirSync`, `rmSync`, `unlinkSync`, `writeSync`, `ftruncateSync`, `truncateSync`, `linkSync`, `symlinkSync`, `utimesSync` and `openSync` for writing; and the matching `node:fs/promises` writers, and asserts there are none. Writes through other APIs (callback `fs`, file handles, Bun file writers, SQLite) are caught only as file changes: the test compares the size, mtime and mode of every file under the Home's config and Codex homes before and after the reads, along with the config file bytes and the live config, and checks that the ownership fence was not re-probed. A separate case reads `/api/usage` against a ledger row, a persisted Antigravity replay session, an undeclared cache and a continuation store far over the memory budget, waits past the replay snapshot debounce, and checks that enforcement ran without demoting any of them, while the Home's own enforcement demotes all three. The same reads made by the Home then back up the store, probe the quota and the account, re-probe ownership and decide the pending provider, which proves the fixture is live. Work a read schedules for later is still attributed to the read when it runs; the test waits past the responses snapshot debounce (2 s) and checks again after the Home reads, but it cannot observe work delayed beyond that. `/api/logs` and `/api/diagnostics/project-config` are deliberately absent: the first carries request detail, the second describes files on the Home. The Home's hub-link listener admits `/v1/link-management/` and `/v1/link-management/codex-settings` (`linkManagementRouteAllowed`) in addition to the data-plane table; the Child's own data-plane ingress keeps the data-plane table, so a local caller on the Child cannot reach these reads without the Child's dashboard session. + +On the Home, `src/server/index/link-management.ts` runs after the link-key admission in `src/server/index/serve-options.ts`. It builds a fresh loopback request with no caller header, cookie, Origin or credential and no principal, so no consent- or capability-gated branch is reachable, calls the management handler, and returns only content headers with `Cache-Control: no-store`. The relayed projections are the dashboard's own: provider and key-pool rows report presence and masked identities, never a credential value. + +On the Child, the dashboard's shared plane is the listener's `/api/machine/hub-relay` base. `/api/machine/status` advertises it with `managementTransport: "relay"` and a `link` block (`linkId`, the Home's name, `readOnly: true`), and the shared plane bootstraps `/api/machine/hub-relay/opencodex-session`, which the listener answers with its own session, so the browser never holds a Home credential. `src/client/link-management-relay.ts` requires a Child dashboard session, refuses anything outside the table with `403 link_child_read_only`, and sends the rest through `relayLinkDataRequest` with the management route table, so the link key is attached only after the connection-bound proof described in [Connection-bound relay authentication](#connection-bound-relay-authentication). The dashboard shows "Managed by Home over Remote Link", and its header quota strip reads the Home's quota. + +`src/cli/link-child.ts` applies the same table to the CLI: `runtimeRequest`, the account client (`LINK_HOME_BASE` in `src/cli/account-api.ts`) and the observation commands either read the Home through `fetchLinkHomeManagement` in `src/client/link-home.ts` or refuse with `linkChildManagementRefusal`, the one message that names the Home and the link and states that no request was sent. `ocx usage` reads `/v1/usage` through `linkBoundFetch`; `ocx status` reads `/v1/hub-state` through the local Child listener, names the Home and the link, and lists the Codex settings the last sync applied; `ocx provider list|show` and `ocx config get` label their output as local and inactive, `ocx config set|unset|import` write locally with the same note, and `ocx provider add|remove|set-default` are refused because they could only change inactive providers. Account commands classify providers from the Home's `/api/providers` projection (`accountClassificationConfig`). + +`src/link/codex-settings.ts` owns the Codex settings document. It carries exactly the OpenCodex config fields the injection plan reads that are policy rather than listener facts: `fastMode`, `multiAgentMode`, `webSearchSidecar.enabled`, and the effective managed sub-agent defaults (`syncCodexSubagentDefaults` with `injectionModel` and `injectionEffort`). Root `model_reasoning_effort` and the `desktop.*` picker keys are Codex app settings OpenCodex writes on no machine, so they are not part of it. Port, hostname, the Desktop authless and client compaction routing forms, and resume history stay the Child's because they describe how the Child's Codex reaches the Child. The document rides on its own read rather than in the catalog because the catalog bytes are written verbatim to `model_catalog_json`. The parse is strict. `syncConnectedClient` and link-mode connect in `src/client/connect.ts` overlay it on the Child's config before injection. A fresh read is recorded in `/link/home-codex-settings.json` (0600, keyed by link id, removed on disconnect) only after the injection that applies it succeeds, because `ocx status` reads the record as applied; connect with `--no-sync` records nothing. A failed read reuses that record, and a Home without the route leaves the Child's local values and says so. `ocx sync` and `ocx status` list the applied values; the record's host name is also how a Home-initiated Child, which has no sidecar alias, names its Home. + +Coverage: `tests/server/link-management-relay.test.ts` (the real hub-link listener: admission, refusals, masked projections, the settings document), `tests/clients/client-machine-listener.test.ts` (the Child relay and session bootstrap), `tests/clients/link-home-settings.test.ts` (parse, overlay, injection and record fallback) and `tests/cli/cli-link-child.test.ts` (the CLI reads, refusals and Home wording). diff --git a/tests/cli/cli-link-child.test.ts b/tests/cli/cli-link-child.test.ts new file mode 100644 index 00000000000..08646ef0e60 --- /dev/null +++ b/tests/cli/cli-link-child.test.ts @@ -0,0 +1,185 @@ +/** + * #6957: management commands on a Remote Link Child either read the Home over the link or are + * refused with one message that names the Home and the link. Nothing is sent for a refusal, and + * no refusal points at a relay transport a link Child never has. + */ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { RuntimeApiError, runtimeRequest } from "../../src/cli/runtime-api"; +import { accountClassificationConfig, apiJson, classifyAccount, LINK_HOME_BASE } from "../../src/cli/account-api"; +import { linkChildLocalNotice, linkChildProviderChangeRefusal, readLinkChildContext } from "../../src/cli/link-child"; +import type { OcxConfig } from "../../src/types"; +import { linkCodexSettingsLines, remoteHubBannerLine, type CliRemoteHubStatus } from "../../src/cli/status"; +import { hubStateFailureReason } from "../../src/client/hub-state"; +import { fetchHubUsage, HubClientError } from "../../src/client/hub-client"; +import { serviceApiTokenFingerprint } from "../../src/lib/service-secrets"; +import type { LiveProxy } from "../../src/server/proxy-liveness"; +import { removeTreeWithRetry } from "../helpers/remove-tree"; + +const KEY = `ocx_data_${"f".repeat(40)}`; +const LINK_ID = `lnk_${"a".repeat(16)}`; +const TUNNEL = 23458; +const clientLive = async () => ({ pid: process.pid, port: 10100, hostname: "127.0.0.1", source: "config", role: "client" }) as LiveProxy; + +let root = ""; +let previousHome: string | undefined; + +function writeChild(transport: "link" | "hub" = "link"): void { + writeFileSync(join(root, "config.json"), JSON.stringify({ + port: 10100, + providers: {}, + defaultProvider: "openai", + runtimeRole: "client", + client: { + serverUrl: transport === "link" ? `http://127.0.0.1:${TUNNEL}` : "https://hub.example.test", + managementUrl: transport === "link" ? `http://127.0.0.1:${TUNNEL}` : "https://hub.example.test", + managementTransport: "direct", + ...(transport === "link" ? { transport: "link", link: { tunnelPort: TUNNEL, linkId: LINK_ID } } : {}), + selectedClients: ["codex"], + tokenEnv: "OPENCODEX_API_AUTH_TOKEN", + apiKeyId: "link-key-1", + tokenFingerprint: serviceApiTokenFingerprint(KEY), + protocolVersion: 1, + connectedAt: "2026-10-01T00:00:00.000Z", + }, + })); + writeFileSync(join(root, "service-api-token"), `${KEY}\n`, { mode: 0o600 }); +} + +function homeServing(body: unknown, seen: Request[], status = 200): typeof fetch { + return (async (input: RequestInfo | URL, init?: RequestInit) => { + seen.push(new Request(String(input), init)); + return Response.json(body, { status }); + }) as typeof fetch; +} + +beforeEach(() => { + previousHome = process.env.OPENCODEX_HOME; + root = mkdtempSync(join(tmpdir(), "ocx-cli-link-child-")); + process.env.OPENCODEX_HOME = root; +}); + +afterEach(() => { + if (previousHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousHome; + if (root) removeTreeWithRetry(root); +}); + +describe("management CLI on a Remote Link Child", () => { + test("an allowlisted read is answered by the Home through the link", async () => { + writeChild(); + const seen: Request[] = []; + const body = await runtimeRequest("/api/providers", {}, { + findLiveProxy: clientLive, + linkHome: { fetchImpl: homeServing([{ name: "home-provider" }], seen) }, + }); + expect(body).toEqual([{ name: "home-provider" }]); + expect(seen).toHaveLength(1); + expect(seen[0]!.url).toBe(`http://127.0.0.1:${TUNNEL}/v1/link-management/api/providers`); + expect(seen[0]!.headers.get("authorization")).toBe(`Bearer ${KEY}`); + }); + + test("a change or an unlisted read is refused with the one Child message and nothing is sent", async () => { + writeChild(); + const seen: Request[] = []; + for (const [method, path] of [["PUT", "/api/settings"], ["GET", "/api/logs"], ["GET", "/api/provider-quotas?refresh=1"]] as const) { + const error = await runtimeRequest(path, { method }, { + findLiveProxy: clientLive, linkHome: { fetchImpl: homeServing({}, seen) }, + }).catch(value => value) as RuntimeApiError; + expect(error).toBeInstanceOf(RuntimeApiError); + expect(error.code).toBe("link_child_management_unavailable"); + expect(error.message).toContain("Remote Link Child managed by its Home"); + expect(error.message).toContain(LINK_ID); + expect(error.message).toContain(`${method} ${path.split("?")[0]}`); + expect(error.message).not.toContain("relay transport"); + } + expect(seen).toHaveLength(0); + }); + + test("a hub client keeps its own refusal", async () => { + writeChild("hub"); + expect(readLinkChildContext()).toBeNull(); + const error = await runtimeRequest("/api/settings", { method: "PUT" }, { findLiveProxy: clientLive }).catch(value => value) as RuntimeApiError; + expect(error.code).toBe("client_role_management_unavailable"); + }); + + test("account reads come from the Home and account changes are refused", async () => { + writeChild(); + const seen: Request[] = []; + const read = await apiJson({ fetchImpl: homeServing({ activeCodexAccountId: "acct-1" }, seen) }, LINK_HOME_BASE, "GET", "/api/codex-auth/active"); + expect(read).toEqual({ status: 200, json: { activeCodexAccountId: "acct-1" } }); + expect(seen[0]!.url).toBe(`http://127.0.0.1:${TUNNEL}/v1/link-management/api/codex-auth/active`); + const write = await apiJson({ fetchImpl: homeServing({}, seen) }, LINK_HOME_BASE, "PUT", "/api/codex-auth/active"); + expect(write.status).toBe(403); + expect(String(write.json.error)).toContain("PUT /api/codex-auth/active runs on the Home"); + expect(seen).toHaveLength(1); + }); + + test("an older Home without the relay is named as the thing to upgrade", async () => { + writeChild(); + const seen: Request[] = []; + const error = await runtimeRequest("/api/settings", {}, { + findLiveProxy: clientLive, + linkHome: { fetchImpl: homeServing({ error: { type: "invalid_request_error", code: "not_found", message: "Unknown endpoint" } }, seen, 404) }, + }).catch(value => value) as RuntimeApiError; + expect(error.status).toBe(503); + expect(error.message).toContain("upgrade the Home"); + }); + + test("local reads are labelled as local and inactive", () => { + writeChild(); + expect(linkChildLocalNotice("The providers")).toContain("inactive while it is linked"); + expect(linkChildProviderChangeRefusal("ocx provider add")).toContain("ocx provider add runs on the Home"); + writeChild("hub"); + expect(linkChildLocalNotice("The providers")).toBeNull(); + expect(linkChildProviderChangeRefusal("ocx provider add")).toBeNull(); + }); + + test("account commands classify providers from the Home, without any credential value", async () => { + writeChild(); + const seen: Request[] = []; + const local = { port: 10100, providers: {} } as unknown as OcxConfig; + const classified = await accountClassificationConfig({ fetchImpl: homeServing([ + { name: "home-keyed", adapter: "openai-chat", baseUrl: "https://example.test/v1", hasApiKey: true, authMode: "key" }, + ], seen) }, local); + if ("error" in classified) throw new Error(classified.error); + const config = classified.config; + expect(seen[0]!.url).toBe(`http://127.0.0.1:${TUNNEL}/v1/link-management/api/providers`); + expect(classifyAccount(config, "home-keyed")).toEqual({ type: "api-key" }); + expect(classifyAccount(local, "home-keyed")).toEqual({ error: 'unknown provider "home-keyed"' }); + // A Home that cannot answer is an error, never a fall back to the inactive local providers. + const failed = await accountClassificationConfig({ fetchImpl: homeServing({}, seen, 500) }, local); + expect(failed).toEqual({ error: `Could not read the provider list from its Home over Remote Link (${LINK_ID}): the Home answered HTTP 500.` }); + const explained = await accountClassificationConfig({ fetchImpl: homeServing({ error: "provider store busy" }, seen, 503) }, local); + expect(explained).toEqual({ error: `Could not read the provider list from its Home over Remote Link (${LINK_ID}): provider store busy.` }); + const older = await accountClassificationConfig({ fetchImpl: homeServing({ error: { code: "not_found" } }, seen, 404) }, local); + expect("error" in older && older.error).toContain("upgrade the Home"); + writeChild("hub"); + expect(await accountClassificationConfig({}, local)).toEqual({ config: local }); + }); + + test("usage and status name the Home instead of a hub", async () => { + const usage = await fetchHubUsage(`http://127.0.0.1:${TUNNEL}`, KEY, new URLSearchParams(), { + fetchImpl: (async () => new Response(null, { status: 404 })) as unknown as typeof fetch, peer: "Home", + }).catch(value => value) as HubClientError; + expect(usage.message).toBe("Home does not support client usage; upgrade the Home"); + expect(hubStateFailureReason(new HubClientError("hub_state_unsupported", "x", 404), "Home")) + .toBe("this Home is too old to report its state; upgrade the Home"); + expect(hubStateFailureReason(new HubClientError("hub_state_unsupported", "x", 404))) + .toBe("this hub is too old to report its state; upgrade the hub"); + + const status: CliRemoteHubStatus = { + connected: true, origin: `http://127.0.0.1:${TUNNEL}`, stateSource: "hub", hubVersion: "2.84.0", + providers: [], oauth: [], subagentModels: [], truncated: false, claudeCodeEnabled: null, + link: { linkId: LINK_ID, homePhrase: "Home home-box", codexSettings: { source: "home", fetchedAt: "2026-10-11T00:00:00.000Z", lines: ["fast mode: on"] } }, + }; + expect(remoteHubBannerLine(status)).toContain(`Home home-box over Remote Link (${LINK_ID})`); + expect(remoteHubBannerLine(status)).not.toMatch(/\bhub\b/); + expect(linkCodexSettingsLines(status.link!)).toEqual([ + "Codex settings from Home home-box (applied 2026-10-11T00:00:00.000Z):", + " fast mode: on", + ]); + }); +}); diff --git a/tests/clients/client-link-relay.test.ts b/tests/clients/client-link-relay.test.ts index 6196f8b1ad4..a8bc3193262 100644 --- a/tests/clients/client-link-relay.test.ts +++ b/tests/clients/client-link-relay.test.ts @@ -445,7 +445,8 @@ describe("client link HTTP relay", () => { contentLength: String(body.length), transferEncoding: null, body, }); expect((await fetch(new URL("/v1/unknown", machine.url))).status).toBe(404); - expect((await fetch(new URL("/api/machine/hub-relay/api/config", machine.url))).status).toBe(404); + // #6957: the management relay exists on a link Child, but only for a Child dashboard session. + expect((await fetch(new URL("/api/machine/hub-relay/api/config", machine.url))).status).toBe(401); // A chunked upload (a streamed body with no Content-Length) passes through as on a standalone. const encoder = new TextEncoder(); diff --git a/tests/clients/client-machine-listener.test.ts b/tests/clients/client-machine-listener.test.ts index 0efcecd5476..02d3a83affe 100644 --- a/tests/clients/client-machine-listener.test.ts +++ b/tests/clients/client-machine-listener.test.ts @@ -463,12 +463,79 @@ describe("client machine listener in link mode", () => { expect((await fetch(url, { method: "POST", headers: await guiHeaders(server, true), body: "{}" })).status).toBe(404); const machine = await fetch(new URL("/api/machine/status", server.url), { headers }); - const body = await machine.json() as { machineBase: string; sharedBase: string; sharedServerOrigin: string }; - expect(body.sharedBase).toBe(body.machineBase); + const body = await machine.json() as { + machineBase: string; sharedBase: string; sharedServerOrigin: string; managementTransport: string; link?: unknown; + }; + // #6957: the dashboard's shared plane is this listener's relay base, read-only, to the Home. + expect(body.sharedBase).toBe(`${body.machineBase}/api/machine/hub-relay`); + expect(body.managementTransport).toBe("relay"); + expect(body.link).toEqual({ linkId: `lnk_${"b".repeat(16)}`, home: "home-mac", readOnly: true }); expect(body.sharedServerOrigin).toBe(body.machineBase); expect(body.machineBase).toBe(`http://127.0.0.1:${server.port}`); }); + test("relays an allowlisted dashboard read to the Home with the link key and no browser credential", async () => { + writeKey(); + const { server, upstream } = linkListener({ reply: () => Response.json([{ name: "home-provider" }]) }); + const url = new URL("/api/machine/hub-relay/api/providers", server.url); + expect((await fetch(url)).status).toBe(401); + expect(upstream).toHaveLength(0); + + const headers = await guiHeaders(server); + headers.set("Cookie", "browser=1"); + const relayed = await fetch(url, { headers }); + expect(relayed.status).toBe(200); + expect(await relayed.json()).toEqual([{ name: "home-provider" }]); + expect(upstream).toHaveLength(1); + const sent = upstream[0]!; + expect(sent.url).toBe(`http://127.0.0.1:${TUNNEL_PORT}/v1/link-management/api/providers`); + expect(sent.method).toBe("GET"); + expect(sent.headers.get("authorization")).toBe(`Bearer ${LINK_KEY}`); + for (const name of ["x-opencodex-api-key", "x-opencodex-gui-origin", "cookie", "origin"]) { + expect({ name, value: sent.headers.get(name) }).toEqual({ name, value: null }); + } + }); + + test("refuses every change and every unlisted read with the Child refusal, before any request", async () => { + writeKey(); + const { server, upstream } = linkListener(); + const cases: Array<[string, string]> = [ + ["PUT", "/api/machine/hub-relay/api/settings"], + ["POST", "/api/machine/hub-relay/api/providers"], + ["GET", "/api/machine/hub-relay/api/logs"], + ["GET", "/api/machine/hub-relay/api/provider-quotas?refresh=1"], + ["GET", "/api/machine/hub-relay/api/oauth/accounts?provider=anthropic"a=1"], + ["GET", "/api/machine/hub-relay/api/models"], + ["GET", "/api/machine/hub-relay/api/diagnostics/project-config"], + ]; + for (const [method, path] of cases) { + const response = await fetch(new URL(path, server.url), { + method, headers: await guiHeaders(server, method !== "GET"), ...(method === "GET" ? {} : { body: "{}" }), + }); + expect({ method, path, status: response.status }).toEqual({ method, path, status: 403 }); + const body = await response.json() as { error: string; code: string }; + expect(body.code).toBe("link_child_read_only"); + expect(body.error).toContain("Remote Link Child managed by Home home-mac"); + expect(body.error).toContain(`lnk_${"b".repeat(16)}`); + expect(body.error).toContain("No request was sent."); + expect(body.error).not.toContain("relay transport"); + } + expect(upstream).toHaveLength(0); + }); + + test("bootstraps the shared plane with this listener's own session and never relays the data-plane prefix", async () => { + writeKey(); + const { server, upstream } = linkListener(); + const bootstrap = await fetch(new URL("/api/machine/hub-relay/opencodex-session", server.url)); + expect(bootstrap.status).toBe(200); + expect(meta(await bootstrap.text(), "opencodex-session-token")).toStartWith("ocx_session_"); + // An unauthenticated local caller cannot reach the Home's management reads through the + // Child's data-plane ingress: that path relays only the data-plane table. + const direct = await fetch(new URL("/v1/link-management/api/providers", server.url)); + expect(direct.status).toBe(404); + expect(upstream).toHaveLength(0); + }); + test("a hub-transport client has no link status route", async () => { const server = startMachineListener(0, { state: connection(), managementAuthState: authState() }); servers.push(server); diff --git a/tests/clients/link-home-settings.test.ts b/tests/clients/link-home-settings.test.ts new file mode 100644 index 00000000000..9f17204c900 --- /dev/null +++ b/tests/clients/link-home-settings.test.ts @@ -0,0 +1,195 @@ +/** + * #6957: the Home's OpenCodex-owned Codex settings on a Remote Link Child. + * + * The projection, the strict parse, the overlay the Codex injector sees, and how a sync picks + * between a fresh Home read, the last recorded read and the Child's local values. + */ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { mkdtempSync, mkdirSync, statSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + describeHomeCodexSettings, + overlayHomeCodexSettings, + parseHomeCodexSettingsDocument, + projectHomeCodexSettings, + type HomeCodexSettingsDocument, +} from "../../src/link/codex-settings"; +import { + fetchLinkHomeManagement, + homeCodexSettingsPath, + linkHomeName, + readHomeCodexSettingsRecord, + recordHomeCodexSettings, + resolveHomeCodexSettings, +} from "../../src/client/link-home"; +import { deriveCodexInjectionPlan } from "../../src/codex/inject/plan"; +import { validateCodexRoutingTarget } from "../../src/codex/inject/routing-target"; +import type { OcxClientConnectionConfig, OcxConfig } from "../../src/types"; +import { installIsolatedCodexHome, type IsolatedCodexHome } from "../helpers/isolated-codex-home"; +import { removeTreeWithRetry } from "../helpers/remove-tree"; + +const LINK_ID = `lnk_${"c".repeat(16)}`; +const KEY = `ocx_data_${"d".repeat(40)}`; +const connection = { + transport: "link", + link: { tunnelPort: 23457, linkId: LINK_ID }, + apiKeyId: "link-key-1", +} as OcxClientConnectionConfig; + +const document: HomeCodexSettingsDocument = { + version: 1, + home: { name: "home-box", version: "2.84.0" }, + settings: { + fastMode: true, + multiAgentMode: "v1", + webSearchSidecarEnabled: false, + subagentDefaults: { model: "anthropic/claude-sonnet-5-5", reasoningEffort: "high" }, + }, +}; + +let root = ""; +let previousHome: string | undefined; +let codexHome: IsolatedCodexHome | null = null; + +beforeEach(() => { + previousHome = process.env.OPENCODEX_HOME; + root = mkdtempSync(join(tmpdir(), "ocx-link-home-")); + process.env.OPENCODEX_HOME = root; + codexHome = installIsolatedCodexHome("ocx-link-home-codex-"); +}); + +afterEach(() => { + codexHome?.restore(); + codexHome = null; + if (previousHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousHome; + if (root) removeTreeWithRetry(root); +}); + +function serving(body: unknown, status = 200, seen: Request[] = []): typeof fetch { + return (async (input: RequestInfo | URL, init?: RequestInit) => { + seen.push(new Request(String(input), init)); + return Response.json(body, { status }); + }) as typeof fetch; +} + +async function applied(): Promise { + recordHomeCodexSettings(LINK_ID, await resolveHomeCodexSettings(connection, KEY, { fetchImpl: serving(document) })); +} + +describe("Home Codex settings document", () => { + test("projects only what the Child's injector writes, and only an effective sub-agent sync", () => { + const projected = projectHomeCodexSettings({ port: 1, providers: {}, injectionModel: "x/y", syncCodexSubagentDefaults: false } as unknown as OcxConfig, "2.84.0"); + expect(projected.settings).toEqual({ fastMode: null, multiAgentMode: null, webSearchSidecarEnabled: true, subagentDefaults: null }); + expect(parseHomeCodexSettingsDocument(projected)).toEqual(projected); + }); + + test("parses strictly: an extra field, an unsafe label or an unknown mode rejects the document", () => { + expect(parseHomeCodexSettingsDocument(document)).toEqual(document); + expect(parseHomeCodexSettingsDocument({ ...document, extra: 1 })).toBeNull(); + expect(parseHomeCodexSettingsDocument({ ...document, home: { ...document.home, name: "mac\u202Eduts" } })).toBeNull(); + expect(parseHomeCodexSettingsDocument({ ...document, settings: { ...document.settings, multiAgentMode: "v3" } })).toBeNull(); + expect(parseHomeCodexSettingsDocument({ ...document, settings: { ...document.settings, model_reasoning_effort: "high" } })).toBeNull(); + expect(parseHomeCodexSettingsDocument({ + ...document, settings: { ...document.settings, subagentDefaults: { model: "m", reasoningEffort: "high\nx" } }, + })).toBeNull(); + }); + + test("the overlay keeps the Child's listener facts and takes the Home's policy", () => { + const local = { + port: 10100, hostname: "127.0.0.1", providers: {}, fastMode: false, multiAgentMode: "v2", + webSearchSidecar: { enabled: true, model: "local-model" }, injectionModel: "local/model", syncCodexSubagentDefaults: false, + } as unknown as OcxConfig; + const applied = overlayHomeCodexSettings(local, document.settings); + expect(applied.port).toBe(10100); + expect(applied.hostname).toBe("127.0.0.1"); + expect(applied.fastMode).toBe(true); + expect(applied.multiAgentMode).toBe("v1"); + expect(applied.webSearchSidecar).toEqual({ enabled: false, model: "local-model" }); + expect(applied.injectionModel).toBe("anthropic/claude-sonnet-5-5"); + expect(applied.injectionEffort).toBe("high"); + expect(applied.syncCodexSubagentDefaults).toBe(true); + const unset = overlayHomeCodexSettings(local, { ...document.settings, fastMode: null, multiAgentMode: null, subagentDefaults: null }); + expect("fastMode" in unset).toBe(false); + expect("multiAgentMode" in unset).toBe(false); + expect(unset.syncCodexSubagentDefaults).toBe(false); + expect(local.fastMode).toBe(false); + }); + + test("the injection plan writes the Home's values into the Child's Codex config", () => { + const local = { port: 10100, providers: {}, fastMode: false, webSearchSidecar: { enabled: true } } as unknown as OcxConfig; + const plan = deriveCodexInjectionPlan("", { + config: overlayHomeCodexSettings(local, document.settings), + routingTarget: validateCodexRoutingTarget({ baseUrl: "http://127.0.0.1:10100/v1", requiresAdmissionToken: false, tokenEnv: "OPENCODEX_API_AUTH_TOKEN" }), + catalogPathOption: null, + journalReadOnly: true, + }); + expect(plan.kind).toBe("ok"); + const content = plan.kind === "ok" ? plan.content : ""; + expect(content).toContain("fast_mode = true"); + expect(content).toContain("anthropic/claude-sonnet-5-5"); + expect(content).toMatch(/web_search = "disabled"/); + }); + + test("describes every applied value on its own line", () => { + expect(describeHomeCodexSettings(document.settings)).toEqual([ + "fast mode: on", + "multi-agent mode: v1", + "web search sidecar: off", + "native sub-agent defaults: anthropic/claude-sonnet-5-5 (high)", + ]); + }); +}); + +describe("resolving the settings a Child sync applies", () => { + test("a fresh read is sent with the link key and recorded privately only once applied", async () => { + const seen: Request[] = []; + const resolved = await resolveHomeCodexSettings(connection, KEY, { fetchImpl: serving(document, 200, seen), now: () => new Date("2026-10-11T00:00:00.000Z") }); + expect(resolved).toEqual({ source: "home", document, fetchedAt: "2026-10-11T00:00:00.000Z" }); + expect(seen[0]!.url).toBe("http://127.0.0.1:23457/v1/link-management/codex-settings"); + expect(seen[0]!.headers.get("authorization")).toBe(`Bearer ${KEY}`); + // Not yet applied: `ocx status` must not report settings an injection never wrote. + expect(readHomeCodexSettingsRecord(LINK_ID)).toBeNull(); + recordHomeCodexSettings(LINK_ID, resolved); + expect(readHomeCodexSettingsRecord(LINK_ID)?.document).toEqual(document); + expect(readHomeCodexSettingsRecord(`lnk_${"e".repeat(16)}`)).toBeNull(); + if (process.platform !== "win32") expect(statSync(homeCodexSettingsPath()).mode & 0o777).toBe(0o600); + }); + + test("a failed read reuses the last record; a Home without the route keeps local values", async () => { + await applied(); + const failing = (async () => { throw new Error("tunnel down"); }) as unknown as typeof fetch; + const cached = await resolveHomeCodexSettings(connection, KEY, { fetchImpl: failing }); + expect(cached).toMatchObject({ source: "cached", document, reason: "the Home is unreachable" }); + const unsupported = await resolveHomeCodexSettings(connection, KEY, { fetchImpl: serving({ error: "not_found" }, 404) }); + expect(unsupported).toEqual({ source: "local", reason: "the Home does not publish Codex settings; upgrade the Home" }); + expect(readHomeCodexSettingsRecord(LINK_ID)).toBeNull(); + const invalid = await resolveHomeCodexSettings(connection, KEY, { fetchImpl: serving({ ...document, extra: true }) }); + expect(invalid).toEqual({ source: "local", reason: "the Home returned an unreadable settings document" }); + }); + + test("names the Home by the Child-initiated alias first, then by the recorded host name", async () => { + expect(linkHomeName(connection, () => null)).toBeNull(); + await applied(); + expect(linkHomeName(connection, () => null)).toBe("home-box"); + mkdirSync(join(root, "link"), { recursive: true }); + expect(linkHomeName(connection, () => ({ + linkId: LINK_ID, alias: "studio-ssh", hubHostKeyFingerprint: `SHA256:${"a".repeat(43)}`, peerListenerPort: 1, tunnelPort: 23457, + }))).toBe("studio-ssh"); + writeFileSync(homeCodexSettingsPath(), "{not json"); + expect(linkHomeName(connection, () => null)).toBeNull(); + }); + + test("a management read outside the allowlist throws before anything is sent", async () => { + const seen: Request[] = []; + const fetchImpl = serving({}, 200, seen); + await expect(fetchLinkHomeManagement(connection, KEY, "/api/logs", {}, { fetchImpl })).rejects.toThrow("not readable"); + await expect(fetchLinkHomeManagement(connection, KEY, "/api/provider-quotas?refresh=1", {}, { fetchImpl })).rejects.toThrow("not readable"); + await expect(fetchLinkHomeManagement(connection, KEY, "//evil.example/api/providers", {}, { fetchImpl })).rejects.toThrow("not readable"); + expect(seen).toHaveLength(0); + const ok = await fetchLinkHomeManagement(connection, KEY, "/api/providers", {}, { fetchImpl }); + expect(ok.status).toBe(200); + expect(seen[0]!.url).toBe("http://127.0.0.1:23457/v1/link-management/api/providers"); + }); +}); diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index 0da96d02b86..c21843184bb 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -138,6 +138,7 @@ "abort-idle-deadline.test.ts": "lib", "tool-envelope-echo-whole-line.test.ts": "adapters", "abort-race.test.ts": "adapters", "account-import.test.ts": "server", "account-pool-management-api.test.ts": "server", "link-listener-admission.test.ts": "server", + "link-management-relay.test.ts": "server", "link-listener-lifecycle.test.ts": "server", "link-listener-mutation.test.ts": "server", "link-join-route.test.ts": "server", "core-link-boundary.test.ts": "lab", "acl-error-classification.test.ts": "lib", @@ -366,6 +367,7 @@ "client-hub-state.test.ts": "clients", "client-hub-usage.test.ts": "clients", "client-injection-guard.test.ts": "codex-integration", "client-lifecycle-lock.test.ts": "clients", "client-machine-listener.test.ts": "clients", "client-runtime.test.ts": "clients", + "link-home-settings.test.ts": "clients", "cli-link-child.test.ts": "cli", "cline-client.test.ts": "clients", "cline-pass-deepseek-v4-tool-replay.test.ts": "providers", "cline-pass-provider.test.ts": "providers", "cline-pass-reasoning-efforts.test.ts": "providers", "cline-provider.test.ts": "providers", "cline-writer.test.ts": "clients", diff --git a/tests/server/link-management-relay.test.ts b/tests/server/link-management-relay.test.ts new file mode 100644 index 00000000000..7eeca5f507a --- /dev/null +++ b/tests/server/link-management-relay.test.ts @@ -0,0 +1,486 @@ +/** + * #6957: the Home end of the Remote Link management relay, through the real hub-link listener. + * + * The link key admits only allowlisted GET reads under /v1/link-management; every other method, + * path or refresh query is a 404 before a handler runs, and the prefix does not exist on the + * public listener at all. The Codex settings document is what a Child applies when it syncs. + */ +import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test"; +import * as childProcess from "node:child_process"; +import * as fs from "node:fs"; +import * as fsPromises from "node:fs/promises"; +import { mkdtempSync, readdirSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { OcxConfig } from "../../src/types"; +import { isPassiveRead } from "../../src/lib/passive-read"; +import { saveCodexAccountCredential } from "../../src/codex/account-store"; +import { linkStorePath } from "../../src/link/paths"; +import { writeLinkStore } from "../../src/link/store"; +import { parseHomeCodexSettingsDocument } from "../../src/link/codex-settings"; +import { LINK_MANAGEMENT_READ_PATHS } from "../../src/link/management-routes"; +import { installIsolatedCodexHome, type IsolatedCodexHome } from "../helpers/isolated-codex-home"; +import { removeTreeWithRetry } from "../helpers/remove-tree"; + +const ENV_KEY = "link-mgmt-env-admission"; +const OTHER_KEY = "link-mgmt-other-admission"; +const LINKED_KEY = "link-mgmt-linked-admission"; +const LINKED_ID = "linked-key"; +const ADMIN_KEY = "link-mgmt-admin-admission"; +const PROVIDER_SECRET = "fixture-provider-credential-not-relayed"; +/** A provider still waiting for its first model choice: discovery would decide it and save. */ +const PENDING_MODELS = Array.from({ length: 20 }, (_, index) => `pending-model-${index + 1}`); +/** The query each allowlisted read needs to reach its handler's work, as the CLI sends it. */ +const READ_QUERY: Record = { "/api/providers/keys": "?name=mock", "/api/oauth/accounts": "?provider=anthropic" }; + +const previous = { + home: process.env.OPENCODEX_HOME, + codexHome: process.env.CODEX_HOME, + apiToken: process.env.OPENCODEX_API_AUTH_TOKEN, + adminToken: process.env.OPENCODEX_ADMIN_AUTH_TOKEN, +}; +let testHome = ""; +let codexHome: IsolatedCodexHome | null = null; +let server: { stop(closeActiveConnections?: boolean): Promise; url: URL } | null = null; +/** Every request a Home handler sends to a provider upstream lands here. */ +let upstream: ReturnType | null = null; +let upstreamHits = 0; +/** Requests that left the machine (answered locally here), and everything a passive read did. */ +let externalCalls: string[] = []; +let passiveEffects: string[] = []; +let restoreWatchers: Array<() => void> = []; + +/** + * Answer every non-loopback request locally, so no test here reaches a real provider, and record + * any fetch, child process or file mutation made inside a passive read, whatever started it. + */ +function watchHomeEffects(): void { + const originalFetch = globalThis.fetch; + globalThis.fetch = ((input: string | URL | Request, init?: RequestInit) => { + const url = typeof input === "string" ? input : input instanceof URL ? input.href : input.url; + if (isPassiveRead()) passiveEffects.push(`fetch ${url}`); + if (/^https?:\/\/(127\.0\.0\.1|localhost|0\.0\.0\.0|\[::1\])[:/]/.test(url)) return originalFetch(input, init); + externalCalls.push(url); + return Promise.resolve(Response.json({ error: "fixture upstream" }, { status: 503 })); + }) as typeof fetch; + restoreWatchers.push(() => { globalThis.fetch = originalFetch; }); + const wrap = (target: object, names: string[], label: string, mutates: (args: unknown[]) => boolean = () => true) => { + for (const name of names) { + const original = (target as Record unknown>)[name]!; + const spy = spyOn(target as Record unknown>, name).mockImplementation(function (this: unknown, ...args: unknown[]) { + if (isPassiveRead() && mutates(args)) passiveEffects.push(`${label}.${name} ${String(args[0])}`); + return original.apply(this, args); + }); + restoreWatchers.push(() => spy.mockRestore()); + } + }; + wrap(childProcess, ["execFile", "execFileSync", "spawn", "spawnSync", "exec", "execSync"], "child_process"); + wrap(Bun, ["spawn", "spawnSync", "write"], "Bun"); + wrap(fs, ["writeFileSync", "appendFileSync", "copyFileSync", "renameSync", "chmodSync", "mkdirSync", "rmSync", "unlinkSync", + "writeSync", "ftruncateSync", "truncateSync", "linkSync", "symlinkSync", "utimesSync"], "fs"); + wrap(fsPromises, ["writeFile", "appendFile", "copyFile", "rename", "chmod", "mkdir", "rm", "unlink", "truncate"], "fs/promises"); + // Opening for reading is a read; any other flag can create or change a file. + const opensForWrite = (args: unknown[]) => args[1] !== undefined && args[1] !== "r" && args[1] !== fs.constants.O_RDONLY; + wrap(fs, ["openSync"], "fs", opensForWrite); + wrap(fsPromises, ["open"], "fs/promises", opensForWrite); +} + +/** Every file under the Home's config and Codex homes, with what a write would change. */ +function durableState(): Record { + const state: Record = {}; + const walk = (dir: string) => { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const path = join(dir, entry.name); + if (entry.isDirectory()) { walk(path); continue; } + try { + const stat = fs.statSync(path); + state[path] = `${stat.size}:${stat.mtimeMs}:${stat.mode}`; + } catch { state[path] = "vanished"; } + } + }; + walk(testHome); + if (codexHome) walk(codexHome.path); + return state; +} + +function key(id: string, value: string) { + return { id, name: id, key: value, createdAt: "2026-09-25T00:00:00.000Z" }; +} + +function config(): OcxConfig { + return { + port: 0, + hostname: "0.0.0.0", + runtimeRole: "hub", + defaultProvider: "mock", + providers: { + mock: { adapter: "openai-chat", baseUrl: "http://127.0.0.1:9/v1", allowPrivateNetwork: true, apiKey: PROVIDER_SECRET, models: ["test-model"] }, + pending: { + adapter: "openai-chat", baseUrl: "http://127.0.0.1:9/v1", allowPrivateNetwork: true, liveModels: false, models: PENDING_MODELS, + initialModelSelection: { version: 1, registrationId: "69596959-6959-4959-8959-695969596959", status: "pending" }, + }, + live: { adapter: "openai-chat", baseUrl: `${upstream!.url.origin}/v1`, allowPrivateNetwork: true, apiKey: "fixture-live-upstream-key" }, + // A provider whose quota the Home probes upstream on a cold cache. + openrouter: { adapter: "openai-chat", baseUrl: "https://openrouter.ai/api/v1", apiKey: "fixture-openrouter-key", liveModels: false, models: ["openrouter/auto"] }, + // Canonical OpenAI: the sidecar reads then ask whether native main may be used. + openai: { adapter: "openai-responses", baseUrl: "https://chatgpt.com/backend-api/codex", authMode: "forward" }, + }, + // A Codex pool account whose token has expired: reading its quota would refresh the token first. + codexAccounts: [{ id: "pool-expired", email: "pool-expired@example.test", plan: "plus", isMain: false }], + apiKeys: [key("other-key", OTHER_KEY), key(LINKED_ID, LINKED_KEY)], + clientIntegrations: { codex: false }, + fastMode: true, + multiAgentMode: "v1", + injectionModel: "mock/test-model", + injectionEffort: "high", + syncCodexSubagentDefaults: true, + webSearchSidecar: { enabled: false }, + } as OcxConfig; +} + +async function linkBase(): Promise { + const linkPort = JSON.parse(await Bun.file(linkStorePath()).text()).listenerPort as number; + return `http://127.0.0.1:${linkPort}`; +} + +function get(base: string, path: string, credential?: string, init: RequestInit = {}): Promise { + return fetch(`${base}${path}`, { + ...init, + headers: { ...(credential ? { authorization: `Bearer ${credential}` } : {}), ...(init.headers as Record | undefined) }, + }); +} + +beforeEach(async () => { + testHome = mkdtempSync(join(tmpdir(), "ocx-link-mgmt-")); + process.env.OPENCODEX_HOME = testHome; + process.env.OPENCODEX_API_AUTH_TOKEN = ENV_KEY; + process.env.OPENCODEX_ADMIN_AUTH_TOKEN = ADMIN_KEY; + upstreamHits = 0; + externalCalls = []; + passiveEffects = []; + watchHomeEffects(); + upstream = Bun.serve({ + port: 0, + hostname: "127.0.0.1", + fetch: () => { + upstreamHits += 1; + return Response.json({ object: "list", data: [{ id: "live-a", object: "model" }, { id: "live-b", object: "model" }] }); + }, + }); + codexHome = installIsolatedCodexHome("ocx-link-mgmt-codex-"); + writeFileSync(join(codexHome.path, "opencodex-catalog.json"), JSON.stringify({ models: [{ slug: "mock/test-model" }] })); + writeFileSync(join(codexHome.path, "config.toml"), ""); + saveCodexAccountCredential("pool-expired", { + accessToken: "access-pool-expired", refreshToken: "refresh-pool-expired", + expiresAt: Date.now() - 60_000, chatgptAccountId: "acct-pool-expired", + }); + // A malformed, loosely permitted OAuth store: a Home load backs it up and tightens it. + writeFileSync(join(testHome, "auth.json"), "{ not json", { mode: 0o644 }); + const { saveConfig } = await import("../../src/config"); + writeLinkStore(linkStorePath(), { + version: 1, + listenerPort: null, + links: [{ + id: "lnk_0123456789abcdef", + alias: "mgmt-test", + direction: "client-initiated", + hostKeyFingerprint: "SHA256:abcdefghijklmnop", + tunnelPort: 2222, + apiKeyId: LINKED_ID, + createdAt: "2026-09-25T00:00:00.000Z", + }], + }); + saveConfig(config()); + const { startServer } = await import("../../src/server"); + server = startServer(0); +}); + +afterEach(async () => { + if (server) await server.stop(true); + server = null; + await upstream?.stop(true); + upstream = null; + for (const restore of restoreWatchers.splice(0).reverse()) restore(); + codexHome?.restore(); + codexHome = null; + if (previous.home === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previous.home; + if (previous.codexHome === undefined) delete process.env.CODEX_HOME; + else process.env.CODEX_HOME = previous.codexHome; + if (previous.apiToken === undefined) delete process.env.OPENCODEX_API_AUTH_TOKEN; + else process.env.OPENCODEX_API_AUTH_TOKEN = previous.apiToken; + if (previous.adminToken === undefined) delete process.env.OPENCODEX_ADMIN_AUTH_TOKEN; + else process.env.OPENCODEX_ADMIN_AUTH_TOKEN = previous.adminToken; + if (testHome) removeTreeWithRetry(testHome); + testHome = ""; +}); + +describe("Remote Link management relay on the Home", () => { + test("admits allowlisted reads only with the link key", async () => { + const base = await linkBase(); + for (const credential of [undefined, ENV_KEY, OTHER_KEY]) { + const refused = await get(base, "/v1/link-management/api/effort-caps", credential); + expect({ credential, status: refused.status }).toEqual({ credential, status: 401 }); + } + const caps = await get(base, "/v1/link-management/api/effort-caps", LINKED_KEY); + expect(caps.status).toBe(200); + expect(caps.headers.get("cache-control")).toBe("no-store"); + expect(Array.isArray((await caps.json() as { efforts?: unknown }).efforts)).toBe(true); + const head = await get(base, "/v1/link-management/api/settings", LINKED_KEY, { method: "HEAD" }); + expect(head.status).toBe(200); + expect(await head.text()).toBe(""); + }); + + test("relays a masked projection and never a provider credential", async () => { + const providers = await get(await linkBase(), "/v1/link-management/api/providers", LINKED_KEY, { + headers: { cookie: "session=browser", "x-opencodex-gui-origin": "http://127.0.0.1:1" }, + }); + expect(providers.status).toBe(200); + const text = await providers.text(); + expect(text).not.toContain(PROVIDER_SECRET); + expect(text).not.toContain(LINKED_KEY); + const rows = JSON.parse(text) as Array<{ name: string; hasApiKey: boolean }>; + expect(rows.find(row => row.name === "mock")?.hasApiKey).toBe(true); + const keys = await get(await linkBase(), "/v1/link-management/api/providers/keys?name=mock", LINKED_KEY); + expect(keys.status).toBe(200); + expect(await keys.text()).not.toContain(PROVIDER_SECRET); + }); + + test("refuses writes, unlisted reads and refresh queries before a handler runs", async () => { + const base = await linkBase(); + const refused: Array<[string, string]> = [ + ["PUT", "/v1/link-management/api/settings"], + ["POST", "/v1/link-management/api/providers"], + ["DELETE", "/v1/link-management/api/providers"], + ["GET", "/v1/link-management/api/logs"], + ["GET", "/v1/link-management/api/config"], + ["GET", "/v1/link-management/api/provider-quotas?refresh=1"], + ["GET", "/v1/link-management/api/codex-auth/accounts?refresh=1"], + // Reads that make the Home do upstream work or persist discovery are not reads here. + ["GET", "/v1/link-management/api/oauth/accounts?provider=anthropic"a=1"], + ["GET", "/v1/link-management/api/providers/keys?name=mock"a=1"], + ["GET", "/v1/link-management/api/models"], + ["GET", "/v1/link-management/api/model-presets"], + ["GET", "/v1/link-management/api/subagent-models"], + ["GET", "/v1/link-management/api/subagent-model-fallback"], + ["GET", "/v1/link-management/api/diagnostics/project-config"], + ["GET", "/v1/link-management/codex-settings?x=1"], + ["GET", "/v1/link-management"], + ]; + for (const [method, path] of refused) { + const response = await get(base, path, LINKED_KEY, { method, ...(method === "GET" ? {} : { body: "{}" }) }); + expect({ method, path, status: response.status }).toEqual({ method, path, status: 404 }); + } + }); + + test("the prefix exists only on the link listener", async () => { + const response = await get(server!.url.origin, "/v1/link-management/api/effort-caps", LINKED_KEY); + expect(response.status).not.toBe(200); + expect(await response.text()).not.toContain("efforts"); + }); + + test("every allowlisted read names a real Home management route", async () => { + const base = await linkBase(); + for (const path of LINK_MANAGEMENT_READ_PATHS) { + // The key-pool read names its provider, as `ocx account` sends it. + const query = path === "/api/providers/keys" ? "?name=mock" : ""; + const response = await get(base, `/v1/link-management${path}${query}`, LINKED_KEY); + await response.body?.cancel(); + expect({ path, routed: response.status !== 404 && response.status !== 401 }).toEqual({ path, routed: true }); + } + }); + + test("no allowlisted read changes the Home config, reaches an upstream or starts a process (#6959)", async () => { + const base = await linkBase(); + const { getConfigPath } = await import("../../src/config"); + const before = readFileSync(getConfigPath(), "utf8"); + expect(JSON.parse(before).providers.pending.initialModelSelection.status).toBe("pending"); + const admin = { headers: { "x-opencodex-api-key": ADMIN_KEY } }; + // The Home's live, in-memory configuration as its own dashboard reads it. + const liveConfig = async () => { + const response = await get(server!.url.origin, "/api/config", undefined, admin); + expect(response.status).toBe(200); + return await response.json() as { providers: Record }; + }; + const liveBefore = await liveConfig(); + expect(liveBefore.providers.pending?.initialModelSelection?.status).toBe("pending"); + // Native main fenced because service ownership was unknown at boot: re-inspecting it from a + // read would activate the native-main lifecycle and its recovery. + const nativeStartup = await import("../../src/codex/native-profile-startup"); + let ownershipProbes = 0; + const fence = nativeStartup.blockNativeMainStartupForUnownedServiceHome("ownership-unknown", { + reprobe: () => { ownershipProbes += 1; return "unknown"; }, + expectedHomeId: "link-6959-home", + startOwnedLifecycle: () => { throw new Error("fixture: not activated"); }, + }); + nativeStartup.__resetNativeMainOwnershipRetries(); + const durableBefore = durableState(); + upstreamHits = 0; + passiveEffects = []; + // Each read is judged against the state the previous read left, so one finding cannot hide another. + const findings: Array<{ method: string; path: string; status: number; configChanged: boolean; effects: string[] }> = []; + let current = before; + for (const method of ["GET", "HEAD"]) { + for (const path of LINK_MANAGEMENT_READ_PATHS) { + const response = await get(base, `/v1/link-management${path}${READ_QUERY[path] ?? ""}`, LINKED_KEY, { method }); + await response.body?.cancel(); + const after = readFileSync(getConfigPath(), "utf8"); + const row = { method, path, status: response.status, configChanged: after !== current, effects: passiveEffects.splice(0) }; + if (row.status !== 200 || row.configChanged || row.effects.length > 0) findings.push(row); + current = after; + } + } + expect(findings).toEqual([]); + expect(ownershipProbes).toBe(0); + expect(durableState()).toEqual(durableBefore); + // Work a read leaves scheduled stays in its passive scope. The longest ordinary delay among the + // known producers is the responses snapshot debounce (SNAPSHOT_DEBOUNCE_MS, 2 s). + await Bun.sleep(2_500); + expect(passiveEffects).toEqual([]); + expect(durableState()).toEqual(durableBefore); + expect(readFileSync(getConfigPath(), "utf8")).toBe(before); + expect(await liveConfig()).toEqual(liveBefore); + expect(upstreamHits).toBe(0); + + // The fixture is live: the same reads made by the Home itself do each kind of work. + const homeRead = async (path: string) => { await (await get(server!.url.origin, path, undefined, admin)).body?.cancel(); }; + const backups = () => readdirSync(testHome).filter(name => name.startsWith("auth.json.invalid-")).length; + const backupsBefore = backups(); + await homeRead("/api/oauth/accounts?provider=anthropic"); + expect(backups()).toBeGreaterThan(backupsBefore); + externalCalls = []; + await homeRead("/api/provider-quotas"); + expect(externalCalls.some(url => url.includes("openrouter.ai"))).toBe(true); + externalCalls = []; + await homeRead("/api/codex-auth/accounts"); + expect(externalCalls.length).toBeGreaterThan(0); + await homeRead("/api/sidecar-settings"); + expect(ownershipProbes).toBeGreaterThan(0); + void fence.release(); + await homeRead("/api/models"); + expect(upstreamHits).toBeGreaterThan(0); + expect(JSON.parse(readFileSync(getConfigPath(), "utf8")).providers.pending.initialModelSelection.status).not.toBe("pending"); + expect((await liveConfig()).providers.pending?.initialModelSelection?.status).not.toBe("pending"); + // Work a link read left scheduled would have run by now, still inside its passive scope. + expect(passiveEffects).toEqual([]); + }); + + test("answers the injection model read from Home config without discovery", async () => { + const response = await get(await linkBase(), "/v1/link-management/api/injection-model", LINKED_KEY); + expect(response.status).toBe(200); + const body = await response.json() as { model: string | null; effort: string | null; efforts: string[]; available: Array<{ namespaced: string }> }; + expect({ model: body.model, effort: body.effort }).toEqual({ model: "mock/test-model", effort: "high" }); + expect(body.efforts).toContain("high"); + const offered = body.available.map(row => row.namespaced); + expect(offered).toContain("mock/test-model"); + expect(offered.some(slug => slug.startsWith("live/") || slug.startsWith("pending/"))).toBe(false); + expect(upstreamHits).toBe(0); + }); + + test("publishes the Home's OpenCodex-owned Codex settings", async () => { + const response = await get(await linkBase(), "/v1/link-management/codex-settings", LINKED_KEY); + expect(response.status).toBe(200); + const document = parseHomeCodexSettingsDocument(await response.json()); + expect(document?.settings).toEqual({ + fastMode: true, + multiAgentMode: "v1", + webSearchSidecarEnabled: false, + subagentDefaults: { model: "mock/test-model", reasoningEffort: "high" }, + }); + expect(document?.home.name.length).toBeGreaterThan(0); + }); +}); + +describe("passive management reads (#6959)", () => { + test("demote only memory-only stores when a usage read meets a full memory budget", async () => { + const memory = await import("../../src/lib/app-owned-memory"); + const { runPassiveRead } = await import("../../src/lib/passive-read"); + const replay = await import("../../src/adapters/google-antigravity-replay"); + // A persisted replay session: evicting it rewrites antigravity-replay.json after a debounce. + replay.__resetAntigravityReplayCache(); + replay.observeAntigravityReplay("gemini-3-pro", "-6959", [{ functionCall: { name: "get_x", args: { a: 1 } }, thoughtSignature: "sig-6959-0123456789abcdef" }]); + expect(replay.antigravityReplaySessionKeysForTests().length).toBe(1); + // A cache whose eviction is not declared memory-only. + let undeclaredDemotions = 0; + const unregisterCache = memory.registerRetainedStore({ + id: "fixture_undeclared_cache", + category: "caches", + snapshot: () => ({ count: 1, bytes: 1024, evictableBytes: 1024, pinnedBytes: 0, oldestAt: 0 }), + evictOldest: () => { undeclaredDemotions += 1; return 0; }, + }); + // A resident continuation far over any budget: demoting it would spill it to disk. + let continuationDemotions = 0; + const unregister = memory.registerRetainedStore({ + id: "fixture_continuation", + category: "continuation", + snapshot: () => ({ count: 1, bytes: 8 * 1024 ** 3, evictableBytes: 8 * 1024 ** 3, pinnedBytes: 0, oldestAt: 1 }), + evictOldest: () => { continuationDemotions += 1; return 0; }, + }); + try { + // One ledger row, so a cold usage read builds and publishes its caches. + writeFileSync(join(testHome, "usage.jsonl"), `${JSON.stringify({ + requestId: "usage-6959", timestamp: Date.now() - 1_000, provider: "mock", model: "test-model", + status: 200, durationMs: 1, usageStatus: "reported", usage: { inputTokens: 1, outputTokens: 1 }, totalTokens: 2, + })}\n`); + const runs = () => memory.appOwnedBytesSnapshot().enforcement.runs; + const before = runs(); + for (const method of ["GET", "HEAD"]) { + for (const query of ["", "?range=7d", "?range=30d"]) { + const response = await get(await linkBase(), `/v1/link-management/api/usage${query}`, LINKED_KEY, { method }); + expect(response.status).toBe(200); + await response.body?.cancel(); + } + } + expect(runs()).toBeGreaterThan(before); + expect(await runPassiveRead(async () => memory.enforceAppOwnedMemoryBudget())).toBeDefined(); + // Past the replay snapshot debounce: a demotion would have scheduled its write by now. + await Bun.sleep(2_500); + expect({ continuationDemotions, undeclaredDemotions, replaySessions: replay.antigravityReplaySessionKeysForTests().length, passiveEffects }) + .toEqual({ continuationDemotions: 0, undeclaredDemotions: 0, replaySessions: 1, passiveEffects: [] }); + // The Home's own enforcement demotes all three. + memory.enforceAppOwnedMemoryBudget(); + expect(continuationDemotions).toBeGreaterThan(0); + expect(undeclaredDemotions).toBeGreaterThan(0); + expect(replay.antigravityReplaySessionKeysForTests().length).toBe(0); + } finally { + unregister(); + unregisterCache(); + replay.__resetAntigravityReplayCache(); + } + }); + + test("discover no models, so a pending provider stays pending", async () => { + const { fetchAllModels } = await import("../../src/server/management/shared"); + const { getConfigPath, loadConfig } = await import("../../src/config"); + const { runPassiveRead } = await import("../../src/lib/passive-read"); + const before = readFileSync(getConfigPath(), "utf8"); + expect(await runPassiveRead(() => fetchAllModels(loadConfig()))).toEqual([]); + expect(readFileSync(getConfigPath(), "utf8")).toBe(before); + expect(upstreamHits).toBe(0); + }); + + test("serve the last startup-health reading without starting a diagnostic", async () => { + const { getCachedStartupHealth, getStartupHealthSnapshot, resetStartupHealthCacheForTests } = await import("../../src/server/startup-health-cache"); + const { runPassiveRead } = await import("../../src/lib/passive-read"); + resetStartupHealthCacheForTests(); + let probes = 0; + let reading: Awaited> | null = null; + const deps = { + probe: async () => { probes += 1; return reading!; }, + waitForProbe: (probe: Promise) => probe, + }; + try { + reading = await runPassiveRead(async () => { + getStartupHealthSnapshot(config(), deps); + return await getCachedStartupHealth(config(), deps); + }); + expect(reading.diagnosticStale).toBe(true); + expect(probes).toBe(0); + // The Home's own read of the same cold cache starts the diagnostic. + await getCachedStartupHealth(config(), deps); + expect(probes).toBe(1); + } finally { + resetStartupHealthCacheForTests(); + } + }); +}); From 6c2708751858d400311ea004767eae0f0feeecbf Mon Sep 17 00:00:00 2001 From: lidge-jun Date: Tue, 13 Oct 2026 01:09:43 +0900 Subject: [PATCH 8/9] feat(gui): opt in custom Responses plaintext reasoning replay (#6961) Squashed from #6961 for the 2.84.0 release train. Closes #6675. - Add a default-off plaintext reasoning replay checkbox to custom Responses provider creation and Settings. Explain that enabling it sends previous reasoning to the endpoint and can increase token use; local/key custom destinations are eligible, and curated/OpenAI/Azure/OAuth/forward surfaces keep their current behavior. - Persist explicit true/false choices through the existing capability, validate boolean/null PATCH values, and retain omitted configuration only when an overwrite keeps the same destination. Hidden stale form values do not enter create or update payloads. - Update all eleven GUI locales and the provider/management documentation. Closes #6675 --- .../docs/reference/configuration/providers.md | 2 +- gui/src/components/add-provider-form-pane.tsx | 6 + .../provider-workspace/ProviderSettings.tsx | 14 +- .../components/provider-workspace/types.ts | 1 + .../responses-reasoning-replay-control.tsx | 16 ++ gui/src/i18n/de.ts | 2 + gui/src/i18n/en.ts | 2 + gui/src/i18n/fr.ts | 2 + gui/src/i18n/ja.ts | 2 + gui/src/i18n/ko.ts | 2 + gui/src/i18n/pt.ts | 2 + gui/src/i18n/ru.ts | 2 + gui/src/i18n/tr.ts | 2 + gui/src/i18n/vi.ts | 2 + gui/src/i18n/zh-TW.ts | 2 + gui/src/i18n/zh.ts | 2 + gui/src/pages/providers-shared.ts | 1 + gui/src/provider-payload.ts | 7 + gui/src/provider-workspace/catalog.ts | 1 + gui/src/responses-reasoning-replay.ts | 17 ++ ...rovider-settings-reasoning-replay.test.tsx | 153 ++++++++++++++++++ scripts/test-layout/layout.json | 2 +- .../management/provider-overwrite-carry.ts | 10 ++ src/server/management/provider-routes.ts | 1 + structure/dashboard-and-usage.md | 2 +- structure/gui-and-management-api.md | 2 +- tests/fixtures/test-layout-expected.json | 3 +- .../management-provider-compat-carry.test.ts | 15 +- ...ement-provider-responses-reasoning.test.ts | 132 +++++++++++++++ 29 files changed, 398 insertions(+), 9 deletions(-) create mode 100644 gui/src/components/responses-reasoning-replay-control.tsx create mode 100644 gui/src/responses-reasoning-replay.ts create mode 100644 gui/tests/provider-settings-reasoning-replay.test.tsx create mode 100644 tests/server/management-provider-responses-reasoning.test.ts diff --git a/docs-site/src/content/docs/reference/configuration/providers.md b/docs-site/src/content/docs/reference/configuration/providers.md index a333e8cca8b..60050d9b663 100644 --- a/docs-site/src/content/docs/reference/configuration/providers.md +++ b/docs-site/src/content/docs/reference/configuration/providers.md @@ -227,7 +227,7 @@ Providers can expose a built-in shorthand, such as `agy` for `google-antigravity | `modelSupportsServiceTier?` | `Record` | Exact upstream model capability overrides. Exact `true` enables canonical Fast for that model; exact `false` narrows provider defaults. An explicit provider-level `supportsServiceTier: false` remains fail-closed and cannot be reopened. Exact `true` does not authorize foreign caller-tier forwarding on Chat. Undeclared models fall back to provider-wide behavior. Management `PATCH /api/providers` merges entries and accepts `null` to clear one. | | `chatServiceTier?` | `boolean` | Provider-wide Chat-wire opt-in for forwarding caller `service_tier` values. On a classified route it governs foreign values such as `flex`, not proxy-owned canonical Fast after capability validation; on an unclassified route it governs every caller value because no Fast capability has been validated. Exact model capability does not authorize foreign forwarding. Responses routes retain their capability-based caller forwarding behavior. | | `promptCacheKey?` | `boolean` | Provider-wide `openai-chat` opt-in for forwarding a `prompt_cache_key`. The adapter forwards the key it is given and never invents one, but the key is not always the caller's: Claude Messages translation derives one from `metadata.user_id`, or from a model/system/tools cohort when no metadata is sent. Default off. Enable only when the upstream documents support, because strict gateways may reject the unknown field with HTTP 400. | -| `preserveResponsesReasoningContent?` | `boolean` | Keep plaintext reasoning content on replayed Responses reasoning items instead of blanking it (blanking is the ChatGPT backend's rule). Enable for upstreams whose contract accepts reasoning replay, such as DeepSeek or a self-hosted Kimi on vLLM. Proxy-minted `ocxr1` envelopes are always stripped. A hand-configured `openai-responses` provider without it logs one warning per host the first time plaintext reasoning is dropped. `preserveReasoningContentModels` does not affect this wire. | +| `preserveResponsesReasoningContent?` | `boolean` | Default off for custom providers. In **Providers → Settings**, enable **Replay plaintext reasoning** for a custom Responses endpoint only if it accepts this history. This sends prior plaintext reasoning to that endpoint and may increase token use. Keep plaintext reasoning content on replayed Responses reasoning items instead of blanking it (blanking is the ChatGPT backend's rule). Enable for upstreams whose contract accepts reasoning replay, such as DeepSeek or a self-hosted Kimi on vLLM. Proxy-minted `ocxr1` envelopes are always stripped. A hand-configured `openai-responses` provider without it logs one warning per host the first time plaintext reasoning is dropped. `preserveReasoningContentModels` does not affect this wire. | | `preserveResponsesInputItemIds?` | `boolean` | Default off. Preserve valid `input[].id` when `store: false` for explicitly trusted launcher relays that require turn identity. Ordinary Responses providers keep stripping IDs; xAI custom-call ID repair remains active. | | `preserveResponsesMessageMetadata?` | `boolean` | Default off. Preserve private `internal_chat_message_metadata_passthrough` on input items for explicitly trusted launcher relays. This forwards private turn/session metadata to that destination; do not enable for ordinary gateways. | | `dropResponsesReasoningItems?` | `boolean` | Remove replayed Responses `reasoning` items from continuation input entirely, instead of forwarding them with blanked content. This is lossy: summaries, item ids, and `encrypted_content` go with them, so no reasoning item from the previous turn remains in the forwarded input. It does not remove `previous_response_id`, so an upstream holding its own turn state may still reach it. Enable only for an upstream that rejects reasoning replay outright — the built-in Volcengine Ark Coding Plan preset sets it because validated tool continuations answered `400 InvalidParameter` otherwise. Set it to `false` on that provider to forward reasoning items again. | diff --git a/gui/src/components/add-provider-form-pane.tsx b/gui/src/components/add-provider-form-pane.tsx index ba60e4902d4..8122f080a1e 100644 --- a/gui/src/components/add-provider-form-pane.tsx +++ b/gui/src/components/add-provider-form-pane.tsx @@ -5,6 +5,8 @@ import type { CatalogPreset } from "./provider-catalog/provider-presets"; import type { ProviderPayloadForm } from "../provider-payload"; import { AddProviderField } from "./add-provider-modal-field"; import { baseUrlForChoice } from "../base-url-choice"; +import { canConfigureResponsesReasoningReplay } from "../responses-reasoning-replay"; +import { ResponsesReasoningReplayControl } from "./responses-reasoning-replay-control"; import { isSubscriptionCliProvider, SUBSCRIPTION_CLI_API_KEY_ALTERNATIVE } from "../provider-workspace/subscription-cli"; const FORM_ADAPTERS = ["openai-responses", "openai-chat", "anthropic", "google", "azure-openai", "cursor"]; @@ -131,6 +133,10 @@ export function AddProviderFormPane({

{t("modal.allowPrivateNetworkHint")}

)} } + {isCustom && canConfigureResponsesReasoningReplay(form) && ( + onFormChange({ ...form, preserveResponsesReasoningContent: checked })} /> + )} {form.authMode === "forward" ? (
{presetDescription(preset)} diff --git a/gui/src/components/provider-workspace/ProviderSettings.tsx b/gui/src/components/provider-workspace/ProviderSettings.tsx index 436dc3a97d5..9856e7b90f9 100644 --- a/gui/src/components/provider-workspace/ProviderSettings.tsx +++ b/gui/src/components/provider-workspace/ProviderSettings.tsx @@ -18,6 +18,8 @@ import { useT } from "../../i18n/shared"; import { IconLock } from "../../icons"; import { isCatalogProviderId } from "../../provider-icons"; import { openAiAccountProviderState } from "../../provider-payload"; +import { canConfigureResponsesReasoningReplay } from "../../responses-reasoning-replay"; +import { ResponsesReasoningReplayControl } from "../responses-reasoning-replay-control"; import { providerSupportsLiveModelDiscovery } from "../../provider-workspace/catalog"; import type { CatalogPreset } from "../provider-catalog/provider-presets"; import { authModeLabel } from "./ProviderRail"; @@ -84,6 +86,7 @@ export default function ProviderSettings({ const [apiKeyTransport, setApiKeyTransport] = useState(item.apiKeyTransport ?? "x-api-key"); const [note, setNote] = useState(item.note ?? ""); const [allowPrivateNetwork, setAllowPrivateNetwork] = useState(item.allowPrivateNetwork ?? false); + const [preserveReasoningReplay, setPreserveReasoningReplay] = useState(item.preserveResponsesReasoningContent === true); const [liveModels, setLiveModels] = useState(savedLiveModels); const [cursorHttpVersion, setCursorHttpVersion] = useState(savedCursorHttpVersion); const [saving, setSaving] = useState(false); @@ -114,6 +117,7 @@ export default function ProviderSettings({ setApiKeyTransport(item.apiKeyTransport ?? "x-api-key"); setNote(item.note ?? ""); setAllowPrivateNetwork(item.allowPrivateNetwork ?? false); + setPreserveReasoningReplay(item.preserveResponsesReasoningContent === true); setLiveModels(savedLiveModels); setCursorHttpVersion(savedCursorHttpVersion); setPacingEnabled(item.requestPacing?.enabled === true); @@ -124,7 +128,7 @@ export default function ProviderSettings({ setMsg(null); setModeMsg(null); queueMicrotask(() => setEndpointChoice(matchChoiceId(baseUrlChoices, item.baseUrl))); - }, [item.adapter, item.baseUrl, item.defaultModel, item.authMode, item.apiKeyTransport, item.keyOptional, item.note, item.allowPrivateNetwork, savedLiveModels, savedCursorHttpVersion, item.requestPacing, baseUrlChoices]); + }, [item.adapter, item.baseUrl, item.defaultModel, item.authMode, item.apiKeyTransport, item.keyOptional, item.note, item.allowPrivateNetwork, item.preserveResponsesReasoningContent, savedLiveModels, savedCursorHttpVersion, item.requestPacing, baseUrlChoices]); /* eslint-enable react-hooks/set-state-in-effect */ // Account mode syncs on its own: a mode PATCH refresh must not reset an in-progress @@ -194,7 +198,10 @@ export default function ProviderSettings({ ...(Object.keys(pacingModels).length > 0 ? { models: pacingModels } : {}), }), [pacingConcurrency, pacingDelay, pacingEnabled, pacingModels, pacingRpm]); - const dirty = adapter.trim() !== item.adapter + const supportsReasoningReplay = !isCatalogProviderId(item.name) + && canConfigureResponsesReasoningReplay({ adapter, baseUrl, authMode }); + const reasoningReplayDirty = supportsReasoningReplay && preserveReasoningReplay !== (item.preserveResponsesReasoningContent === true); + const dirty = reasoningReplayDirty || adapter.trim() !== item.adapter || baseUrl.trim() !== item.baseUrl || defaultModel.trim() !== (item.defaultModel ?? "") || authMode !== String(item.authMode ?? (item.keyOptional ? "local" : "key")) @@ -262,6 +269,7 @@ export default function ProviderSettings({ // Keep omitted legacy values omitted unless the user actually changes this toggle. // Otherwise an unrelated settings save manufactures `liveModels: true` provenance. if (liveModelDiscoverySupported && liveModels !== (item.liveModels !== false)) patch.liveModels = liveModels; + if (reasoningReplayDirty) patch.preserveResponsesReasoningContent = preserveReasoningReplay; if (adapter.trim() === "cursor" && cursorHttpVersion !== savedCursorHttpVersion) { patch.upstreamHttpVersion = cursorHttpVersion === "http1.1" ? "http1.1" : null; } @@ -328,6 +336,7 @@ export default function ProviderSettings({ setDefaultModel(item.defaultModel ?? ""); setAuthMode(initialAuth); setApiKeyTransport(item.apiKeyTransport ?? "x-api-key"); setNote(item.note ?? ""); setAllowPrivateNetwork(item.allowPrivateNetwork ?? false); setLiveModels(savedLiveModels); + setPreserveReasoningReplay(item.preserveResponsesReasoningContent === true); setCursorHttpVersion(savedCursorHttpVersion); setMsg(null); setPacingEnabled(item.requestPacing?.enabled === true); setPacingRpm(numberDraft(item.requestPacing?.requestsPerMinute)); setPacingDelay(numberDraft(item.requestPacing?.minIntervalMs)); @@ -502,6 +511,7 @@ export default function ProviderSettings({ setAllowPrivateNetwork(e.target.checked)} /> {t("pws.allowPrivateNetwork")} + {supportsReasoningReplay && }