From f4bf2ee44797ada2e0c3cefc97bf85b949e558b6 Mon Sep 17 00:00:00 2001 From: bryding <481770+bryding@users.noreply.github.com> Date: Thu, 8 Oct 2026 21:53:55 -0600 Subject: [PATCH 1/2] w738, w739: each person's Claude token pool, the orchestrators and the ops worker on it, the dispatcher's reserve A person's runs use that person's own vault Claude tokens (several per person: people//claude-tokens/ on the FFBox host, loaded by fff-vm vault-sync), picked by server/tokenPool.ts at every process start: the weekly window that resets soonest first, the 5-hour reset as tie-break; no new process at 80% of the 5-hour window, one at a time at 95% weekly, retired at 99%, used up at 100; a session keeps its token only while it passes. A worker, standing agent and the ops worker are held with the reason when the pool is capped (no fallback); a person's orchestrator uses the pool up and then stops with a message. No shared Claude token, nobody else's. A person with no vault token keeps today's account. - claudeAccounts.orchestrator "vault" (an owner's setting; the dispatcher stays on the token file): poolRunEnv gives the orchestrator and the ops worker the person's one token with every other Claude credential removed. - The ops worker starts a fresh conversation and process for every new job and every change of caller. - Limits are config vault.pool.* (80/95/99, 5% a day, 20%): one system-wide set anyone can change with set_app_config, checked 0-100, one-at-a-time not above retired, logged with who, read at each process start. - The dispatcher's credential (resolved from config) and the host token keep a reserve of 5% of the weekly window per day left and 20% of the 5-hour window; the dispatcher may run into it; system_status and a dashboard banner show it. - Usage: a token without user:profile (403) is read from the rate-limit headers as FFBox does; near-limit tokens are polled every 2 minutes; the SDK's rate_limit_event raises a meter between polls. - fffctl vault list and system_status show each pool; vault.sh loads claude-tokens/. Request: w738 Request: w739 Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 21 ++ README.md | 2 +- deploy/vm/host/vault.sh | 29 ++- deploy/vm/test/ci-vm-e2e.sh | 17 +- docs/accounts.md | 27 ++- docs/machines.md | 2 +- docs/ops-worker.md | 6 +- docs/vault.md | 161 +++++++++++--- server/agents.ts | 36 +++- server/appConfig.ts | 36 +++- server/config.ts | 22 +- server/index.ts | 71 +++++- server/opsWorker.test.ts | 65 ++++++ server/opsWorker.ts | 18 +- server/secrets.ts | 140 +++++++++++- server/tokenPool.test.ts | 188 ++++++++++++++++ server/tokenPool.ts | 237 ++++++++++++++++++++ server/usage.test.ts | 68 ++++++ server/usage.ts | 70 +++++- server/vault.test.ts | 96 ++++----- server/vault.ts | 166 +++++++++----- server/vaultCli.ts | 41 ++++ server/vaultPool.test.ts | 416 ++++++++++++++++++++++++++++++++++++ shared/types.ts | 11 + web/src/App.tsx | 8 +- 25 files changed, 1770 insertions(+), 184 deletions(-) create mode 100644 server/tokenPool.test.ts create mode 100644 server/tokenPool.ts create mode 100644 server/vaultPool.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index fee5b240..e272477a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,27 @@ Add your change under **[Unreleased]** in the same pull request. `npm run releas ## [Unreleased] +- **Each person's runs use that person's own pool of Claude tokens, and their orchestrator and the ops worker run on it too** (w738, + w739, Lothsahn). A person can have several tokens in the vault (host files `people//claude-token` and + `claude-tokens/`, loaded by `sudo fff-vm vault-sync`). The pick (`server/tokenPool.ts`): that person's tokens only (no shared + Claude token, no one else's), the one whose weekly window resets soonest first, the 5-hour reset as the tie-break; no new process + on a token at 80% of its 5-hour window, one process at a time at 95% weekly, retired at 99%, used up at 100; a session keeps its + token only while it passes. A worker, a standing agent and the ops worker are held with the reason and the next reset when the + whole pool is over its caps (no fallback to anyone else's token, the dispatcher's or the host token); a person's orchestrator + uses the pool up and then stops with a message. A person with no vault token keeps today's account. New: + `claudeAccounts.orchestrator = "vault"` (an owner's setting; the dispatcher stays on the token file; each orchestrator reads its + person's pool at every session start with every other credential removed, redacted everywhere); the ops worker starts a + fresh conversation and process for every new job and every change of caller, on that caller's pool; the pool limits are config + `vault.pool.*` (80 / 95 / 99, 5% a day, 20%), one set that anyone can change with `set_app_config` (checked 0-100, one at a time + not above retired, logged with who), read at each process start. The dispatcher's token (whatever `claudeAccounts.dispatcher` + names) and the host token keep a reserve of 5% of the weekly window per day left until its reset and 20% of the 5-hour window: + the dispatcher may always run into it; `system_status` and a dashboard warning show it. Usage numbers: a token without the + `user:profile` scope (every `claude setup-token` token, the dispatcher's too: HTTP 403) is read from the API's rate-limit + headers, as FFBox does (`scripts/claude_keys.py`); tokens near a limit are polled every 2 minutes; the SDK's rate-limit events + raise a meter between polls. `fffctl vault list` shows each pool; `system_status` shows each pool, its agents and the reserves; a + banner shows a pool used up or over its caps. Needs a portal deploy and, on the FFBox host, the new `vault.sh` (`install.sh + --guest-only` or `fff-vm` update). + - **`set_app_config` says that `machines.claudeFromVault` takes a machine** (w737, Lothsahn: "update LothDesktop to take machines.claudeFromVault true"). The setting was always per machine (`appConfig.ts` `perMachine`, `nextPerMachine`; `machine: "lothdesktop"` writes `{ "lothdesktop": true }` and leaves BEAST, m3, m5 and biscuit off), but the tool's text diff --git a/README.md b/README.md index feeb5fed..58bf496d 100644 --- a/README.md +++ b/README.md @@ -167,7 +167,7 @@ which is set per role and per machine ([docs/accounts.md](docs/accounts.md)): co puts the orchestrators, the dispatcher or this host's own daemon's workers on this host's login instead of the token, and `machines.useHostClaudeEnv` puts a Mac's agents on that Mac's login. The token vault ([docs/vault.md](docs/vault.md)) holds Claude tokens and the workers' other secrets on the portal and hands each -machine run what it may have, picking the Claude token by plan headroom. A login is shown by +machine run what it may have, picking the Claude token from the person's own pool by plan meters. A login is shown by its email and one login signed in on several computers is one account; a token by its last 4 characters, never its value. Each account lists where it is used and which agents run on it, and an agent's details name its account when there is more than one. The collapsed footer shows the diff --git a/deploy/vm/host/vault.sh b/deploy/vm/host/vault.sh index ef827d2a..32e20fb5 100644 --- a/deploy/vm/host/vault.sh +++ b/deploy/vm/host/vault.sh @@ -5,11 +5,16 @@ # lothsahn, 2026-10-06: each person's Claude and GitHub tokens are kept on this host, root only, next to what the # installer already keeps (decision D2: root here can read the VM anyway). The VM's vault gets them from here: # /etc/fff-vm/secrets/people//claude-token one 'claude setup-token' token (sk-ant-oat01-...) +# /etc/fff-vm/secrets/people//claude-tokens/ +# more of them, one token per file (w739: a person's pool; the +# is 1-16 of a-z 0-9 . _ - and starts with a letter or digit) # /etc/fff-vm/secrets/people//github-token one fine-grained GitHub token (github_pat_...) # /etc/fff-vm/secrets/vault.key the spare copy of the VM's vault key (kept in step with it) # Each value goes into the VM only over ssh's stdin, to `fffctl vault put`, and is never on a command line, in a log or in # the VM's daily backup (fff-backup leaves data/vault.json out; this host is where a rebuilt VM gets them back from). -# The vault entries it makes are named host--claude and host--github; one whose file is gone is removed. +# The vault entries it makes are named host--claude (claude-token), host--claude- (each file of +# claude-tokens/) and host--github; one whose file is gone is removed. All of a person's Claude entries are their pool: +# the portal gives their runs those tokens only, soonest weekly reset first (docs/vault.md, section 4). VAULT_PEOPLE=$FFF_VM_ETC/secrets/people VAULT_KEY_COPY=$FFF_VM_ETC/secrets/vault.key @@ -69,11 +74,27 @@ vault_sync() { person=$(basename "$dir") [[ "$person" =~ ^[A-Za-z0-9._-]{1,30}$ ]] || { warn "vault: $dir is not named for a portal user id; skipped"; rc=1; continue; } lower=$(tr '[:upper:]' '[:lower:]' <<<"$person") - for kind in claude github; do - file=$dir$kind-token + # The single files, then the pool's extra tokens (claude-tokens/): "kind|file|vault entry name" per token. + local -a todo=() + for kind in claude github; do todo+=("$kind|$dir$kind-token|host-$lower-$kind"); done + if [ -d "${dir}claude-tokens" ]; then + chmod 0700 "${dir}claude-tokens" + for file in "${dir}claude-tokens"/*; do + [ -f "$file" ] || continue + local slot + slot=$(tr '[:upper:]' '[:lower:]' <<<"$(basename "$file")") + [[ "$slot" =~ ^[a-z0-9][a-z0-9._-]{0,15}$ ]] || { warn "vault: $file is not named 1-16 of a-z 0-9 . _ - (starting with a letter or digit); skipped"; rc=1; continue; } + todo+=("claude|$file|host-$lower-claude-$slot") + done + fi + local item + for item in "${todo[@]}"; do + kind=${item%%|*} + file=${item#*|} + name=${file#*|} + file=${file%%|*} [ -s "$file" ] || continue chmod 0600 "$file" - name=host-$lower-$kind if ! vault_value_ok "$kind" "$file"; then warn "vault: $file is not $([ "$kind" = claude ] && echo "one 'claude setup-token' token (sk-ant-oat01-...)" || echo 'one fine-grained GitHub token (github_pat_...)'); skipped (its content is not shown)" rc=1 diff --git a/deploy/vm/test/ci-vm-e2e.sh b/deploy/vm/test/ci-vm-e2e.sh index 44d4027b..fe944052 100755 --- a/deploy/vm/test/ci-vm-e2e.sh +++ b/deploy/vm/test/ci-vm-e2e.sh @@ -248,17 +248,29 @@ echo "ok: the vault key is root's and loaded by the portal; values and credentia rnd() { local s; s=$(head -c 96 /dev/urandom | base64 -w0 | tr -dc 'A-Za-z0-9'); printf '%s' "${s:0:$1}"; } ct="sk-ant-oat01-$(rnd 44)" gt="github_pat_$(rnd 40)" -install -d -m 0700 /etc/fff-vm/secrets/people/ci /etc/fff-vm/secrets/people/ci2 +ct2="sk-ant-oat01-$(rnd 44)" +install -d -m 0700 /etc/fff-vm/secrets/people/ci /etc/fff-vm/secrets/people/ci2 /etc/fff-vm/secrets/people/ci/claude-tokens printf '%s\n' "$ct" | install -m 0600 /dev/stdin /etc/fff-vm/secrets/people/ci/claude-token +# A second Claude token for the same person (w739: the pool), and a file the pool refuses by its name. +printf '%s\n' "$ct2" | install -m 0600 /dev/stdin /etc/fff-vm/secrets/people/ci/claude-tokens/Second +printf '%s\n' "sk-ant-oat01-$(rnd 44)" | install -m 0600 /dev/stdin "/etc/fff-vm/secrets/people/ci/claude-tokens/bad name!" printf '%s\n' "$gt" | install -m 0600 /dev/stdin /etc/fff-vm/secrets/people/ci/github-token printf 'ghp_%s\n' "$(rnd 36)" | install -m 0600 /dev/stdin /etc/fff-vm/secrets/people/ci2/github-token out=$(/usr/local/sbin/fff-vm vault-sync 2>&1) || true echo "$out" -for v in "$ct" "$gt"; do if printf '%s' "$out" | matches -F "${v:13:24}"; then fail "fff-vm vault-sync printed a token"; fi; done +for v in "$ct" "$gt" "$ct2"; do if printf '%s' "$out" | matches -F "${v:13:24}"; then fail "fff-vm vault-sync printed a token"; fi; done +printf '%s' "$out" | matches 'not named 1-16 of a-z' || fail "a pool file with a bad name was not refused: $out" printf '%s' "$out" | matches 'not one fine-grained GitHub token' || fail "a classic GitHub token was not refused" list=$(g 'sudo fffctl vault list') printf '%s' "$list" | matches -F "${ct: -4}" || fail "host-ci-claude is not in the vault: $list" printf '%s' "$list" | matches '^host-ci-github ' || fail "host-ci-github is not in the vault: $list" +# The pool: both Claude tokens are the person's (owner ci), and the CLI lists the pool with their state and last four characters. +printf '%s' "$list" | matches '^host-ci-claude-second ' || fail "the second token of the pool is not in the vault: $list" +printf '%s' "$list" | matches -F "${ct2: -4}" || fail "the second token's last four characters are not listed: $list" +printf '%s' "$list" | matches 'Claude token pools' || fail "fffctl vault list has no pool section: $list" +printf '%s' "$list" | matches -E '^ host-ci-claude-second +…' || fail "the pool section does not list the second token: $list" +printf '%s' "$list" | matches -E 'ok +5-hour' || fail "the pool section shows no state and meters: $list" +if printf '%s' "$list" | matches 'bad'; then fail "the refused pool file went in"; fi if printf '%s' "$list" | matches '^host-ci2-'; then fail "the refused token went in"; fi [ "$(stat -c '%a %U' /etc/fff-vm/secrets/vault.key)" = "600 root" ] || fail "the key's spare copy is not 0600 root" # The file is named to sha256sum under sudo: a "<" redirect would be opened by the admin's own shell, which cannot read it. @@ -267,6 +279,7 @@ if printf '%s' "$list" | matches '^host-ci2-'; then fail "the refused token went out=$(/usr/local/sbin/fff-vm vault-sync 2>&1) || true printf '%s' "$out" | matches 'unchanged host-ci-claude' || fail "a second sync was not a no-op: $out" printf '%s' "$out" | matches 'unchanged host-ci-github' || fail "a second sync was not a no-op: $out" +printf '%s' "$out" | matches 'unchanged host-ci-claude-second' || fail "a second sync was not a no-op for the pool's second token: $out" rm -rf /etc/fff-vm/secrets/people/ci /etc/fff-vm/secrets/people/ci2 /usr/local/sbin/fff-vm vault-sync >/dev/null 2>&1 || true if g 'sudo fffctl vault list --names' | matches '^host-ci-'; then fail "a removed person's entries stayed in the vault"; fi diff --git a/docs/accounts.md b/docs/accounts.md index ef8ce213..1404e03e 100644 --- a/docs/accounts.md +++ b/docs/accounts.md @@ -14,7 +14,7 @@ Every agent the portal starts runs on one of three kinds of Claude credential: | Agents | Config | Values | Default | |---|---|---|---| -| The orchestrator | `claudeAccounts.orchestrator` | `"token"` or `"login"` (this host's) | `"token"` | +| The orchestrator (and the ops worker working for a person) | `claudeAccounts.orchestrator` | `"token"`, `"login"` (this host's), `"tokenfile"` or `"vault"` (each person's own pool, an owner's setting; w738) | `"token"` | | The dispatcher (w464) | `claudeAccounts.dispatcher` | same; once set it also overrides the system payer's own token in `userClaudeEnv` | unset: the system payer's own token if any, else `claudeAccounts.orchestrator`'s | | Workers and standing agents on a Mac | `machines.useHostClaudeEnv` | `true` (host token) or `false` (the Mac's login); global, or per machine | `true` | | Workers and standing agents on this host's own daemon ([beast-machine.md](beast-machine.md)) | `claudeAccounts.workers`, unless `machines.useHostClaudeEnv` names the machine | `"token"` or `"login"`; "login" is this host's login, with the rest of `claudeEnv` (e.g. `CLAUDE_CONFIG_DIR`) kept | `"token"` | @@ -68,12 +68,31 @@ the access token and the refresh token have both expired (`hostLoginProblem`, `s without the `user:profile` scope is accepted: agents need only `user:inference`, and only the usage meters need `user:profile`. On macOS a missing file proves nothing, because the login is in the Keychain. -### The token vault (w512) +### The token vault (w512) and the pools (w738, w739) + +`claudeAccounts.orchestrator` also takes `"vault"` (an owner's switch in `set_app_config`): a person's orchestrator +starts with `CLAUDE_CODE_OAUTH_TOKEN` set to a token of **that person's pool** (their Claude entries in the vault, +`people//claude-token` and `claude-tokens/` on the FFBox host), read at each session start so a new token +applies to the next session with no restart, and every other Claude credential removed first, as for the token file +(`poolRunEnv`, `server/secrets.ts`). Which of their tokens, and what happens when they are over their caps or used up: +[vault.md](vault.md) section 4. The orchestration worker works the same way, on the pool of the person whose orchestrator +gave it the job, and starts a fresh conversation and process for every new job and every change of caller +([ops-worker.md](ops-worker.md)). **The dispatcher never takes `"vault"`:** it keeps the shared token file (a dispatcher +that inherits `"vault"` from the orchestrator's setting is on the token file, else the host token), and that +credential and the host token keep a reserve for their own roles. A person with no vault token yet falls back to the +token file, as before. `system_status` names the account per role ("each person's own vault pool (token file …dAAA for a +person with no vault token)"), each person's pool with each token's state, and the reserves. The pool limits are config +`vault.pool.*`, one set for everyone, which any person's orchestrator may change: `sessionHoldPercent` 80, +`onePerWeeklyPercent` 95, `retireWeeklyPercent` 99, `reservePerDayPercent` 5, `reserveSessionPercent` 20. + +The meters: a session on a pool token shows on that token's account (a live session by the account its process started +on; a stopped orchestrator by the last token it was handed). + +#### Machine runs Machine runs (workers and standing agents on a machine) can take their Claude token from the portal's token vault instead: config `machines.claudeFromVault` (`true`, `false`, or per machine with `"*"`; default `false`; owner-only in -`set_app_config`, with `machine: ""` for one machine alone) names the machines. Each run then gets one vault token, chosen by plan headroom with the run's -person's own token first, as `CLAUDE_CODE_OAUTH_TOKEN` alone (`LaunchSpec.login` drops the daemon's own credentials). A +`set_app_config`, with `machine: ""` for one machine alone) names the machines. Each run then gets one vault token from its person's pool (docs/vault.md section 4), as `CLAUDE_CODE_OAUTH_TOKEN` alone (`LaunchSpec.login` drops the daemon's own credentials). A person's own token in `userClaudeEnv` still wins for their work. With no eligible vault token the run falls back to what this page describes. The vault, its other secrets and the cut-over: [vault.md](vault.md). diff --git a/docs/machines.md b/docs/machines.md index a84d6810..673e724c 100644 --- a/docs/machines.md +++ b/docs/machines.md @@ -85,7 +85,7 @@ before redeploying by hand. deletes them. Force pushes and pushes to the game repo's master/main stay refused. The daemon's own folder (`~/.ff-factory`, or the machine's `app_dir`; it holds the token) is protected. - **Claude account.** With the token vault on for a machine (config `machines.claudeFromVault`), its runs get a vault - token chosen per run by plan headroom, and the other secrets granted to that machine, in the same launch spec + token from the run person's pool (soonest weekly reset first, within the pool's caps), and the other secrets granted to that machine, in the same launch spec ([vault.md](vault.md)). Otherwise: portal-run agents on a Mac (workers and standing agents) use this host's `claudeEnv`, so with `CLAUDE_CODE_OAUTH_TOKEN` set (`set_app_config`, write-only) they run on the same Claude account as the agents here, not on the Mac's own login. The portal puts it in the launch spec, diff --git a/docs/ops-worker.md b/docs/ops-worker.md index c059c2ec..6b04828d 100644 --- a/docs/ops-worker.md +++ b/docs/ops-worker.md @@ -24,7 +24,7 @@ A person's own orchestrator (Lothsahn's or Ben's) has the `ops_worker` tool: | action | what it does | |---|---| -| `send` (`text`, `fresh`) | gives it a job or a follow-up. `fresh: true` starts a new conversation. A **new job** needs a turn the person started with a message of their own. Within that job (12 hours), the orchestrator's harness turns (a check-in, a timer, its `[ops worker]` report) may follow up, with no count (w627) | +| `send` (`text`, `fresh`) | gives it a job or a follow-up. **Every new job starts a fresh conversation and a fresh process** (w738): `fresh: true` asks for it explicitly, and a new job is also the first message after the last job ended (12 hours), a message from the other person's orchestrator, and a deploy. Nothing, no context and no token, carries over from one job to the next. A **new job** needs a turn the person started with a message of their own. Within that job (12 hours), the orchestrator's harness turns (a check-in, a timer, its `[ops worker]` report) may follow up, with no count (w627) | | `deploy` (`text`: the person's words) | a portal deploy (`fffctl update`). Only in a turn the person started with their own message: never a check-in, a timer, a relayed report or a job's follow-up. See [Deploys](#deploys) | | `status` | its state, the job and whose it is, its limits, and its last 20 steps | | `interrupt` | ends its turn | @@ -224,7 +224,7 @@ Fixed in `OPS_LIMITS` (`server/opsWorker.ts`), like the worker itself: | Setting | Value | Basis | |---|---|---| | model, effort | `opus`, `medium` | sourced: the game repo's CLAUDE.md drops the driver to medium "for purely operational sessions" | -| account | the orchestrators' (`claudeAccounts.orchestrator`): in the VM, the subscription token file | sourced: D4 / design 5.2 put the orchestrators on it, and the worker works only for their people. The account must be a token: the VM's claude.ai login belongs to `fff`, which `fff-ops` cannot read, so the launcher refuses with "no Claude credential" | +| account | the orchestrators' (`claudeAccounts.orchestrator`): in the VM, the subscription token file, or with `"vault"` (w738) **the Claude token pool of the person whose orchestrator gave it the job**, picked at the start of every job by the pool rules in docs/vault.md section 4: the caps hold it ("held" with the reason and the next reset, like that person's other workers), and it never falls to the dispatcher's token. A caller with no vault token yet keeps the token file. A new job from the other person's orchestrator starts a fresh process on that person's token. | sourced: D4 / design 5.2 put the orchestrators on it, and the worker works only for their people. The account must be a token: the VM's claude.ai login belongs to `fff`, which `fff-ops` cannot read, so the launcher refuses with "no Claude credential" | | spend cap | $25 per process (the SDK's `maxBudgetUsd`, which the CLI enforces) | a guess: a runaway guard. On the subscription token, the cost is plan usage, and the dollar figure is the SDK's estimate | | idle stop | 1 hour, then the process stops and the conversation stays | sourced: the same hour idle workers get (`IDLE_REAP_MS`), the prompt cache's lifetime | | turn limit | 2 hours, then FF Factory interrupts the turn | a guess: an install with its waits fits, and longer waits use `wake_me` | @@ -233,7 +233,7 @@ Fixed in `OPS_LIMITS` (`server/opsWorker.ts`), like the worker itself: | memory | 1536 MB (`MemoryMax`) | measured basis: an idle claude process used 100-300 MB resident and 450-650 MB committed (BEAST, 2026-10-04, orchestrators.md). The VM has 4 GiB | The session record stays for good, with its conversation. `fresh: true` starts a new conversation: the transcript -goes on, with a line marking the new job and whose it is. +goes on, with a line marking the new job and whose it is. Every new job does this (w738), not only `fresh: true`; a follow-up within the job keeps the conversation and the process, and with them the token it started on. ## Audit diff --git a/docs/vault.md b/docs/vault.md index b9f47042..39c06e38 100644 --- a/docs/vault.md +++ b/docs/vault.md @@ -135,27 +135,100 @@ expire. When the portal is back, the next start gets current values, so a rotate ## 4. Which Claude token a run gets -`pickClaude` (`server/vault.ts`) chooses among the `claude` entries: - -1. **Eligible:** enabled, granted to the run's role and machine, and either shared or owned by the person the run is - for. A run nobody asked for counts as the system payer's (config `systemPayer`, else the owner). -2. **The person's own first:** an entry the run's person owns ranks above the shared ones. -3. **Room left:** an entry whose 5-hour or weekly meter is at 95% or more ranks below every entry with room. Among the - rest, the most headroom first, `100 - max(session %, weekly %)`, from the numbers the usage meters already poll for - each vault token (`UsageTracker`, every `usagePollMinutes`). An entry with no numbers yet counts as 50%. -4. **Spread:** then the fewest live sessions on that token, then the name. -5. **Sticky per session:** a session keeps its token while it stays eligible and under 95%, so a resumed process does - not hop accounts. The choice is made at each process start, never mid-process. - -One token can serve several machines at once. The meters show its 5-hour and weekly use across all of them, because -those limits are per account (section 7). - -**Order with the existing switches.** A person's own token in config `userClaudeEnv` still wins for their work, as -before. Next comes the vault, on machines `machines.claudeFromVault` names (owner-only in `set_app_config`). With no -eligible entry, or no key, the run falls back to what the machine had without the vault, the host token or its own login -(`machines.useHostClaudeEnv`), and `system_status` shows a warning for 24 hours. A vault token is given with -`LaunchSpec.login` set, so the daemon drops its own environment's credentials (an `ANTHROPIC_API_KEY` there would -outrank the token) and the token is the process's only one. +A person's runs use **that person's own tokens, their pool, and nobody else's** (lothsahn, 2026-10-08: "each operator is +only going to use its own pool going forward--except for the dispatcher"). The pick is `pickPool` (`server/tokenPool.ts`, +pure functions), called from `Vault.forRun` (`server/vault.ts`) at every process start, resumes included. + +**Whose pool.** The run's person: its requester, else the system payer (config `systemPayer`, else the owner) for work +nobody asked for by name; `vault.unattributed` (section 10) names whose pool intake, FFBox and nightly work use. A Claude +entry with no owner (`share: anyone`, added by hand before the pool) is in nobody's pool and is never handed out: +`fffctl vault list` names it as "not in any pool". A shared Claude token was **retired** from the pick; GitHub and env +entries are unchanged. A worker or standing agent also needs the entry granted to its role and machine (`roles`, +`machines`); the ops worker takes the workers' grants on no machine; a person's orchestrator takes no grant (the +account setting is its consent). + +**The order.** The token whose **weekly** window resets soonest first (it is the capacity that would otherwise be lost +at the reset), the 5-hour reset as the tie-break, then the name. A token with no reset time yet goes last. The 5-hour +window refills every five hours whatever is done, so ranking by it would save nothing; weekly is the default key. +The meters are the ones `UsageTracker` polls (section 4a). + +**The limits** (one system-wide set, config `vault.pool.*`, percent 0 to 100, defaults below; **anyone** may change them +with `set_app_config`, not only an owner; the one-at-a-time limit may not be above the retire limit; each change is +logged in the portal journal with who made it; they apply at the next process start, no restart): + +| Key | Default | A token at or past it | +|---|---|---| +| `vault.pool.sessionHoldPercent` | 80 | its **5-hour** meter: no new process on it until that window resets | +| `vault.pool.onePerWeeklyPercent` | 95 | its **weekly** meter: **one** live process at a time | +| `vault.pool.retireWeeklyPercent` | 99 | its weekly meter: **retired**, handed to nobody until the weekly window resets | +| (at 100) | | either meter: **used up**, handed to nobody until it resets (a rejection by the API counts as 100) | + +A token with no numbers yet counts as 50% on both meters, so it passes every rule and ranks last. "Live" counts the +sessions whose process started on the token, not the asking session itself, and a pick made in the last 20 seconds before +its process shows (two starts cannot both take the last slot). + +**Sticky per session.** A session keeps its token only while the token still passes these checks; otherwise it moves to +the next. The choice is made at each process start, never mid-process. + +**When the pool has no token that may serve the run** (every one held, one-at-a-time with a process running, retired or +used up): + +| Role | What happens | +|---|---| +| A worker, a standing agent, the ops worker for them | **Held** with the reason ("lothsahn's Claude token pool is held: 2 tokens: all over their caps or used up, the first frees up Fri 14:00Z"): a worker's first prompt queues and is tried again, a standing agent's run ends "Could not start: " and the next scheduled run tries again; no fallback to the host token, the machine's login, the dispatcher's token or anyone else's | +| A person's own **orchestrator** (`claudeAccounts.orchestrator = "vault"`) | **Uses the pool up**: while some token is only past its caps (held, one-at-a-time, retired) but not used up, it runs on the one with the most room (the soonest reset on a tie), and the one-at-a-time rule does not bind it. Once every token is used up it **stops** with the reason and the next reset on the dashboard; there is no override and no fallback | +| The **dispatcher** | Never on a pool: it keeps the shared token file (or whatever `claudeAccounts.dispatcher` names) and is never held | + +**The transition.** A person with **no** Claude token in the vault keeps today's account, so nothing stops at deploy: +the machine's login or the host token for their workers (`machines.useHostClaudeEnv`), the token file for their +orchestrator and for the ops worker working for them. `system_status` warns ("no vault Claude token for a workers run +on m3 for sam; it runs on host token …abcd instead") for 24 hours. The same fallback applies when the vault cannot open +their tokens (no key, a wrong key): nothing the person can wait for. The moment a person has one token, only their pool +is used. + +**The dispatcher's reserve.** The credential the dispatcher runs on (`claudeAccounts.dispatcher`, resolved from config +at each look: the token file today, else the host token or this host's login) and the host token +(`claudeEnv.CLAUDE_CODE_OAUTH_TOKEN`, which biscuit's agents and any person without a pool run on) keep a buffer for +their own roles: **5% of the weekly window for each day left until that window resets** (35% with seven days to go, 5% +with one, 0 at the reset; `vault.pool.reservePerDayPercent`) and **20% of the 5-hour window** +(`vault.pool.reserveSessionPercent`). The reserve is the dispatcher's: it may always run into it, up to 100%, and +nothing checks it before the dispatcher starts. Nothing else is *refused* by it, because after the pool change nothing +else lands on these credentials except a person with no vault token yet (their orchestrator, the ops worker for them), +who has no other account; `system_status` shows each reserve ("the dispatcher's token file …dAAA: weekly 41%, reserve +25% (5 days to reset); 5-hour 12%, reserve 20%") and, while a meter is inside the buffer with such runs on the credential, +the dashboard shows a warning. Until a credential's first reading arrives it counts as inside its reserve. One token is +one account: two tokens of one Claude account are not matched, so such a token would have its own meters and its own +reserve. + +**Order with the existing switches.** A person's own token in config `userClaudeEnv` still wins for their work on a +machine, as before. Then the vault, on machines `machines.claudeFromVault` names (owner-only in `set_app_config`; +per machine with `machine: ""`, section 6). A vault token is given with `LaunchSpec.login` set, so the daemon +drops its own environment's credentials (an `ANTHROPIC_API_KEY` there would outrank the token) and the token is the +process's only one. A person's orchestrator on `"vault"` and the ops worker work the same way, in the portal: the one +token as `CLAUDE_CODE_OAUTH_TOKEN`, every other Claude credential removed first. + +### 4a. Where the numbers come from, and how stale they are + +- **The usage endpoint** (`/api/oauth/usage`) needs the `user:profile` scope. A token from `claude setup-token` has only + `user:inference` and answers **403** *(sourced: community reports of the same 403, e.g. claude-code issues 22450, + 24200; measured on the dispatcher's token file by lothsahn, 2026-10-09)*. Re-issuing cannot add the scope: it is fixed + in the CLI's authorization URL, and an interactive `/login` token that has it rotates its refresh token, so it does not + suit unattended use. +- **The rate-limit headers** of any `/v1/messages` reply (`anthropic-ratelimit-unified-5h-*`, `-7d-*`: utilization as a + fraction, reset as epoch seconds) carry the same two windows and need only `user:inference`. FFBox reads usage this + way for its setup-tokens (*sourced:* `scripts/claude_keys.py` in Final-Factory/ffbox, commit 57c35a5b4b, which says + the 403 was "measured 2026-09-04" and the headers answer; it takes them from a one-token Haiku call and from a 429 + reply too). `fetchTokenLimits` does the same here. It already ran when the endpoint answered 429; it now also runs on + **403**, and the endpoint is not asked again for that token for six hours. It costs one Haiku request with one output + token. Not measured here with an inference-only token: the first poll after the deploy logs `via the API's rate-limit + headers` for it. +- **The SDK's `rate_limit_event`** (from the same headers on a running session's own requests) raises a token's meter + between polls and never lowers it (a guess at its frequency and scale; it is a floor, the next poll replaces it). +- **How stale:** the regular poll is every `usagePollMinutes` (15). A token near a limit (5-hour at 60% or more, weekly + at 90% or more, or with no numbers) is also polled alone every **2 minutes**. How far a token can overshoot a limit in + an interval depends on the work on it and was **not measured** (no poll history existed to measure from): every poll + logs the percentages (`usage: … Weekly 41%, Session 33%`), so a week of the journal gives the largest rise per interval. + Between polls a pick can be a few percent stale; the caps leave 20% of the 5-hour window and 1% of the week for it. ## 5. Security review @@ -253,9 +326,12 @@ work is the risk; lothsahn decided each person's tokens serve only their own wor ## 9. Follow-ups -- The orchestrators and the dispatcher on vault tokens (a `"vault"` value for `claudeAccounts`), retiring - `claudeTokenFile`. The workers do not need it; it would put D4's token in the vault too. -- The meters attribute a stopped vault session by the current config rather than the token it ran on, as for every +- **Built (w738, w739):** the orchestrators on vault tokens, a `"vault"` value for `claudeAccounts.orchestrator`: each + person's orchestrator, and the ops worker working for them, runs on that person's pool (section 4); the token file stays + the dispatcher's and the fallback for a person with no vault token. Retiring `claudeTokenFile` entirely would put the + dispatcher's token in the vault too: not done. +- The meters attribute a stopped vault session of an orchestrator by the token it ran on last (`Vault.lastPickOf`, + this portal run); a worker's stopped session by the current config rather than the token it ran on, as for every account (`sessionSource`); live sessions are exact. - A dedicated OS user for the portal's own agents would make the key boundary real rather than a seatbelt (section 5). - The seal authenticates a value with its entry's id and kind, not its grants: someone who can write `data/` (the @@ -295,13 +371,18 @@ the vault gets nobody else's: it runs as it did before the vault (the machine's ```text /etc/fff-vm/secrets/ 0700 root people//claude-token 0600: one 'claude setup-token' token (sk-ant-oat01-...) + people//claude-tokens/ 0600: more of them, one token per file: the person's pool (w739); 0700 folder people//github-token 0600: one fine-grained GitHub token (github_pat_...) vault.key 0600: the spare copy of the VM's vault key ``` +`claude-token` stays and is a pool of one. A `` is 1 to 16 of `a-z 0-9 . _ -`, starting with a letter or digit +(upper case is read as lower). Each file is an entry `host--claude-`; all of a person's Claude entries are +their pool (section 4). + **How the VM gets them: pushed in, not shared.** `sudo fff-vm vault-sync` (`deploy/vm/host/vault.sh`) sends each file over ssh's stdin to `fffctl vault put` in the VM, as the installer already sends the Claude and GitHub tokens -(`push_secret`, `deploy/vm/host/guest.sh`). The entries are `host--claude` and `host--github`, `share: +(`push_secret`, `deploy/vm/host/guest.sh`). The entries are `host--claude`, `host--claude-` and `host--github`, `share: owner`, for workers and standing agents on every machine. A second run changes nothing; a new value rotates its entry; a removed file removes its entry. Entries not named `host-…` (added by hand) are left alone. The installer's guest step runs it too, so a re-run or a rebuilt VM is filled again. @@ -365,10 +446,37 @@ Ben is not on the FFBox host: lothsahn runs the same commands there, and Ben typ (the two can share one ssh session to the host, for instance with `tmux`), or Ben hands the two values to lothsahn through a channel that is not chat (in person, or a password manager's share). +**2a. More than one Claude token for a person (w739):** the same, one file per token, in the pool folder (each its own +`claude setup-token` from a different Claude account, or the same account's second login): + +```bash +sudo install -d -m 0700 /etc/fff-vm/secrets/people//claude-tokens +sudo sh -c 'umask 077; read -rs -p "Claude token (second): " t && printf "%s\n" "$t" > /etc/fff-vm/secrets/people//claude-tokens/second'; echo +sudo fff-vm vault-sync # "added host--claude-second"; values never shown +sudo fff-vm ssh 'sudo fffctl vault list' # the pool section: both tokens, their state, meters and resets +``` + +From then on only the person's pool serves their runs (section 4). Someone with no token in the vault is unchanged +until they add one. + **3. Check:** `sudo fff-vm ssh 'sudo fffctl vault list'` shows `host-lothsahn-claude`, `host-lothsahn-github`, `host-ben-claude`, `host-ben-github`, each with its last four characters; within a minute the usage meters show "vault: host--claude …abcd" accounts. +**3a. The pool works** (after the deploy and a `vault-sync`): `sudo fff-vm ssh 'sudo fffctl vault list'` ends with a "Claude +token pools" section, one line per token: its last four characters, its state (`ok`, `held`, `one-at-a-time`, `retired`, +`exhausted`), its 5-hour and weekly percent with their resets, and how many agents ran on it at the portal's last save. +`system_status` has a "Claude token pool, " line per person with the same, the agents running on each token, and +a verdict, plus a line per reserved credential ("the dispatcher's token file …dAAA: weekly 41%, reserve 25% (5 days to +reset)…"). A token whose line says "no numbers yet" has not been polled: wait for the next poll (2 to 15 minutes). + +**3b. The orchestrators on their own pool** (lothsahn; owner-only): once a person's tokens are in, `set_app_config +claudeAccounts.orchestrator "vault"`. It applies to the next session start of each orchestrator (a running one keeps its +token until its process restarts, so restart them from the dashboard or wait for the next idle stop), and the ops worker +picks the pool of whichever person's orchestrator gave it the job, at the start of every job. `system_status`'s "Claude +account per agent" line then says "each person's own vault pool", and each pool line names the orchestrator's token. +Rollback: `set_app_config claudeAccounts.orchestrator "tokenfile"`. + **4. Max's Discord token, for the posting machine only** (in the VM; unchanged): `sudo fffctl vault add --kind env --env FFDISCORD_APP_TOKEN --name max-discord --share anyone --machines lothdesktop --file /tmp/d && shred -u /tmp/d`. @@ -383,7 +491,8 @@ token; one for Ben's request shows Ben's. **Rotating a token later:** replace its file (step 2), then `sudo fff-vm vault-sync`. **Removing a person's:** delete their files, then `sudo fff-vm vault-sync`; revoke the tokens where they were made. -**Rollback, any time:** `set_app_config machines.claudeFromVault false machine: ""`. A machine to cut off +**Rollback, any time:** `set_app_config machines.claudeFromVault false machine: ""` (the workers), `claudeAccounts.orchestrator +"tokenfile"` (the orchestrators and the ops worker). A machine to cut off entirely: `sudo fff-vm ssh 'sudo fffctl machine-credential revoke '`. ## Runbook diff --git a/server/agents.ts b/server/agents.ts index 21b3cfd9..06780064 100644 --- a/server/agents.ts +++ b/server/agents.ts @@ -36,7 +36,7 @@ import { WORK_BLOCKER_KINDS, WORK_OPEN, WORK_PRIORITIES, type WorkBlockerKind, t import { attachmentForMachine, publicRef, publishableFile, uploadForMachine, type AttachmentStore } from './attachments.ts'; import { REVIEW_DEFAULTS, type ReviewStore } from './review.ts'; import { INBOX_DIR, MAX_ATTACHMENTS, attachmentLine, fmtBytes, publishedAttachmentText } from '../shared/attachments.ts'; -import { accountSource, dispatcherOwnAccount, hostAccount, hostProcessEnv, machineRunEnv } from './secrets.ts'; +import { SECRET_KEYS, accountSource, dispatcherOwnAccount, hostAccount, hostProcessEnv, machineRunEnv, maskSecret, poolRunEnv } from './secrets.ts'; import { Identity, claudeEnvFor, forLine } from './identity.ts'; import { crossOwnerLine, loopGuards, Orchestrators, OWNER_LOOP_MESSAGES_PER_HOUR, PERSON_MESSAGE_CHARS } from './orchestrators.ts'; import { beltFor, type BeltRole } from './belts.ts'; @@ -1964,7 +1964,7 @@ Stills, clips and notes for a review (the visual checklist, a playtest, a before ), tool( 'set_app_config', - `Change one cosmetic setting of this app in its config.json (the old file is kept as config.json.prev). It applies at once and survives restarts. Allowed keys only: ${SETTABLE_KEYS.join(', ')}. ownerName: the user's name, which agents' prompts then use (new sessions); voice.vocabulary: extra words the speech-to-text should spell right (a list, or one comma-separated string); voice.ttsVoice: the default Kokoro voice ("af_heart", "bm_george", …); publicGitIdentity.name / .email: the identity agents commit with in public repos such as this app's own (the guard refuses pushes there with other emails; GitHub noreply addresses are always fine); hostGuard.devDriveVhdx: the sandbox Dev Drive's .vhdx path; publicUrl: the portal's base URL that machines and the outside watchdog reach it at (the Tailscale Funnel URL); claudeEnv.CLAUDE_CODE_OAUTH_TOKEN: the Claude account's OAuth token the agents run on (sk-ant-oat01-…, from "claude setup-token"), write-only: it is never shown back, only "set (…last 4)", and redacted from transcripts; userClaudeEnv.CLAUDE_CODE_OAUTH_TOKEN (with user: a user id): that person's own Claude token, which agents working for them run on instead (same rules; only when that person asked for it); claudeAccounts.orchestrator / .workers: which Claude account this host's orchestrator (you) and the workers of this host's own daemon run on: "token" (claudeEnv's token, the default) or "login" (the claude.ai login stored on this host; refused when none is stored or it has expired); a person's own token still wins for their work; machines.useHostClaudeEnv (optionally with machine: a machine id): true (default) runs that Mac's agents (workers and standing agents there) on this host's token, false on the Mac's own login; without machine it sets every machine not named; machines.claudeFromVault (an owner's only; optionally with machine: a machine id, e.g. "lothdesktop"): true runs that machine's agents on a Claude token from the token vault (docs/vault.md), false on the host token or the machine's own login as before; with machine it changes that machine alone and leaves the others as they were, without it every machine not named; a machine whose vault cannot give a token runs on its host token or own login, it is not refused; systemPayer: the user id automatic work (scheduled standing runs, intake-triggered FFBox work) is attributed and billed to (default the owner); providers.ffbox.enabled: true lets FFBox's connector connect (read-only reports: capacity, conversations, intake), false drops it at once (default false); providers.ffbox.token: FFBox's connector token (ffpv1_…), write-only, stored only as its SHA-256; providers.ffbox.devRequests (an owner's only, the whole block): { enabled (default true), perHour (0-1000, default 20), maxFiles (0-10, default 10), maxRequestMB (1-500, default 500) }, unknown keys refused; placement.prefer: the computers new game-repo work goes to first, in order (machine ids such as "lothdesktop", "m5", "beast"; a list or one comma-separated string; null clears), and placement.avoid: computers kept off unless nothing else has room, each with why ({ "beast": "BEAST unstable, 2026-10-05" }; null clears) (docs/machines.md, "Placing work"); attachments.maxMB: the largest file a person may attach to a message (1-4096 MB, default 200); attachments.retentionDays: how many days an attached file nobody sent on is kept (1-3650, default 30) (docs/attachments.md); orchestrator.compactAtTokens: an orchestrator (the dispatcher too) compacts its conversation by itself between turns once its context reaches this many tokens (0 = off, else 50,000-900,000, default 200,000); orchestrator.compactAtTurnUsd: or once a turn cost at least this many USD with the context at 100,000 tokens or more (0 = off, else 0.05-50, default 1) (docs/orchestrators.md, "Compacting a conversation"); orchestrator.messagesPerPerson: message_person messages a person's orchestrator may send one person until either of the two writes to their own orchestrator (1-100, default 10; Lothsahn's and Ben's orchestrators have no count between them, w627); orchestrator.filingsPerMessage / orchestrator.followUpsPerMessage: filings, and follow-ups to one worker, between two messages of its person (1-100, default 3 each) (docs/orchestrators.md, "Loops, limits and safety"); hostGuard.cleanup.ageRules: JSON list of { "path", "olderThanDays" (>= 3) } whose old entries each clean-up pass removes (never a drive root, the home folder, the sandboxes, this app or a protected path); hostGuard.cleanup.everyMinutes: how often this host's clean-up runs (0 = only below the soft threshold, else 15-1440, default 60); hostGuard.cleanup.softFreeGB: below this much free space it runs every 15 minutes with the cache-emptying rules, and tells you when it cannot get back above (default warnFreeGB + 40 = 120; must be above warnFreeGB); machines.cleanup.everyMinutes / machines.cleanup.softFreeGB (optionally with machine): the same for the machines' daemons (defaults 60 and 80 GB). usagePollMinutes: how often every Claude account's plan usage is polled, here and by the machines' daemons (5-240, default 15; the usage endpoint rate-limits). intake.ffbox (an owner's only: refused unless the person who asked is an owner): the FFBox intake's whole block, replaced, so a key left out takes its default (docs/intake.md): an object with enabled (default false), branches, diagnoses, requests (default true once enabled), boardCheck (answer FFBox's ledger check), escalations (both default false), repo ("owner/name" as board answers name the game repo; default from repo.url), dailyCap (0-200, default 10), match { high (default 0.7), medium (default 0.45) } (board_check's match bands, 0-1), autoApprove { enabled (default false), maxPerDay (0-100, default 3) } and desync { enabled (default true), maxPerDay (0-100, default 10) } (Lothsahn's FFBox desync PR policy: such a PR is approved at once into a review-and-merge request under the policy); unknown keys are refused; it applies at once, and FFBox's connection stays up (a board_check or request while it is off is answered not_enabled). value null removes the key (back to the default). intake.reviewers (an owner's only, like intake.ffbox): who may approve or decline intake requests and answer design questions: a list of user ids (or one comma-separated string), each a login that exists (e.g. ["ben", "lothsahn"]; unknown ids are refused and nothing is written); the whole list is replaced, and null removes it (then only the owner decides); it applies at once (docs/intake.md). Only when the user asked for the change.`, + `Change one cosmetic setting of this app in its config.json (the old file is kept as config.json.prev). It applies at once and survives restarts. Allowed keys only: ${SETTABLE_KEYS.join(', ')}. ownerName: the user's name, which agents' prompts then use (new sessions); voice.vocabulary: extra words the speech-to-text should spell right (a list, or one comma-separated string); voice.ttsVoice: the default Kokoro voice ("af_heart", "bm_george", …); publicGitIdentity.name / .email: the identity agents commit with in public repos such as this app's own (the guard refuses pushes there with other emails; GitHub noreply addresses are always fine); hostGuard.devDriveVhdx: the sandbox Dev Drive's .vhdx path; publicUrl: the portal's base URL that machines and the outside watchdog reach it at (the Tailscale Funnel URL); claudeEnv.CLAUDE_CODE_OAUTH_TOKEN: the Claude account's OAuth token the agents run on (sk-ant-oat01-…, from "claude setup-token"), write-only: it is never shown back, only "set (…last 4)", and redacted from transcripts; userClaudeEnv.CLAUDE_CODE_OAUTH_TOKEN (with user: a user id): that person's own Claude token, which agents working for them run on instead (same rules; only when that person asked for it); claudeAccounts.orchestrator / .workers: which Claude account this host's orchestrator (you) and the workers of this host's own daemon run on: "token" (claudeEnv's token, the default) or "login" (the claude.ai login stored on this host; refused when none is stored or it has expired); a person's own token still wins for their work; claudeAccounts.orchestrator also takes "vault" (an owner's only): each person's orchestrator runs on that person's own Claude token pool from the vault (docs/vault.md), falling back to the token file for a person with none; vault.pool.sessionHoldPercent (80) / onePerWeeklyPercent (95) / retireWeeklyPercent (99) / reservePerDayPercent (5) / reserveSessionPercent (20) (anyone may set these; one system-wide set, percent 0-100, the one-at-a-time limit not above the retire limit): the Claude token pool's limits, read at each process start (docs/vault.md, section 4); machines.useHostClaudeEnv (optionally with machine: a machine id): true (default) runs that Mac's agents (workers and standing agents there) on this host's token, false on the Mac's own login; without machine it sets every machine not named; machines.claudeFromVault (an owner's only; optionally with machine: a machine id, e.g. "lothdesktop"): true runs that machine's agents on a Claude token from the token vault (docs/vault.md), false on the host token or the machine's own login as before; with machine it changes that machine alone and leaves the others as they were, without it every machine not named; a machine whose vault cannot give a token runs on its host token or own login, it is not refused; systemPayer: the user id automatic work (scheduled standing runs, intake-triggered FFBox work) is attributed and billed to (default the owner); providers.ffbox.enabled: true lets FFBox's connector connect (read-only reports: capacity, conversations, intake), false drops it at once (default false); providers.ffbox.token: FFBox's connector token (ffpv1_…), write-only, stored only as its SHA-256; providers.ffbox.devRequests (an owner's only, the whole block): { enabled (default true), perHour (0-1000, default 20), maxFiles (0-10, default 10), maxRequestMB (1-500, default 500) }, unknown keys refused; placement.prefer: the computers new game-repo work goes to first, in order (machine ids such as "lothdesktop", "m5", "beast"; a list or one comma-separated string; null clears), and placement.avoid: computers kept off unless nothing else has room, each with why ({ "beast": "BEAST unstable, 2026-10-05" }; null clears) (docs/machines.md, "Placing work"); attachments.maxMB: the largest file a person may attach to a message (1-4096 MB, default 200); attachments.retentionDays: how many days an attached file nobody sent on is kept (1-3650, default 30) (docs/attachments.md); orchestrator.compactAtTokens: an orchestrator (the dispatcher too) compacts its conversation by itself between turns once its context reaches this many tokens (0 = off, else 50,000-900,000, default 200,000); orchestrator.compactAtTurnUsd: or once a turn cost at least this many USD with the context at 100,000 tokens or more (0 = off, else 0.05-50, default 1) (docs/orchestrators.md, "Compacting a conversation"); orchestrator.messagesPerPerson: message_person messages a person's orchestrator may send one person until either of the two writes to their own orchestrator (1-100, default 10; Lothsahn's and Ben's orchestrators have no count between them, w627); orchestrator.filingsPerMessage / orchestrator.followUpsPerMessage: filings, and follow-ups to one worker, between two messages of its person (1-100, default 3 each) (docs/orchestrators.md, "Loops, limits and safety"); hostGuard.cleanup.ageRules: JSON list of { "path", "olderThanDays" (>= 3) } whose old entries each clean-up pass removes (never a drive root, the home folder, the sandboxes, this app or a protected path); hostGuard.cleanup.everyMinutes: how often this host's clean-up runs (0 = only below the soft threshold, else 15-1440, default 60); hostGuard.cleanup.softFreeGB: below this much free space it runs every 15 minutes with the cache-emptying rules, and tells you when it cannot get back above (default warnFreeGB + 40 = 120; must be above warnFreeGB); machines.cleanup.everyMinutes / machines.cleanup.softFreeGB (optionally with machine): the same for the machines' daemons (defaults 60 and 80 GB). usagePollMinutes: how often every Claude account's plan usage is polled, here and by the machines' daemons (5-240, default 15; the usage endpoint rate-limits). intake.ffbox (an owner's only: refused unless the person who asked is an owner): the FFBox intake's whole block, replaced, so a key left out takes its default (docs/intake.md): an object with enabled (default false), branches, diagnoses, requests (default true once enabled), boardCheck (answer FFBox's ledger check), escalations (both default false), repo ("owner/name" as board answers name the game repo; default from repo.url), dailyCap (0-200, default 10), match { high (default 0.7), medium (default 0.45) } (board_check's match bands, 0-1), autoApprove { enabled (default false), maxPerDay (0-100, default 3) } and desync { enabled (default true), maxPerDay (0-100, default 10) } (Lothsahn's FFBox desync PR policy: such a PR is approved at once into a review-and-merge request under the policy); unknown keys are refused; it applies at once, and FFBox's connection stays up (a board_check or request while it is off is answered not_enabled). value null removes the key (back to the default). intake.reviewers (an owner's only, like intake.ffbox): who may approve or decline intake requests and answer design questions: a list of user ids (or one comma-separated string), each a login that exists (e.g. ["ben", "lothsahn"]; unknown ids are refused and nothing is written); the whole list is replaced, and null removes it (then only the owner decides); it applies at once (docs/intake.md). Only when the user asked for the change.`, { key: z.enum(SETTABLE_KEYS), // An object (intake.ffbox) is a loose object, unknown keys kept for normalizeSetting to name: z.record breaks the @@ -1976,13 +1976,20 @@ Stills, clips and notes for a review (the visual checklist, a playtest, a before }, wrap(async (a) => { const { key, value, user, machine } = a; - if (OWNER_ONLY_KEYS.has(key)) { + // The orchestrators' own vault pools ("vault") are an owner's switch, like machines.claudeFromVault; the pool's limits are anyone's. + if (OWNER_ONLY_KEYS.has(key) || (key === 'claudeAccounts.orchestrator' && value === 'vault')) { const problem = this.ownerOnlyProblem(key, ctx, (a as { work_id?: unknown }).work_id); if (problem) throw new Error(problem); } if (user && !this.identity.get(user)) throw new Error(`no login "${user}"; the logins are ${this.identity.list().map((u) => u.userId).join(', ') || '(none)'}`); if (machine && !this.machines.list().some((m) => m.id === machine)) throw new Error(`no machine "${machine}"; the machines are ${this.machines.list().map((m) => m.id).join(', ') || '(none)'}`); const { before, after } = setAppConfig(configPath(), this.cfg, key, value, { user: user && this.identity.get(user)?.userId, machine, users: this.identity.list().map((u) => u.userId) }); + // Who changed what, for the journal (fffctl logs); a secret key's values are masked, the pool limits are numbers. + { + const by = ctx.owner ? `${ctx.owner.displayName} (${ctx.owner.userId})` : ctx.role === 'dispatcher' ? `the dispatcher, for ${this.store.work.get(String((a as { work_id?: unknown }).work_id ?? '').trim().toLowerCase())?.requestedBy.userId ?? 'a request'}` : 'an orchestrator'; + const show = (v: unknown) => (SECRET_KEYS.has(key) ? maskSecret(v) : JSON.stringify(v ?? null)); + console.log(`app config: ${key}${machine ? ` (${machine})` : ''}${user ? ` (${user})` : ''} ${show(before)} -> ${show(after)}, set by ${by}`); + } if (key === 'publicUrl') this.machines.pushOutsideWatch(); // the outside watchdog watches this URL if (key.startsWith('machines.cleanup.')) this.machines.pushCleanupConfig(); if (key === 'usagePollMinutes') this.usagePollChanged?.(); @@ -1990,6 +1997,7 @@ Stills, clips and notes for a review (the visual checklist, a playtest, a before if (key.startsWith('intake.')) { return `${key}: ${JSON.stringify(before ?? null)} → ${JSON.stringify(after ?? null)}. Written to config.json and applied to the running server. FFBox's connection stays up. The Dispatcher page's Intake tab shows the switches in effect.`; } + if (key.startsWith('vault.pool.')) return `${key}: ${JSON.stringify(before ?? null)} → ${JSON.stringify(after ?? null)} (percent). The token pool's limits are one set for everyone; the next process start on any Claude token reads it, with no restart. system_status shows each pool's state.`; if (key === 'providers.ffbox.token') return `${key}: set. Written to config.json as its SHA-256 only (providers.ffbox.tokenSha256); the connector's next connection must use it. The value is never shown.`; if (key === 'userClaudeEnv.CLAUDE_CODE_OAUTH_TOKEN') { return `${key} for ${user}: ${before} → ${after}. Written to config.json. Agents started for ${user} from now on run on it; running ones keep their account until their process restarts. The value is never shown.`; @@ -2944,11 +2952,19 @@ ${this.worldBrief(false)} * it has one (claudeAccounts.dispatcher), a person's orchestrator on their own token when they have one, else the * orchestrator role's account. A role on the token file (w464) reads it now and takes nobody's own token. */ - orchestratorEnv(owner: Requester | undefined): Record { + orchestratorEnv(owner: Requester | undefined, sessionId?: string): Record { const role = !owner && dispatcherOwnAccount(this.cfg) ? 'dispatcher' : 'orchestrator'; - const base = hostProcessEnv(this.cfg, role); - if (role === 'dispatcher' || hostAccount(this.cfg, role) === 'tokenfile') return base; - return claudeEnvFor(this.cfg, owner ?? this.identity.systemPayer(), base); + const plain = (): Record => { + const base = hostProcessEnv(this.cfg, role); + // The token file, or "vault" with a token file to fall back to (w738), takes nobody's own token. + const account = hostAccount(this.cfg, role); + if (role === 'dispatcher' || account === 'tokenfile' || (account === 'vault' && this.cfg.claudeTokenFile)) return base; + return claudeEnvFor(this.cfg, owner ?? this.identity.systemPayer(), base); + }; + // A person's orchestrator on "vault" (claudeAccounts.orchestrator, w738) runs on that person's own pool token, picked + // now; the dispatcher never does (it keeps the shared token file). + if (role === 'orchestrator' && owner) return poolRunEnv(this.cfg, { role: 'orchestrator', personId: owner.userId, sessionId }, plain).env; + return plain(); } /** @@ -2981,7 +2997,9 @@ ${this.worldBrief(false)} settings: { autoMemoryEnabled: false, disableClaudeAiConnectors: true }, hooks: { PreToolUse: [{ hooks: [opsGuard()] }] }, maxBudgetUsd: OPS_LIMITS.budgetUsd, - env: hostProcessEnv(this.cfg, 'orchestrator'), + // On "vault" the worker runs on the pool of the person whose orchestrator gave it this job (w738), picked at each process + // start; OpsWorker.send starts a fresh process for every new job and every change of caller. + env: poolRunEnv(this.cfg, { role: 'ops', personId: info.requestedBy?.userId, sessionId: info.id }, () => hostProcessEnv(this.cfg, 'orchestrator')).env, systemPrompt: { type: 'preset', preset: 'claude_code', append: opsBrief(this.cfg.ownerName) }, spawnClaudeCodeProcess: opsSpawner(), }; @@ -3017,7 +3035,7 @@ ${this.worldBrief(false)} // (w464: Lothsahn's account, whoever the system payer is), else on the system payer's. Without one, what config // claudeAccounts.orchestrator picks: the host token, or this host's stored claude.ai login. // A role on the token file (w464, change 18) runs on it alone: a person's own token does not override it. - env: this.orchestratorEnv(owner), + env: this.orchestratorEnv(owner, info.id), systemPrompt: { type: 'preset', preset: 'claude_code', append: `${owner ? this.personalBrief(owner) : this.dispatcherBrief()}\n\n${memoryBrief(memory, owner?.displayName)}${opsAllowedOrchestrator(info) ? `\n\n${dataBrief(this.cfg.dataDir)}` : ''}` }, ...(this.cfg.claudeExecutable ? { pathToClaudeCodeExecutable: this.cfg.claudeExecutable } : {}), }; diff --git a/server/appConfig.ts b/server/appConfig.ts index 291d1f61..23903f33 100644 --- a/server/appConfig.ts +++ b/server/appConfig.ts @@ -1,7 +1,7 @@ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; -import { DEFAULT_CLEANUP, DEFAULT_USAGE_POLL_MINUTES, LOOP_GUARD_RANGE, ROOT, TOKEN_FILE_ROLES, VOICE_DEFAULTS, type ClaudeAccount, type Config, type HostRole, type IntakeConfig } from './config.ts'; +import { DEFAULT_CLEANUP, DEFAULT_USAGE_POLL_MINUTES, LOOP_GUARD_RANGE, ROOT, TOKEN_FILE_ROLES, VAULT_ROLES_ACCOUNT, VOICE_DEFAULTS, type ClaudeAccount, type Config, type HostRole, type IntakeConfig } from './config.ts'; import { OAUTH_TOKEN, SECRET_KEYS, hostLoginProblem, maskSecret, readTokenFile } from './secrets.ts'; import { PROVIDER_TOKEN, tokenSha256 } from './providerProtocol.ts'; import { USER_ID } from './identity.ts'; @@ -10,6 +10,7 @@ import { refuseInDryRun } from './dryRun.ts'; import { placeId } from './placement.ts'; import { DEV_DEFAULTS, type DevRequestsConfig } from './devRequests.ts'; import { STALE_OUTPUT_DEFAULTS, staleOutputSettings, type StaleOutputSettings } from './staleOutput.ts'; +import { poolLimits } from './tokenPool.ts'; import { AUTO_COMPACT_LIMITS } from './autoCompact.ts'; /** @@ -54,6 +55,13 @@ export const SETTABLE_KEYS = [ 'machines.useHostClaudeEnv', // Whether a machine's runs take their Claude token from the token vault (docs/vault.md, w512; optional `machine`). The owner's only. 'machines.claudeFromVault', + // The Claude token pool's limits (w739, docs/vault.md section 4): one system-wide set, anyone may change them (not owner-only). + // Each is a percent, 0-100; the one-at-a-time limit may not be above the retire limit. + 'vault.pool.sessionHoldPercent', + 'vault.pool.onePerWeeklyPercent', + 'vault.pool.retireWeeklyPercent', + 'vault.pool.reservePerDayPercent', + 'vault.pool.reserveSessionPercent', // Who automatic work (scheduled standing runs, intake-triggered FFBox work) is attributed and billed to. 'systemPayer', // Files people attach to messages (docs/attachments.md): the largest one, and how long one nobody sends on is kept. @@ -305,6 +313,11 @@ export function normalizeSetting(key: SettableKey, value: unknown, cfg?: Config, if (cfg) readTokenFile(cfg); return v; } + if (v === 'vault') { + // w738: each person's orchestrator on that person's own vault Claude token; the dispatcher stays on the token file. + if (!VAULT_ROLES_ACCOUNT.includes(role)) throw new Error(`${key} cannot be "vault": only ${VAULT_ROLES_ACCOUNT.join(', ')} run on a person's own vault tokens (the dispatcher stays on the token file)`); + return v; + } if (v !== 'login' && v !== 'token') throw new Error(`${key} is "login" (this host's stored claude.ai login) or "token" (config claudeEnv's)${TOKEN_FILE_ROLES.includes(role) ? ', or "tokenfile" (config claudeTokenFile\'s)' : ''}`); // Refuse a switch that would leave the role unable to start: the stored login must be there and alive. const problem = v === 'login' && cfg ? hostLoginProblem(cfg) : undefined; @@ -340,6 +353,19 @@ export function normalizeSetting(key: SettableKey, value: unknown, cfg?: Config, if (!Number.isInteger(n) || n < 5 || n > 240) throw new Error('usagePollMinutes is a whole number of minutes from 5 to 240'); return n; } + case 'vault.pool.sessionHoldPercent': + case 'vault.pool.onePerWeeklyPercent': + case 'vault.pool.retireWeeklyPercent': + case 'vault.pool.reservePerDayPercent': + case 'vault.pool.reserveSessionPercent': { + // The token pool's limits (w739): a percent, 0-100 (a number or its text); one at a time may not be above retired. + const n = typeof value === 'string' && value.trim() !== '' ? Number(value) : value; + if (typeof n !== 'number' || !Number.isFinite(n) || n < 0 || n > 100) throw new Error(`${key} is a percent from 0 to 100`); + const now = poolLimits(cfg); + if (key === 'vault.pool.onePerWeeklyPercent' && n > now.retireWeekly) throw new Error(`${key} (${n}) cannot be above vault.pool.retireWeeklyPercent (${now.retireWeekly}): one at a time starts before a token is retired`); + if (key === 'vault.pool.retireWeeklyPercent' && n < now.onePerWeekly) throw new Error(`${key} (${n}) cannot be below vault.pool.onePerWeeklyPercent (${now.onePerWeekly}): one at a time starts before a token is retired`); + return n; + } case 'attachments.maxMB': { const n = Number(value); if (!Number.isInteger(n) || n < 1 || n > 4096) throw new Error('attachments.maxMB is a whole number of megabytes from 1 to 4096'); @@ -559,6 +585,14 @@ export function setAppConfig(file: string, cfg: Config, key: SettableKey, value: cfg.claudeAccounts = accounts; } else if (key === 'machines.useHostClaudeEnv') cfg.machines = { ...cfg.machines, useHostClaudeEnv: next as boolean | Record | undefined }; else if (key === 'machines.claudeFromVault') cfg.machines = { ...cfg.machines, claudeFromVault: next as boolean | Record | undefined }; + else if (key.startsWith('vault.pool.')) { + // Live: the pool picks read the limits at each process start (poolLimits). + const pool = { ...cfg.vault?.pool } as Record; + const field = key.slice('vault.pool.'.length); + if (v === undefined) delete pool[field]; + else pool[field] = v as number; + cfg.vault = { ...cfg.vault, pool: pool as NonNullable['pool'] }; + } else if (key === 'systemPayer') cfg.systemPayer = v as string | undefined; else if (key === 'providers.ffbox.enabled' || key === 'providers.ffbox.token' || key === 'providers.ffbox.devRequests') { const ffbox = { ...cfg.providers?.ffbox }; diff --git a/server/config.ts b/server/config.ts index 8e56bb5a..b179ff6b 100644 --- a/server/config.ts +++ b/server/config.ts @@ -7,6 +7,7 @@ import path from 'node:path'; import type { PermissionMode } from '../shared/types.ts'; import { checkObject, dataRecoveries, readJsonDurable } from './durable.ts'; import { staleOutputSettings, type StaleOutputSettings } from './staleOutput.ts'; +import { checkPoolConfig, type PoolConfig } from './tokenPool.ts'; /** What a portal-run agent runs on (docs/accounts.md): the computer's stored claude.ai login, or config claudeEnv's token. */ @@ -15,8 +16,10 @@ import { staleOutputSettings, type StaleOutputSettings } from './staleOutput.ts' * (w464, docs/portal-on-ffbox-host.md change 18) the OAuth token in config claudeTokenFile, read at each session start * and given to that process alone. "tokenfile" is for the orchestrator, dispatcher and standing roles, never workers. */ -export type ClaudeAccount = 'login' | 'token' | 'tokenfile'; -export const CLAUDE_ACCOUNTS: readonly ClaudeAccount[] = ['login', 'token', 'tokenfile']; +export type ClaudeAccount = 'login' | 'token' | 'tokenfile' | 'vault'; +export const CLAUDE_ACCOUNTS: readonly ClaudeAccount[] = ['login', 'token', 'tokenfile', 'vault']; +/** The role that may run on a person's own vault pool (w738): the orchestrators, each on its own person's tokens. */ +export const VAULT_ROLES_ACCOUNT: readonly string[] = ['orchestrator']; /** The roles that may run on the token file (TOKEN_FILE): never workers, which run on machines and other people's work. */ export const TOKEN_FILE_ROLES: readonly string[] = ['orchestrator', 'dispatcher']; /** The roles config claudeAccounts picks an account for, on this host. */ @@ -165,7 +168,7 @@ export interface Config { useHostClaudeEnv?: boolean | Record; /** * The token vault (docs/vault.md, w512; default false): a machine's runs take their Claude token from the vault, - * chosen per run by plan headroom, instead of the host token or the machine's own login. Same shape as + * taken per run from the run person's pool (docs/vault.md section 4), instead of the host token or the machine's own login. Same shape as * useHostClaudeEnv: true, false, or per machine with "*" for the rest. A person's own token still wins for their work. */ claudeFromVault?: boolean | Record; @@ -221,6 +224,12 @@ export interface Config { * the nightly regression sentry and its delegations). Portal user ids; defaults in server/vault.ts UNATTRIBUTED_DEFAULTS. */ unattributed?: Partial>; + /** + * The Claude token pool's limits (w739, docs/vault.md section 4), one system-wide set that anyone can change with + * set_app_config: sessionHoldPercent (80), onePerWeeklyPercent (95), retireWeeklyPercent (99), reservePerDayPercent + * (5: the dispatcher's token keeps this per day left until its weekly reset) and reserveSessionPercent (20). + */ + pool?: PoolConfig; }; /** * Providers (docs/ffbox-integration.md): FFBox, whose connector dials out to /provider. `enabled` (default @@ -754,13 +763,16 @@ export function windowsPathsOffWindows(cfg: Pick & Partial>) { +export function checkAccountConfig(cfg: Pick & Partial>) { + const poolProblem = checkPoolConfig(cfg.vault?.pool); + if (poolProblem) throw new Error(poolProblem); const a: unknown = cfg.claudeAccounts; if (a !== undefined) { if (typeof a !== 'object' || a === null || Array.isArray(a)) throw new Error('config claudeAccounts is an object, e.g. { "orchestrator": "login" }'); for (const [role, v] of Object.entries(a)) { if (!HOST_ROLES.includes(role as HostRole)) throw new Error(`config claudeAccounts.${role}: no such role (${HOST_ROLES.join(', ')})`); - if (!CLAUDE_ACCOUNTS.includes(v as ClaudeAccount)) throw new Error(`config claudeAccounts.${role} is "login" or "token" (or "tokenfile" for ${TOKEN_FILE_ROLES.join(', ')})`); + if (!CLAUDE_ACCOUNTS.includes(v as ClaudeAccount)) throw new Error(`config claudeAccounts.${role} is "login" or "token" (or "tokenfile" for ${TOKEN_FILE_ROLES.join(', ')}, "vault" for ${VAULT_ROLES_ACCOUNT.join(', ')})`) + if (v === 'vault' && !VAULT_ROLES_ACCOUNT.includes(role)) throw new Error(`config claudeAccounts.${role} cannot be "vault": only ${VAULT_ROLES_ACCOUNT.join(', ')} run on a person's own vault tokens (the dispatcher stays on the token file)`);; if (v === 'tokenfile' && !TOKEN_FILE_ROLES.includes(role)) throw new Error(`config claudeAccounts.${role} cannot be "tokenfile": only ${TOKEN_FILE_ROLES.join(', ')} run on the token file`); if (v === 'tokenfile' && !(cfg as Partial>).claudeTokenFile) throw new Error(`config claudeAccounts.${role} is "tokenfile" but config claudeTokenFile names no file`); } diff --git a/server/index.ts b/server/index.ts index 13c2d6ad..5b34d7cc 100644 --- a/server/index.ts +++ b/server/index.ts @@ -39,7 +39,8 @@ import { DispatcherChatRefused } from './orchestrators.ts'; import { OPS_PEOPLE, OPS_REFUSED } from './opsWorker.ts'; import { backupMemory, healMemory, memoryRootOf } from './orchestratorMemory.ts'; import { describeMemoryGit, versionMemory } from './memoryGit.ts'; -import { accountSetupLines, addSecretValues, claudeFromVault, hostAccount, hostRole, hostRoleOf, scrubTranscripts, shownRoles, tokenFileToken, usesHostClaudeEnv } from './secrets.ts'; +import { accountSetupLines, addSecretValues, claudeFromVault, hostAccount, reserveLines, hostRole, hostRoleOf, scrubTranscripts, shownRoles, tokenFileToken, usesHostClaudeEnv } from './secrets.ts'; +import { poolLimits, clock } from './tokenPool.ts'; import { VAULT_FILE, VAULT_KINDS, VAULT_ROLES, Vault, keySource, setVaultContext, vaultStatusLine, type VaultKind, type VaultRole, type VaultShare } from './vault.ts'; import { collectNetwork, loadOutsideWatchState, outsideWatchConfig, saveOutsideWatchState, watchedPortalUrl, watcherOf } from './outsideWatch.ts'; import { endMaybeGzip } from './compress.ts'; @@ -56,7 +57,7 @@ import { DRY_RUN_BANNER, DRY_RUN_WHY, defuseConfig, dryRun } from './dryRun.ts'; import { portalPublicKey, tailnetAddress } from './machineSsh.ts'; import { machineSshHttp } from './machineSshHttp.ts'; import { MAX_DICTATION_SECONDS, MAX_TTS_CHARS, buildVoicePrompt, wavSeconds, type SpeakRequest, type TranscribeRequest, type VocabularySource } from '../shared/voice.ts'; -import type { AppState, CreateSandboxRequest, HostStatus, Machine, PermissionDecisionRequest, Requester, ServerEvent, SessionInfo, SessionKind, StandingAgentInput, StartSessionRequest, SystemStats } from '../shared/types.ts'; +import type { AppState, CreateSandboxRequest, HostStatus, Machine, TokenWarning, PermissionDecisionRequest, Requester, ServerEvent, SessionInfo, SessionKind, StandingAgentInput, StartSessionRequest, SystemStats } from '../shared/types.ts'; import { slugify } from './sandboxes.ts'; const WEB = path.join(ROOT, 'web', 'dist'); @@ -1661,10 +1662,13 @@ function fileTokenEntry(): { token: string; label: string }[] { usage.personTokens = () => [...personTokens(), ...fileTokenEntry(), ...vault.claudeTokens()]; /** Recent reasons a run did not get what the vault should give it (a fallback, an entry that would not open), newest last. */ const vaultProblems: { at: string; line: string }[] = []; +/** Live sessions whose process started on this credential key, not counting `except` (the asking session's own). */ +const liveOnKey = (key: string, except?: string) => [...store.sessions.values()].filter((s) => s.id !== except && s.account === key && s.status !== 'stopped' && s.status !== 'error').length; setVaultContext({ vault, usageOf: (fp) => usage.entries.get(`token:${fp}`)?.usage, - liveOn: (fp) => [...store.sessions.values()].filter((s) => s.account === `token:${fp}` && s.status !== 'stopped' && s.status !== 'error').length, + liveOn: (fp, except) => liveOnKey(`token:${fp}`, except), + limits: () => poolLimits(cfg), payer: () => identity.systemPayer().userId, onProblem: (line) => { if (vaultProblems.at(-1)?.line !== line) console.warn(`vault: ${line}`); @@ -1679,8 +1683,15 @@ setVaultContext({ function sourceOf(s: SessionInfo, token: string | undefined, toMachine: (id: string) => string | undefined) { const role = hostRoleOf(cfg, s); const login = () => hostAccount(cfg, role) === 'login'; + // A person's orchestrator (or the ops worker for them) on "vault" (w738) ran on the pool token it was handed last; a + // session with no pick (or one before a restart) on the token file it falls back to. + if (s.kind !== 'worker' && !s.machineId && hostAccount(cfg, role) === 'vault') { + const running = !!s.account && !!s.status && s.status !== 'stopped' && s.status !== 'error'; + const fp = running ? undefined : vault.lastPickOf(s.id); + if (fp) return `token:${fp}`; + } // A role on the token file (w464) ran on it, whoever the session was for. - if (s.kind !== 'worker' && !s.machineId && hostAccount(cfg, role) === 'tokenfile') { + if (s.kind !== 'worker' && !s.machineId && (hostAccount(cfg, role) === 'tokenfile' || (hostAccount(cfg, role) === 'vault' && cfg.claudeTokenFile))) { const t = tokenFileToken(cfg); if (t) return sessionSource({ ...s, requestedBy: undefined }, t, toMachine, () => undefined, () => false); } @@ -1744,6 +1755,8 @@ bus.on('event', (e: ServerEvent) => { machines.onUsage = (id, account, u) => usage.report(id, account, u); for (const s of sessions.sessions.values()) usage.recordCost(s.info.id, s.info.costUsd); // baselines sessions.events.on('result', (s: { info: { id: string; costUsd: number } }) => usage.recordCost(s.info.id, s.info.costUsd)); +// The SDK's rate_limit_event (the API's rate-limit headers on the session's own requests) raises its token's meter between polls (w739). +sessions.events.on('rateLimit', (s: { info: { account?: string } }, info: { rateLimitType?: string; utilization?: number; resetsAt?: number }) => usage.noteRateLimit(s.info.account, info)); /** Whose account each orchestrator runs on (docs/orchestrators.md), for system_status: a person without a token of their own is on the owner's. */ function orchestratorAccountsLine() { const people = identity.list(); @@ -1773,7 +1786,7 @@ agents.usagePollChanged = () => { agents.machineStatusLines = () => machines.list().map((m) => machineLoadLine(m, machines.statsOf(m.id), machines.isOnline(m.id), machines.protocolOf(m.id))); agents.extraStatusLines = () => { const ffbox = providers.statusLine(); - return [...(ffbox ? [ffbox] : []), max.statusLine(), ...outsideWatchLines(), ...pathHealth.statusLines(), ...unitWatchdog.statusLines(), ...vaultLines(), ...(retiredKeys ? [retiredKeys] : [])]; + return [...(ffbox ? [ffbox] : []), max.statusLine(), ...outsideWatchLines(), ...pathHealth.statusLines(), ...unitWatchdog.statusLines(), ...vaultLines(), ...tokenReport().lines, ...(retiredKeys ? [retiredKeys] : [])]; }; /** The vault's summary for system_status, and its fallbacks in the last 24 hours (docs/vault.md). Never a value. */ const vaultLines = () => { @@ -1781,6 +1794,54 @@ const vaultLines = () => { const recent = vaultProblems.filter((p) => Date.now() - Date.parse(p.at) < 24 * 3_600_000); return [...(line ? [line] : []), ...(recent.length ? [`WARNING: the vault fell short ${recent.length} time(s) in 24 h; latest (${recent.at(-1)!.at}): ${recent.at(-1)!.line}`] : [])]; }; +/** + * Each person's Claude token pool as the rules read it now, and the dispatcher's (and host token's) reserve (w739, + * docs/vault.md section 4): system_status lines and the dashboard's warnings. Only names, last four characters and numbers. + */ +function tokenReport(now = Date.now()): { lines: string[]; warnings: TokenWarning[] } { + const usageOfKey = (key: string) => usage.entries.get(key)?.usage; + const titlesOn = (key: string) => [...store.sessions.values()].filter((s) => s.account === key && s.status !== 'stopped' && s.status !== 'error').map((s) => s.title); + const pools = vault.pools({ usageOf: (fp) => usageOfKey(`token:${fp}`), liveOn: (fp, except) => liveOnKey(`token:${fp}`, except), limits: () => poolLimits(cfg), now: () => now }); + const lines: string[] = []; + const warnings: TokenWarning[] = []; + for (const [person, list] of [...pools].sort((a, b) => a[0].localeCompare(b[0]))) { + const each = list.map((t) => { + const on = titlesOn(`token:${t.entry.fingerprint}`); + return `${t.entry.name} …${t.entry.last4} ${t.view.state} (${t.view.why}${t.view.weeklyResetsAt ? `; weekly resets ${clock(t.view.weeklyResetsAt)}` : ''}${on.length ? `; ${on.length} running: ${on.slice(0, 3).join(', ')}${on.length > 3 ? ', …' : ''}` : ''})`; + }); + const states = list.map((t) => t.view.state); + const usedUp = states.every((x) => x === 'exhausted'); + const noneOk = !states.includes('ok'); + const next = list.map((t) => t.view.until).filter((x): x is string => !!x).sort()[0]; + const verdict = usedUp + ? `all used up: ${person}'s workers wait${hostAccount(cfg, 'orchestrator') === 'vault' ? " and their orchestrator is stopped" : ''} until ${clock(next)}` + : noneOk + ? `all over their caps: ${person}'s workers wait until ${clock(next)}${hostAccount(cfg, 'orchestrator') === 'vault' ? '; their orchestrator uses the pool up (the token with the most room)' : ''}` + : 'workers and agents served'; + lines.push(`Claude token pool, ${person} (${list.length} token${list.length === 1 ? '' : 's'}, soonest weekly reset first; caps ${poolLimits(cfg).sessionHold}% 5-hour / ${poolLimits(cfg).onePerWeekly}% one at a time / ${poolLimits(cfg).retireWeekly}% retired): ${each.join('; ')}. ${verdict}`); + if (usedUp) warnings.push({ id: `pool:${person}`, kind: 'exhausted', text: `${person}'s Claude tokens are all used up: their workers wait${hostAccount(cfg, 'orchestrator') === 'vault' ? ' and their orchestrator is stopped' : ''} until the first one resets (${clock(next)}). No override.` }); + else if (noneOk) warnings.push({ id: `pool:${person}`, kind: 'held', text: `${person}'s Claude token pool is over its caps: their workers wait until ${clock(next)}${hostAccount(cfg, 'orchestrator') === 'vault' ? '; their orchestrator keeps running on the token with the most room until it is used up' : ''}.` }); + } + const others = (key: string, role: 'dispatcher' | 'host-token') => + role === 'dispatcher' ? [...store.sessions.values()].filter((s) => s.account === key && s.status !== 'stopped' && s.status !== 'error' && !(s.kind === 'orchestrator' && s.orchestratorRole === 'dispatcher')).length : 0; + for (const r of reserveLines(cfg, usageOfKey, others, now)) { + lines.push(r.line); + if (r.warning) warnings.push({ id: `reserve:${r.cred.key}`, kind: 'reserve', text: r.warning }); + } + return { lines, warnings }; +} +let tokenWarningsShown = ''; +/** The dashboard's token banners follow the meters and the sessions: refreshed on a timer, broadcast when they change. */ +function refreshTokenWarnings() { + const w = tokenReport().warnings; + const sig = JSON.stringify(w); + if (sig === tokenWarningsShown) return; + tokenWarningsShown = sig; + host.tokenWarnings = w.length ? w : undefined; + broadcast({ type: 'host', host: { ...host, drain: drainer.status } }); +} +setInterval(refreshTokenWarnings, 20_000).unref(); +setTimeout(refreshTokenWarnings, 8_000).unref(); const outsideWatchLines = () => { const w = watcher(); const c = watchConfig(); diff --git a/server/opsWorker.test.ts b/server/opsWorker.test.ts index b2b88c71..8810f9fc 100644 --- a/server/opsWorker.test.ts +++ b/server/opsWorker.test.ts @@ -420,6 +420,71 @@ test('the one session: only an allowed orchestrator reaches it, a job needs its ops.close(); }); +test("every new job and every change of caller starts a fresh conversation and process, on the caller's token (w738); a follow-up keeps both", async () => { + setQueryForTesting(fakeQuery({ stepMs: 1 }) as never); + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ffsb-ops-f-')); + const store = new Store(dir); + const sessions = new SessionManager({} as Config, store); + let personTurn = true; + // The options factory is what picks the token at each process start (Agents.opsOptions reads info.requestedBy). + const starts: (string | undefined)[] = []; + let clock = Date.now(); + const ops = new OpsWorker({ + sessions, + store, + options: (info) => { + starts.push(info.requestedBy?.userId); + return {}; + }, + tellOrchestrator: () => {}, + personTurn: () => personTurn, + file: path.join(dir, 'ops-worker.json'), + now: () => clock, + }); + const loth = orch('lo', LOTH); + const ben = orch('bo', BEN); + const settle = () => new Promise((r) => setTimeout(r, 300)); + // 1. Lothsahn's job: a process on lothsahn. + assert.match(ops.send(loth, 'job one'), /new job of Lothsahn's/); + await settle(); + const h = sessions.get(OPS_ID); + const first = h.info.sdkSessionId; + assert.ok(first, 'a conversation'); + assert.deepEqual(starts, ['lothsahn']); + // 2. A follow-up within the job (a harness turn of the same orchestrator): the same process, the same conversation. + personTurn = false; + assert.doesNotMatch(ops.send(loth, 'follow-up'), /new job/); + await settle(); + assert.deepEqual(starts, ['lothsahn'], 'no new process'); + assert.equal(h.info.sdkSessionId, first); + // 3. Ben's orchestrator, in Ben's own turn: another caller. A fresh conversation, a new process on Ben. + personTurn = true; + assert.match(ops.send(ben, "ben's job"), /new job of Ben's/); + await settle(); + assert.deepEqual(starts, ['lothsahn', 'ben']); + assert.notEqual(h.info.sdkSessionId, first, 'nothing of the last job, context or token, carries over'); + assert.ok(store.readTranscript(OPS_ID, 80).some((e) => e.kind === 'system' && e.text.includes("A new job from Ben's orchestrator: a fresh conversation"))); + const second = h.info.sdkSessionId; + // 4. Ben's follow-up keeps it; 5. Lothsahn's next job (the job was Ben's: a change of caller again) is fresh on Lothsahn. + personTurn = false; + ops.send(ben, 'and one more thing'); + await settle(); + assert.deepEqual(starts, ['lothsahn', 'ben']); + personTurn = true; + assert.match(ops.send(loth, 'job two'), /new job of Lothsahn's/); + await settle(); + assert.deepEqual(starts, ['lothsahn', 'ben', 'lothsahn']); + assert.notEqual(h.info.sdkSessionId, second); + // 6. The same person's next job, after theirs has gone quiet for good: still a new job, still fresh. + const third = h.info.sdkSessionId; + clock += OPS_LIMITS.jobMs + 60_000; + assert.match(ops.send(loth, 'job three'), /new job of Lothsahn's/); + await settle(); + assert.equal(starts.length, 4, 'a process of its own'); + assert.notEqual(h.info.sdkSessionId, third); + ops.close(); +}); + test('w631: a job sent for ledger requests: the ids are checked, the worker is told how to close them, and its turn end goes to the ledger', async () => { setQueryForTesting(fakeQuery({ stepMs: 1 }) as never); const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ffsb-ops-w-')); diff --git a/server/opsWorker.ts b/server/opsWorker.ts index a0a6716d..de9491c6 100644 --- a/server/opsWorker.ts +++ b/server/opsWorker.ts @@ -585,7 +585,11 @@ export class OpsWorker { if (this.job && this.job.by.userId.toLowerCase() !== person.userId.toLowerCase() && isMidTurn(h.info)) { throw new Error(`the orchestration worker is busy with ${this.job.by.displayName}'s job (${this.job.what}); wait for its turn to end, or ask them`); } - if (fresh) { + // Every new job starts a fresh conversation and a fresh process (w738): nothing, no context and no token, carries over from + // one job to the next, and a job from the other person's orchestrator runs on that person's own pool token. A follow-up + // within the job keeps both. + // (A turn still running for the same person, on a job that only expired, is not cut off: it is the same person's token.) + if (fresh || (opening && !isMidTurn(h.info))) { if (h.live) h.stop(true); delete h.info.sdkSessionId; this.d.store.putSession(h.info); @@ -619,12 +623,22 @@ export class OpsWorker { if (!this.d.personTurn(caller.id)) throw new Error(`a portal deploy needs ${person.displayName}'s own words in this turn (this turn is the harness's: a check-in, a timer or a relayed report); ask them`); const ids = this.workIdsOf(workIds); const at = new Date(this.now()); + const previous = this.job; + const h = this.handle(); + if (previous && previous.by.userId.toLowerCase() !== person.userId.toLowerCase() && isMidTurn(h.info)) { + throw new Error(`the orchestration worker is busy with ${previous.by.displayName}'s job (${previous.what}); wait for its turn to end, or ask them`); + } const grant = { by: person.userId, name: person.displayName, at: at.toISOString(), expires: new Date(at.getTime() + OPS_LIMITS.deployGrantMs).toISOString() }; fs.writeFileSync(path.join(path.dirname(this.d.file), OPS_GRANT), `${JSON.stringify(grant)}\n`, { mode: 0o600 }); this.deploying = { by: person, at: at.toISOString() }; this.job = { by: person, at: at.toISOString(), what: 'deploy the portal (fffctl update)', ...(ids.length ? { workIds: ids } : {}) }; this.save(); - const h = this.handle(); + // A deploy is a new job too: a fresh conversation and process, on this person's token (the restart that follows resumes it). + if (!isMidTurn(h.info)) { + if (h.live) h.stop(true); + delete h.info.sdkSessionId; + } + this.d.store.append(OPS_ID, { kind: 'system', text: `A new job (a deploy) from ${person.displayName}'s orchestrator: a fresh conversation.` }); h.info.requestedBy = person; this.d.store.putSession(h.info); console.log(`ops-worker: ${person.userId} asked for a portal deploy in a turn of their own; grant until ${grant.expires}`); diff --git a/server/secrets.ts b/server/secrets.ts index a18a9128..f07c3a32 100644 --- a/server/secrets.ts +++ b/server/secrets.ts @@ -4,8 +4,10 @@ import path from 'node:path'; import { HOST_ROLES, TOKEN_FILE_ROLES, roleNames, type ClaudeAccount, type Config, type HostRole } from './config.ts'; import type { Requester, SessionInfo, SessionKind } from '../shared/types.ts'; import { claudeEnvFor, userToken } from './identity.ts'; -import { vaultContext, type VaultContext, type VaultRole } from './vault.ts'; -import { credentialsFile, loginUnusable, readStoredLogin, usageEnv } from './usage.ts'; +import { PoolHeldError, vaultContext, type VaultContext, type VaultRole } from './vault.ts'; +import { HOST_LOGIN, credentialsFile, loginUnusable, readStoredLogin, tokenKey, usageEnv } from './usage.ts'; +import { clock, poolLimits, reserveOf, type PoolLimits, type Reserve } from './tokenPool.ts'; +import type { PlanUsage } from '../shared/types.ts'; import { writeFileDurable } from './durable.ts'; /** @@ -160,8 +162,11 @@ export const hostRole = (kind: SessionKind): HostRole => (kind === 'orchestrator * The account this host's agents of `role` run on (config claudeAccounts; default the token; the dispatcher, unset: * the orchestrator's). Workers never run on the token file: a "tokenfile" there (refused at config load) reads as the token. */ -export function hostAccount(cfg: Pick, role: HostRole): ClaudeAccount { +export function hostAccount(cfg: Pick & Partial>, role: HostRole): ClaudeAccount { const v = cfg.claudeAccounts?.[role] ?? (role === 'dispatcher' ? cfg.claudeAccounts?.orchestrator : undefined); + // "vault" (w738) is the orchestrators'; the dispatcher that inherits it stays on the token file (else the host token), and + // so does anyone with no vault tokens: `poolRunEnv` falls back to this. + if (v === 'vault') return role === 'orchestrator' ? 'vault' : cfg.claudeTokenFile ? 'tokenfile' : 'token'; if (v === 'tokenfile') return TOKEN_FILE_ROLES.includes(role) ? 'tokenfile' : 'token'; return v === 'login' ? 'login' : 'token'; } @@ -199,13 +204,19 @@ export function hostRoleOf(cfg: Pick, info: Pick & Partial>, role: HostRole): ClaudeAccount => { + const a = hostAccount(cfg, role); + return a === 'vault' ? (cfg.claudeTokenFile ? 'tokenfile' : 'token') : a; +}; + /** * Config claudeEnv as a host agent of `role` gets it: whole ("token"), or without its credentials ("login"), so * the agent falls back to the claude.ai login stored on this host. Its other variables (CLAUDE_CONFIG_DIR…) stay. */ export function hostClaudeEnv(cfg: Pick & Partial>, role: HostRole): Record { const env = { ...cfg.claudeEnv }; - const a = hostAccount(cfg, role); + const a = baseAccount(cfg, role); return a === 'tokenfile' ? withTokenFile(cfg, env) : a === 'login' ? usageEnv(env) : env; } @@ -216,7 +227,7 @@ export function hostClaudeEnv(cfg: Pick */ export function hostProcessEnv(cfg: Pick & Partial>, role: HostRole, env: Record = process.env): Record { const all = { ...env, ...cfg.claudeEnv }; - const a = hostAccount(cfg, role); + const a = baseAccount(cfg, role); return a === 'tokenfile' ? withTokenFile(cfg, all) : a === 'login' ? usageEnv(all) : all; } @@ -284,7 +295,7 @@ export interface MachineRunEnv { /** * The Claude account and the vault's secrets a run on a machine gets (docs/vault.md, sections 3 and 4). In order: a * person's own token (config userClaudeEnv) for their work, as before; then, on a machine config machines.claudeFromVault - * names, a vault Claude token chosen by plan headroom; else the machine's account without the vault (hostClaudeEnvFor: + * names, a Claude token from the run person's vault pool (server/tokenPool.ts); else the machine's account without the vault (hostClaudeEnvFor: * the host token or its own login). Every other vault entry granted to the run's role and machine is added as * environment. With a vault token, `login` is true so the daemon's own credentials (an API key in its environment would * outrank the token) are dropped and only the token remains. Without a vault context, exactly what it was before. @@ -305,15 +316,124 @@ export function machineRunEnv( const own = userToken(cfg, billed?.userId); if (!ctx) return plain; const wantClaude = vaultOn && !own; - const s = ctx.vault.forRun({ machineId: refId(machine), role: run.role, userId: person, sessionId: run.sessionId }, { claude: wantClaude, usageOf: ctx.usageOf, liveOn: ctx.liveOn }); + const s = ctx.vault.forRun({ machineId: refId(machine), role: run.role, userId: person, sessionId: run.sessionId }, { claude: wantClaude, usageOf: ctx.usageOf, liveOn: ctx.liveOn, limits: ctx.limits }); for (const p of s.problems) ctx.onProblem?.(p); if (s.claude) { - return { env: { ...usageEnv(base), ...s.env, CLAUDE_CODE_OAUTH_TOKEN: s.claude.token }, login: true, account: `vault token ${s.claude.entry.name} …${s.claude.entry.last4} (picked by plan headroom, docs/vault.md)` }; + return { env: { ...usageEnv(base), ...s.env, CLAUDE_CODE_OAUTH_TOKEN: s.claude.token }, login: true, account: poolAccount(s.claude, person) }; } + // The person has Claude tokens and none may serve this run (every one over its caps or used up): the run waits for the + // first to free up. No other person's token, no shared token, not the dispatcher's, not the host token (w739). + if (wantClaude && s.claudeHold) throw new PoolHeldError(`${person ?? 'this run'}'s Claude token pool is held: ${s.claudeHold.why}`, s.claudeHold.next); + // No token of their own (or the vault cannot open them): today's account, as before the pool (docs/vault.md, "Cut-over"). if (wantClaude) ctx.onProblem?.(`no vault Claude token for a ${run.role} run on ${refId(machine)}${person ? ` for ${person}` : ''}; it runs on ${plain.account} instead`); return { ...plain, env: { ...base, ...s.env } }; } +/** "vault token ben-max …abcd" for a run's account line, safe to show. */ +function poolAccount(c: { entry: { name: string; last4: string }; how: 'normal' | 'over-cap' }, person: string | undefined): string { + return `vault token ${c.entry.name} …${c.entry.last4} (${person ? `${person}'s pool` : 'a pool'}${c.how === 'over-cap' ? ', over its caps: used up first' : ''}, docs/vault.md)`; +} + +/** + * The environment of a portal-side Claude process for a person (w738): their own orchestrator ("orchestrator"), or the + * orchestration worker working for them ("ops"). With `claudeAccounts.orchestrator` set to "vault" and the vault holding + * Claude tokens for the person, it is exactly that person's pool token as CLAUDE_CODE_OAUTH_TOKEN, every other Claude + * credential removed first (as the token file's does), picked now so a new token applies to the next session start. + * All their tokens used up or capped: an orchestrator uses its pool up and then stops, the ops worker is held like a + * worker; both throw PoolHeldError (the session start fails with the reason and the next reset). No tokens of their own + * yet (the transition), or the vault cannot open them: `fallback`, today's account. + */ +export function poolRunEnv( + cfg: Pick & Partial>, + run: { role: 'orchestrator' | 'ops'; personId: string | undefined; sessionId?: string }, + fallback: () => Record, + ctx: VaultContext | undefined = vaultContext(), +): { env: Record; account: string } { + const plain = () => ({ env: fallback(), account: baseAccount(cfg, 'orchestrator') === 'tokenfile' ? 'token file' : 'host token' }); + if (hostAccount(cfg, 'orchestrator') !== 'vault' || !ctx || !run.personId) return plain(); + const s = ctx.vault.forRun({ machineId: 'portal', role: run.role, userId: run.personId, sessionId: run.sessionId }, { claude: true, usageOf: ctx.usageOf, liveOn: ctx.liveOn, limits: ctx.limits }); + for (const p of s.problems) ctx.onProblem?.(p); + if (s.claude) { + const all = { ...process.env, ...cfg.claudeEnv }; + return { env: { ...usageEnv(all), CLAUDE_CODE_OAUTH_TOKEN: s.claude.token }, account: poolAccount(s.claude, run.personId) }; + } + if (s.claudeHold) throw new PoolHeldError(`${run.personId}'s Claude token pool is ${run.role === 'orchestrator' ? 'used up' : 'held'}: ${s.claudeHold.why}`, s.claudeHold.next); + return plain(); +} + +// ---------------------------------------------------------------- the dispatcher's reserve (w739) + +/** A credential kept back for one role: the dispatcher's, and the host token's. */ +export interface ReservedCredential { + /** The usage meters' key (tokenKey), or HOST_LOGIN. */ + key: string; + /** "the dispatcher's token file …dAAA" */ + what: string; + role: 'dispatcher' | 'host-token'; +} + +/** + * The credentials with a reserve, worked out from config now (never a fixed token): whatever claudeAccounts.dispatcher + * resolves to (the token file today, else the host token or this host's login), and the host token + * (claudeEnv.CLAUDE_CODE_OAUTH_TOKEN) that machine agents and the people with no pool yet run on. When they are one + * credential it is the dispatcher's. One token is one account: no matching of two tokens to one account. + */ +export function reservedCredentials(cfg: Pick & Partial>): ReservedCredential[] { + const out: ReservedCredential[] = []; + const a = hostAccount(cfg, 'dispatcher'); + const hostTok = cfg.claudeEnv?.CLAUDE_CODE_OAUTH_TOKEN; + if (a === 'tokenfile') { + const t = tokenFileToken(cfg); + if (t) out.push({ key: tokenKey(t), what: `the dispatcher's token file …${t.slice(-4)}`, role: 'dispatcher' }); + } else if (a === 'token') { + if (hostTok) out.push({ key: tokenKey(hostTok), what: `the dispatcher's host token …${hostTok.slice(-4)}`, role: 'dispatcher' }); + } else if (a === 'login') out.push({ key: HOST_LOGIN, what: "the dispatcher's host login", role: 'dispatcher' }); + if (hostTok && !out.some((c) => c.key === tokenKey(hostTok))) out.push({ key: tokenKey(hostTok), what: `the host token …${hostTok.slice(-4)}`, role: 'host-token' }); + return out; +} + +export interface ReserveLine { + cred: ReservedCredential; + reserve: Reserve; + /** The status line: "dispatcher token …dAAA: weekly 41%, reserve 25% (5 days to reset); 5-hour 12%, reserve 20%". */ + line: string; + /** Set when runs other than the owner's are live on it while it is inside its reserve: the dashboard's banner. */ + warning?: string; +} + +/** + * system_status's line for each reserved credential and the dashboard's warning. The reserve is the dispatcher's: it may + * run into it up to 100%, nothing here ever holds the dispatcher. Other runs landing on it (a person with no vault tokens + * yet, whose orchestrator is on the token file) are not refused, because they have nothing else: the warning says they + * are using the buffer. `othersOn`: how many live sessions other than the owner's run on a credential key. + */ +export function reserveLines( + cfg: Pick & Partial>, + usageOf: (key: string) => PlanUsage | undefined, + othersOn: (key: string, role: ReservedCredential['role']) => number, + now = Date.now(), +): ReserveLine[] { + const limits: PoolLimits = poolLimits(cfg); + return reservedCredentials(cfg).map((cred) => { + const u = usageOf(cred.key); + const r = reserveOf(u, limits, now); + const w = u?.weekly; + const sess = u?.session; + const name = cred.what.replace(/^the /, ''); + const nums = r.noReading + ? 'no reading yet (counted as inside its reserve)' + : `weekly ${Math.round(w?.percent ?? 50)}%, reserve ${Math.round(r.weekly)}% (${r.daysLeft.toFixed(r.daysLeft < 1 ? 1 : 0)} day${Math.round(r.daysLeft) === 1 ? '' : 's'} to reset); 5-hour ${Math.round(sess?.percent ?? 50)}%, reserve ${Math.round(r.session)}%`; + const others = r.inReserve ? othersOn(cred.key, cred.role) : 0; + const who = cred.role === 'dispatcher' ? 'the dispatcher' : 'its own roles'; + return { + cred, + reserve: r, + line: `${name}: ${nums}${r.inReserve ? `; inside its reserve, kept for ${who}` : ''}${others ? `; ${others} other run${others === 1 ? '' : 's'} using it` : ''}`, + ...(others ? { warning: `${name} is the last token with room for ${others} run${others === 1 ? '' : 's'} and is inside the buffer kept for ${who}${w ? ` (weekly ${Math.round(w.percent)}%, reserve ${Math.round(r.weekly)}%, resets ${clock(w.resetsAt)})` : ''}` } : {}), + }; + }); +} + /** * system_status's lines on which account each kind of agent runs on (docs/accounts.md), safe to show: this * host's roles, then every machine. `hostName`: this host, `hostToken`: the token its agents would get (usage.ts @@ -322,7 +442,9 @@ export function machineRunEnv( export function accountSetupLines(cfg: Pick & Partial>, hostName: string, hostToken: string | undefined, machineIds: MachineRef[], people: string[] = []): string[] { const fileToken = tokenFileToken(cfg); const here = (role: HostRole) => - hostAccount(cfg, role) === 'tokenfile' + hostAccount(cfg, role) === 'vault' + ? `each person's own vault pool (${fileToken ? `token file …${fileToken.slice(-4)}` : hostToken ? `host token …${hostToken.slice(-4)}` : 'no token'} for a person with no vault token)` + : hostAccount(cfg, role) === 'tokenfile' ? fileToken ? `token file …${fileToken.slice(-4)}` : 'token file (UNREADABLE: its sessions will not start)' diff --git a/server/tokenPool.test.ts b/server/tokenPool.test.ts new file mode 100644 index 00000000..5f09f9ec --- /dev/null +++ b/server/tokenPool.test.ts @@ -0,0 +1,188 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { POOL_DEFAULTS, bySoonestReset, checkPoolConfig, clock, heldReason, judge, metersOf, pickPool, poolLimits, reserveOf, type PoolToken } from './tokenPool.ts'; +import type { PlanUsage } from '../shared/types.ts'; + +const NOW = Date.parse('2026-10-09T12:00:00Z'); +const DAY = 86_400_000; +const iso = (ms: number) => new Date(ms).toISOString(); + +/** A token with meters: session/weekly percent, resets in hours (session) and days (weekly) from NOW. */ +const tok = (name: string, o: { session?: number; weekly?: number; sessionHours?: number; weeklyDays?: number; others?: number; none?: boolean; owner?: string } = {}): PoolToken => ({ + id: name, + name, + fingerprint: name, + owner: o.owner ?? 'ben', + others: o.others ?? 0, + usage: o.none + ? undefined + : ({ + available: true, + asOf: iso(NOW), + models: [], + session: { label: '5-hour', percent: o.session ?? 10, ...(o.sessionHours !== undefined ? { resetsAt: iso(NOW + o.sessionHours * 3_600_000) } : {}) }, + weekly: { label: 'weekly', percent: o.weekly ?? 20, ...(o.weeklyDays !== undefined ? { resetsAt: iso(NOW + o.weeklyDays * DAY) } : {}) }, + } satisfies PlanUsage), +}); + +const L = POOL_DEFAULTS; + +test('pool limits: defaults 80 / 95 / 99 and 5% a day + 20%, config overrides, bad values refused', () => { + assert.deepEqual(poolLimits(undefined), { sessionHold: 80, onePerWeekly: 95, retireWeekly: 99, reservePerDay: 5, reserveSession: 20 }); + assert.equal(poolLimits({ vault: { pool: { sessionHoldPercent: 70, reservePerDayPercent: 8 } } }).sessionHold, 70); + assert.equal(poolLimits({ vault: { pool: { sessionHoldPercent: 70, reservePerDayPercent: 8 } } }).reservePerDay, 8); + assert.equal(poolLimits({ vault: { pool: { sessionHoldPercent: 700 } } }).sessionHold, 80, 'a value out of range is the default'); + assert.equal(checkPoolConfig(undefined), undefined); + assert.equal(checkPoolConfig({ sessionHoldPercent: 0, retireWeeklyPercent: 100 }), undefined, '0 and 100 are allowed'); + assert.match(checkPoolConfig({ sessionHoldPercent: 101 }) ?? '', /percent from 0 to 100/); + assert.match(checkPoolConfig({ retireWeeklyPercent: -1 }) ?? '', /percent from 0 to 100/); + assert.match(checkPoolConfig({ onePerWeeklyPercent: 'x' }) ?? '', /percent from 0 to 100/); + assert.match(checkPoolConfig({ onePerWeeklyPercent: 99.5 }) ?? '', /above retireWeeklyPercent/, '99.5 is above the default 99'); + assert.match(checkPoolConfig({ onePerWeeklyPercent: 90, retireWeeklyPercent: 85 }) ?? '', /above retireWeeklyPercent/); + assert.match(checkPoolConfig({ nope: 1 }) ?? '', /not a setting/); + assert.match(checkPoolConfig([] as never) ?? '', /is an object/); + // A hand-edited file that breaks the order is read safely. + assert.equal(poolLimits({ vault: { pool: { onePerWeeklyPercent: 97, retireWeeklyPercent: 90 } } }).onePerWeekly, 90); +}); + +test('judge: ok, held at the 5-hour limit, one at a time at 95% weekly, retired at 99%, exhausted at 100, no numbers counted as 50%', () => { + assert.equal(judge(tok('a', { session: 79.9, weekly: 94.9 }), L).state, 'ok'); + const held = judge(tok('a', { session: 80, weekly: 10, sessionHours: 2 }), L); + assert.equal(held.state, 'held'); + assert.match(held.why, /5-hour 80% \(>= 80\): held until \w{3} \d\d:\d\dZ/); + assert.equal(held.until, iso(NOW + 2 * 3_600_000)); + assert.equal(judge(tok('a', { weekly: 95, others: 0 }), L).state, 'ok', 'one at a time: the first process is fine'); + const one = judge(tok('a', { weekly: 95, others: 1 }), L); + assert.equal(one.state, 'one-at-a-time'); + assert.match(one.why, /1 running/); + assert.equal(judge(tok('a', { weekly: 99, weeklyDays: 2 }), L).state, 'retired'); + assert.equal(judge(tok('a', { weekly: 99 }), L).until, undefined); + assert.equal(judge(tok('a', { weekly: 100, weeklyDays: 2 }), L).state, 'exhausted'); + assert.equal(judge(tok('a', { session: 100, sessionHours: 1 }), L).state, 'exhausted'); + // Retired outranks held and one-at-a-time; used up outranks retired. + assert.equal(judge(tok('a', { session: 85, weekly: 99.5 }), L).state, 'retired'); + const none = judge(tok('a', { none: true }), L); + assert.equal(none.state, 'ok'); + assert.equal(none.known, false); + assert.equal(none.session, 50); + assert.match(none.why, /no numbers yet/); + assert.deepEqual(metersOf({ available: false, asOf: '', models: [] }), { session: 50, weekly: 50, known: false }); + // The limits are the config's. + const strict = { ...L, sessionHold: 50 }; + assert.equal(judge(tok('a', { session: 60 }), strict).state, 'held'); +}); + +test('order: the weekly reset that comes first, then the 5-hour reset, then the name; no reset time last', () => { + const a = tok('a', { weeklyDays: 5, sessionHours: 4 }); + const b = tok('b', { weeklyDays: 2, sessionHours: 4 }); + const c = tok('c', { weeklyDays: 2, sessionHours: 1 }); + const d = tok('d', { none: true }); + const e = tok('e'); + assert.deepEqual([a, b, c, d, e].sort(bySoonestReset).map((t) => t.name), ['c', 'b', 'a', 'd', 'e'], 'c before b on the 5-hour tie-break; the ones with no reset last, by name'); + assert.equal(bySoonestReset(tok('x', { weeklyDays: 1 }), tok('y', { weeklyDays: 1 })), -1, 'equal resets: the name'); +}); + +test('pickPool: the soonest weekly reset is drained first; a token past its cap is skipped for the next', () => { + const tokens = [tok('late', { weeklyDays: 6 }), tok('soon', { weeklyDays: 1 }), tok('mid', { weeklyDays: 3 })]; + const p = pickPool(tokens, { orchestrator: false, limits: L }); + assert.equal(p.token?.name, 'soon'); + assert.equal(p.how, 'normal'); + // The soonest one reaches the session cap: the next soonest. + tokens[1] = tok('soon', { weeklyDays: 1, session: 82, sessionHours: 3 }); + assert.equal(pickPool(tokens, { orchestrator: false, limits: L }).token?.name, 'mid'); + // The window resets (its meter is low again): the soonest again. + tokens[1] = tok('soon', { weeklyDays: 1, session: 5, sessionHours: 5 }); + assert.equal(pickPool(tokens, { orchestrator: false, limits: L }).token?.name, 'soon'); +}); + +test('pickPool: one at a time at 95% weekly; retired at 99%; a token with no numbers still serves, last', () => { + const busy = tok('busy', { weekly: 96, weeklyDays: 1, others: 1 }); + const other = tok('other', { weekly: 30, weeklyDays: 4 }); + assert.equal(pickPool([busy, other], { orchestrator: false, limits: L }).token?.name, 'other', 'busy has a process: the next one'); + assert.equal(pickPool([tok('busy', { weekly: 96, weeklyDays: 1, others: 0 }), other], { orchestrator: false, limits: L }).token?.name, 'busy', 'idle at 96%: it is the soonest, and may take one process'); + assert.equal(pickPool([tok('r', { weekly: 99, weeklyDays: 1 }), other], { orchestrator: false, limits: L }).token?.name, 'other', 'retired'); + const p = pickPool([tok('fresh', { none: true }), tok('known', { weeklyDays: 5 })], { orchestrator: false, limits: L }); + assert.equal(p.token?.name, 'known', 'the one with a reset time first'); + assert.equal(pickPool([tok('fresh', { none: true })], { orchestrator: false, limits: L }).token?.name, 'fresh', 'a token with no numbers yet is served'); +}); + +test('pickPool: a session keeps its token while it passes, and moves off when it does not', () => { + const a = tok('a', { weeklyDays: 5 }); + const b = tok('b', { weeklyDays: 1 }); + assert.equal(pickPool([a, b], { orchestrator: false, limits: L, sticky: 'a' }).token?.name, 'a', 'sticky: a, though b resets sooner'); + const capped = tok('a', { weeklyDays: 5, session: 85, sessionHours: 2 }); + assert.equal(pickPool([capped, b], { orchestrator: false, limits: L, sticky: 'a' }).token?.name, 'b', 'past the 5-hour cap: moved to the next'); + const retired = tok('a', { weeklyDays: 5, weekly: 99 }); + assert.equal(pickPool([retired, b], { orchestrator: false, limits: L, sticky: 'a' }).token?.name, 'b'); + // One at a time counts the other sessions only: the sticky session's own process is not "another". + const oneSelf = tok('a', { weeklyDays: 5, weekly: 96, others: 0 }); + assert.equal(pickPool([oneSelf, b], { orchestrator: false, limits: L, sticky: 'a' }).token?.name, 'a'); + assert.equal(pickPool([tok('a', { weeklyDays: 5, weekly: 96, others: 1 }), b], { orchestrator: false, limits: L, sticky: 'a' }).token?.name, 'b'); +}); + +test('pickPool: every token capped holds a worker with the reason and the first reset; the orchestrator uses the pool up', () => { + const a = tok('a', { session: 90, sessionHours: 2, weekly: 40, weeklyDays: 3 }); + const b = tok('b', { session: 85, sessionHours: 4, weekly: 30, weeklyDays: 5 }); + const worker = pickPool([a, b], { orchestrator: false, limits: L }); + assert.equal(worker.token, undefined); + assert.match(worker.held?.why ?? '', /2 tokens: all over their caps or used up, the first frees up \w{3} \d\d:\d\dZ/); + assert.equal(worker.held?.next, iso(NOW + 2 * 3_600_000), 'the first token to free up'); + // The orchestrator still runs: the token with the most room (b: max meter 85 vs a: 90), over its cap. + const orch = pickPool([a, b], { orchestrator: true, limits: L }); + assert.equal(orch.token?.name, 'b'); + assert.equal(orch.how, 'over-cap'); + // Equal room: the soonest reset. + const x = tok('x', { session: 90, weeklyDays: 4, sessionHours: 1 }); + const y = tok('y', { session: 90, weeklyDays: 2, sessionHours: 1 }); + assert.equal(pickPool([x, y], { orchestrator: true, limits: L }).token?.name, 'y'); + // One at a time does not bind the orchestrator; retired does not stop it; both are "capped". + const one = tok('one', { weekly: 97, weeklyDays: 2, others: 3 }); + assert.equal(pickPool([one], { orchestrator: true, limits: L }).token?.name, 'one'); + assert.equal(pickPool([tok('ret', { weekly: 99.5, weeklyDays: 2 })], { orchestrator: true, limits: L }).how, 'over-cap'); + // Its own sticky token stays while capped. + assert.equal(pickPool([a, b], { orchestrator: true, limits: L, sticky: 'a' }).token?.name, 'a'); +}); + +test('pickPool: used up is used up for everyone; a pool with one token left serves the orchestrator', () => { + const dead = tok('dead', { weekly: 100, weeklyDays: 2 }); + const dead2 = tok('dead2', { session: 100, sessionHours: 1 }); + for (const orchestrator of [false, true]) { + const p = pickPool([dead, dead2], { orchestrator, limits: L }); + assert.equal(p.token, undefined, `orchestrator=${orchestrator}`); + assert.match(p.held?.why ?? '', /all used up/); + assert.equal(p.held?.next, iso(NOW + 1 * 3_600_000), 'the 5-hour window frees first'); + } + const capped = tok('capped', { session: 91, sessionHours: 1 }); + assert.equal(pickPool([dead, capped], { orchestrator: true, limits: L }).token?.name, 'capped'); + assert.equal(pickPool([dead, capped], { orchestrator: false, limits: L }).token, undefined); + assert.equal(pickPool([], { orchestrator: true, limits: L }).token, undefined, 'no tokens: the caller decides (the transition)'); + assert.match(heldReason([dead], new Map([[dead.id, judge(dead, L)]])).why, /its token is all used up/); + assert.equal(clock(undefined), 'its next reset'); + assert.equal(clock('2026-10-09T14:05:00Z'), 'Fri 14:05Z'); +}); + +test("the dispatcher's reserve: 5% a day to the weekly reset and 20% of the 5-hour window, 0 at the reset", () => { + const u = (weekly: number, session: number, days?: number): PlanUsage => ({ available: true, asOf: '', models: [], weekly: { label: 'w', percent: weekly, ...(days !== undefined ? { resetsAt: iso(NOW + days * DAY) } : {}) }, session: { label: 's', percent: session } }); + const r7 = reserveOf(u(41, 12, 7), L, NOW); + assert.equal(r7.weekly, 35); + assert.equal(r7.session, 20); + assert.equal(r7.inReserve, false); + assert.equal(reserveOf(u(41, 12, 5), L, NOW).weekly, 25); + assert.equal(reserveOf(u(41, 12, 1), L, NOW).weekly, 5); + assert.equal(reserveOf(u(41, 12, 0), L, NOW).weekly, 0); + assert.equal(reserveOf(u(41, 12, 9), L, NOW).weekly, 35, 'never more than a week'); + assert.equal(reserveOf(u(41, 12, -1), L, NOW).weekly, 0, 'a reset in the past: nothing kept'); + assert.equal(reserveOf(u(41, 12), L, NOW).weekly, 35, 'no reset time: the whole week'); + // Inside the reserve: weekly at or past 100 - reserve, or the 5-hour window at or past 80. + assert.equal(reserveOf(u(65, 12, 7), L, NOW).inReserve, true, '65 >= 100 - 35'); + assert.equal(reserveOf(u(64, 12, 7), L, NOW).inReserve, false); + assert.equal(reserveOf(u(64, 12, 1), L, NOW).inReserve, false, 'near the reset the reserve is small'); + assert.equal(reserveOf(u(96, 12, 1), L, NOW).inReserve, true); + assert.equal(reserveOf(u(10, 80, 7), L, NOW).inReserve, true, 'the 5-hour side'); + // No reading yet: counted as inside the reserve. + const none = reserveOf(undefined, L, NOW); + assert.equal(none.noReading, true); + assert.equal(none.inReserve, true); + // The per-day figure is the config's. + assert.equal(reserveOf(u(41, 12, 5), { ...L, reservePerDay: 2 }, NOW).weekly, 10); +}); diff --git a/server/tokenPool.ts b/server/tokenPool.ts new file mode 100644 index 00000000..87f4714e --- /dev/null +++ b/server/tokenPool.ts @@ -0,0 +1,237 @@ +// The Claude token pool (w739, docs/vault.md section 4): which of a person's tokens a run gets, by the plan meters. +// +// Pure functions, no I/O: server/vault.ts feeds them the entries, the meters and the live counts; server/secrets.ts +// decides who may be held. The rules (lothsahn, 2026-10-08/09): +// - a person's runs use that person's own tokens only (their pool): no shared Claude token, no one else's; +// - among the pool, the token whose WEEKLY window resets soonest first, the 5-hour reset as the tie-break; +// - 5-hour meter at or above `sessionHold` (80): no new process on it until that window resets; +// - weekly at or above `onePerWeekly` (95): one live process at a time; +// - weekly at or above `retireWeekly` (99): not handed out until the weekly window resets; +// - a meter at 100 (or a rejection): used up, handed to nobody until it resets; +// - a person's own orchestrator, when every pool token is past its caps but none is used up, still runs on the one with +// the most room until it is used up: the caps keep workers from eating the person's own use, they do not lock them out. +// A token with no numbers yet counts as 50% on both meters: it passes every rule and ranks after the ones with a reset. +// +// The dispatcher's reserve (reserveOf) is a different thing: the credential the dispatcher runs on keeps a buffer that +// shrinks as its weekly window runs out, for the dispatcher alone. + +import type { PlanUsage } from '../shared/types.ts'; + +/** The system-wide limits, percent (config vault.pool.*). */ +export interface PoolLimits { + /** 5-hour meter at or above: no new process on the token. */ + sessionHold: number; + /** Weekly meter at or above: one live process at a time. */ + onePerWeekly: number; + /** Weekly meter at or above: the token is not handed out until the weekly window resets. */ + retireWeekly: number; + /** The dispatcher's token keeps this many percent of the weekly window per day left until that window resets. */ + reservePerDay: number; + /** ... and this many percent of the 5-hour window. */ + reserveSession: number; +} + +export const POOL_DEFAULTS: Readonly = { sessionHold: 80, onePerWeekly: 95, retireWeekly: 99, reservePerDay: 5, reserveSession: 20 }; + +/** Config vault.pool: the same, with the unit in the name. Anything unset is the default. */ +export interface PoolConfig { + sessionHoldPercent?: number; + onePerWeeklyPercent?: number; + retireWeeklyPercent?: number; + reservePerDayPercent?: number; + reserveSessionPercent?: number; +} + +export const POOL_CONFIG_KEYS = ['sessionHoldPercent', 'onePerWeeklyPercent', 'retireWeeklyPercent', 'reservePerDayPercent', 'reserveSessionPercent'] as const; + +const pct = (v: unknown, d: number) => (typeof v === 'number' && Number.isFinite(v) && v >= 0 && v <= 100 ? v : d); + +/** The limits in effect, read at every pick: a change applies to the next process, with no restart. */ +export function poolLimits(cfg: { vault?: { pool?: PoolConfig } } | undefined): PoolLimits { + const p = cfg?.vault?.pool; + const out: PoolLimits = { + sessionHold: pct(p?.sessionHoldPercent, POOL_DEFAULTS.sessionHold), + onePerWeekly: pct(p?.onePerWeeklyPercent, POOL_DEFAULTS.onePerWeekly), + retireWeekly: pct(p?.retireWeeklyPercent, POOL_DEFAULTS.retireWeekly), + reservePerDay: pct(p?.reservePerDayPercent, POOL_DEFAULTS.reservePerDay), + reserveSession: pct(p?.reserveSessionPercent, POOL_DEFAULTS.reserveSession), + }; + // A hand-edited config.json that breaks the order is read as the retire limit for both (checkPoolConfig refuses it at load). + if (out.onePerWeekly > out.retireWeekly) out.onePerWeekly = out.retireWeekly; + return out; +} + +/** Why a vault.pool block is not usable, or undefined (config load, set_app_config). */ +export function checkPoolConfig(p: unknown): string | undefined { + if (p === undefined) return undefined; + if (!p || typeof p !== 'object' || Array.isArray(p)) return `config vault.pool is an object: ${POOL_CONFIG_KEYS.join(', ')}`; + const o = p as Record; + for (const k of Object.keys(o)) if (!(POOL_CONFIG_KEYS as readonly string[]).includes(k)) return `config vault.pool.${k} is not a setting (${POOL_CONFIG_KEYS.join(', ')})`; + for (const k of POOL_CONFIG_KEYS) { + const v = o[k]; + if (v !== undefined && !(typeof v === 'number' && Number.isFinite(v) && v >= 0 && v <= 100)) return `config vault.pool.${k} is a percent from 0 to 100`; + } + const one = (o.onePerWeeklyPercent as number | undefined) ?? POOL_DEFAULTS.onePerWeekly; + const retire = (o.retireWeeklyPercent as number | undefined) ?? POOL_DEFAULTS.retireWeekly; + if (one > retire) return `config vault.pool.onePerWeeklyPercent (${one}) is above retireWeeklyPercent (${retire}): one at a time must start before a token is retired`; + return undefined; +} + +// ---------------------------------------------------------------- one token + +/** What the pick knows of one Claude token. */ +export interface PoolToken { + id: string; + name: string; + fingerprint: string; + /** The person whose pool it is. */ + owner?: string; + /** Its account's plan meters, when polled. */ + usage?: PlanUsage; + /** Live processes on it, not counting the session that asks. */ + others: number; +} + +export type PoolState = 'ok' | 'held' | 'one-at-a-time' | 'retired' | 'exhausted'; + +export interface PoolView { + state: PoolState; + /** One line, no token: "5-hour 83% (>= 80), held until 14:05Z". */ + why: string; + session: number; + weekly: number; + /** False when there are no numbers (the 50% default). */ + known: boolean; + sessionResetsAt?: string; + weeklyResetsAt?: string; + /** When the state ends by itself (the window that holds it resets), ISO. */ + until?: string; +} + +/** The meters of a token, or the 50% default. */ +export function metersOf(u: PlanUsage | undefined): { session: number; weekly: number; known: boolean; sessionResetsAt?: string; weeklyResetsAt?: string } { + if (!u?.available) return { session: 50, weekly: 50, known: false }; + const s = u.session?.percent; + const w = u.weekly?.percent; + const known = typeof s === 'number' || typeof w === 'number'; + return { session: typeof s === 'number' ? s : 50, weekly: typeof w === 'number' ? w : 50, known, sessionResetsAt: u.session?.resetsAt, weeklyResetsAt: u.weekly?.resetsAt }; +} + +/** "Fri 14:05Z" for a reset time, or "its next reset". */ +export function clock(iso: string | undefined): string { + const t = iso ? Date.parse(iso) : NaN; + if (!Number.isFinite(t)) return 'its next reset'; + const d = new Date(t); + return `${['Sun', 'Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat'][d.getUTCDay()]} ${d.toISOString().slice(11, 16)}Z`; +} + +/** The state of a token under the limits, for a worker, a standing agent or the ops worker (the strictest reading). */ +export function judge(t: PoolToken, limits: PoolLimits): PoolView { + const m = metersOf(t.usage); + const base = { session: m.session, weekly: m.weekly, known: m.known, sessionResetsAt: m.sessionResetsAt, weeklyResetsAt: m.weeklyResetsAt }; + const numbers = `5-hour ${Math.round(m.session)}%, weekly ${Math.round(m.weekly)}%`; + if (m.weekly >= 100 || m.session >= 100) { + const until = m.weekly >= 100 ? m.weeklyResetsAt : m.sessionResetsAt; + return { ...base, state: 'exhausted', why: `${numbers}: used up until ${clock(until)}`, ...(until ? { until } : {}) }; + } + if (m.weekly >= limits.retireWeekly) return { ...base, state: 'retired', why: `weekly ${Math.round(m.weekly)}% (>= ${limits.retireWeekly}): retired until ${clock(m.weeklyResetsAt)}`, ...(m.weeklyResetsAt ? { until: m.weeklyResetsAt } : {}) }; + if (m.session >= limits.sessionHold) return { ...base, state: 'held', why: `5-hour ${Math.round(m.session)}% (>= ${limits.sessionHold}): held until ${clock(m.sessionResetsAt)}`, ...(m.sessionResetsAt ? { until: m.sessionResetsAt } : {}) }; + if (m.weekly >= limits.onePerWeekly && t.others >= 1) return { ...base, state: 'one-at-a-time', why: `weekly ${Math.round(m.weekly)}% (>= ${limits.onePerWeekly}): one process at a time, ${t.others} running` }; + return { ...base, state: 'ok', why: m.known ? numbers : 'no numbers yet (counted as 50%)' }; +} + +const ms = (iso: string | undefined) => { + const t = iso ? Date.parse(iso) : NaN; + return Number.isFinite(t) ? t : Infinity; +}; + +/** Soonest weekly reset first, the 5-hour reset next, then the name. A token with no reset time goes last. */ +export function bySoonestReset(a: PoolToken, b: PoolToken): number { + const x = metersOf(a.usage); + const y = metersOf(b.usage); + const wx = ms(x.weeklyResetsAt); + const wy = ms(y.weeklyResetsAt); + if (wx !== wy) return wx < wy ? -1 : 1; + const sx = ms(x.sessionResetsAt); + const sy = ms(y.sessionResetsAt); + if (sx !== sy) return sx < sy ? -1 : 1; + return a.name.localeCompare(b.name); +} + +/** Room left on the fuller meter (100 minus it), for the over-cap order. */ +const room = (t: PoolToken) => { + const m = metersOf(t.usage); + return 100 - Math.max(m.session, m.weekly); +}; + +export interface PoolPick { + /** The token, when the run may start. */ + token?: PoolToken; + /** A normal pick, or the orchestrator's use-it-up of a token past its caps. */ + how?: 'normal' | 'over-cap'; + /** Every token's state, by id. */ + views: Map; + /** The pool has tokens and none may serve this run now: why, and when the first one frees up (ISO). */ + held?: { why: string; next?: string }; +} + +/** + * The token for a run over a person's pool (`tokens` are the pool; the caller keeps other people's out). `orchestrator`: + * a person's own orchestrator, whose pool is used up past the caps. `sticky`: the id the session ran on last, kept while + * it still passes. No tokens at all is the caller's business (the transition: today's account), not a hold. + */ +export function pickPool(tokens: readonly PoolToken[], o: { orchestrator: boolean; limits: PoolLimits; sticky?: string }): PoolPick { + const views = new Map(tokens.map((t) => [t.id, judge(t, o.limits)] as const)); + const ok = (t: PoolToken) => views.get(t.id)!.state === 'ok'; + const kept = o.sticky ? tokens.find((t) => t.id === o.sticky && ok(t)) : undefined; + const best = kept ?? [...tokens].filter(ok).sort(bySoonestReset)[0]; + if (best) return { token: best, how: 'normal', views }; + if (o.orchestrator) { + const capped = tokens.filter((t) => ['held', 'one-at-a-time', 'retired'].includes(views.get(t.id)!.state)); + const keptCapped = o.sticky ? capped.find((t) => t.id === o.sticky) : undefined; + const use = keptCapped ?? [...capped].sort((a, b) => room(b) - room(a) || bySoonestReset(a, b))[0]; + if (use) return { token: use, how: 'over-cap', views }; + } + return { views, held: heldReason(tokens, views) }; +} + +/** Why no token of a non-empty pool serves a run, and when the first frees up. */ +export function heldReason(tokens: readonly PoolToken[], views: ReadonlyMap): { why: string; next?: string } { + const list = tokens.map((t) => views.get(t.id)!); + const nexts = list.map((v) => v.until).filter((x): x is string => !!x).sort((a, b) => ms(a) - ms(b)); + const next = nexts[0]; + const states = [...new Set(list.map((v) => v.state))]; + const all = states.length === 1 && states[0] === 'exhausted' ? 'all used up' : 'all over their caps or used up'; + return { why: `${tokens.length === 1 ? 'its token is' : `${tokens.length} tokens:`} ${all}${next ? `, the first frees up ${clock(next)}` : ''}`, ...(next ? { next } : {}) }; +} + +// ---------------------------------------------------------------- the dispatcher's reserve + +export interface Reserve { + /** Percent of the weekly window kept back: reservePerDay x days left to the reset (all of it, 7 days, when the reset is unknown). */ + weekly: number; + /** Percent of the 5-hour window kept back. */ + session: number; + daysLeft: number; + /** The meters have never been read: counted as in the reserve (the dispatcher's token before its first reading). */ + noReading: boolean; + /** Either meter has reached the buffer: runs other than the dispatcher's are eating into it. */ + inReserve: boolean; +} + +/** + * The buffer a reserved credential keeps for its own role: reservePerDay percent of the weekly window for each day + * left until that window resets (35 with 7 days, 5 with one day, 0 at the reset), and reserveSession percent of the + * 5-hour window. Others may use the credential only while weekly < 100 - weekly and session < 100 - session. The role + * the reserve is for is never held by it: it may run into the reserve up to 100%. + */ +export function reserveOf(u: PlanUsage | undefined, limits: PoolLimits, now: number): Reserve { + const m = metersOf(u); + const reset = ms(m.weeklyResetsAt); + const daysLeft = Number.isFinite(reset) ? Math.min(7, Math.max(0, (reset - now) / 86_400_000)) : 7; + const weekly = Math.min(100, limits.reservePerDay * daysLeft); + const session = Math.min(100, limits.reserveSession); + const noReading = !m.known; + const inReserve = noReading || m.weekly >= 100 - weekly || m.session >= 100 - session; + return { weekly, session, daysLeft, noReading, inReserve }; +} diff --git a/server/usage.test.ts b/server/usage.test.ts index 918dbc11..2c38a448 100644 --- a/server/usage.test.ts +++ b/server/usage.test.ts @@ -686,3 +686,71 @@ test('tracker: one poll at start, then one every usagePollMinutes; Refresh polls await settle(); assert.equal(calls, 6); }); + +// ---------------------------------------------------------------- w739: tokens without user:profile, near-limit polling, rate-limit events + +test('tracker: a token without the user:profile scope (403) is read from the rate-limit headers, and the endpoint is not asked again for hours', async () => { + // Every `claude setup-token` token answers the usage endpoint "403 oauth_scope_insufficient"; FFBox reads the same two windows + // off a /v1/messages reply (ffbox scripts/claude_keys.py, measured 2026-09-04 there). + const { t, calls, logs } = trackerWith( + async () => { + throw new UsageFetchError('the token may not read usage (HTTP 403: OAuth token does not meet scope requirement user:profile)', undefined, 403); + }, + async () => ({ rate_limits_available: true, rate_limits: { five_hour: { utilization: 33, resets_at: '2026-09-28T04:40:00+00:00' }, seven_day: { utilization: 41, resets_at: '2026-10-02T03:00:00+00:00' } } }), + ); + await t.refresh(); + const u = t.entries.get(tokenKey(TOKEN))!.usage!; + assert.deepEqual([u.available, u.weekly?.percent, u.session?.percent, u.source], [true, 41, 33, 'rate-limit headers']); + assert.equal(u.weekly?.resetsAt, '2026-10-02T03:00:00+00:00'); + assert.deepEqual([calls.token.length, calls.limits.length], [1, 1]); + assert.match(logs.find((l) => l.includes(tokenKey(TOKEN))) ?? '', /via the API's rate-limit headers \(the usage endpoint needs the user:profile scope|via the API's rate-limit headers \(the token may not read usage/); + await t.refresh(); + assert.deepEqual([calls.token.length, calls.limits.length], [1, 2], 'the endpoint is not asked again; the headers are'); + assert.ok(![...logs, JSON.stringify([...t.entries])].some((l) => l.includes(TOKEN.slice(13, 40))), 'no token in the log or the numbers'); +}); + +test('tracker: a token near a pool limit is polled alone between the regular polls; one far from every limit is not', async () => { + let session = 33; + const { t, calls } = trackerWith(async () => ({ rate_limits_available: true, rate_limits: endpointBody(41, session, 0) })); + await t.refresh(); + assert.deepEqual(t.nearTokens(), [], '33% of the 5-hour window and 41% of the week: far'); + await t.refreshNear(); + assert.equal(calls.token.length, 1, 'nothing near, nothing polled'); + session = 61; + await t.refresh(); + assert.deepEqual(t.nearTokens().map((x) => tokenKey(x.token)), [tokenKey(TOKEN)], '61% of the 5-hour window: near'); + const logins = calls.login.length; + session = 64; + await t.refreshNear(); + assert.equal(calls.token.length, 3, 'the near token alone'); + assert.equal(calls.login.length, logins, "the host login's CLI is not run for it"); + assert.equal(t.entries.get(tokenKey(TOKEN))!.usage!.session?.percent, 64); + // 90% of the week is near too. + session = 10; + const weekly = trackerWith(async () => ({ rate_limits_available: true, rate_limits: endpointBody(91, 10, 0) })); + await weekly.t.refresh(); + assert.equal(weekly.t.nearTokens().length, 1); +}); + +test("tracker: a session's rate_limit_event raises its token's meter and never lowers it; the next poll replaces it", async () => { + const { t } = trackerWith(async () => ({ rate_limits_available: true, rate_limits: endpointBody(41, 33, 0) })); + await t.refresh(); + const key = tokenKey(TOKEN); + const sess = () => t.entries.get(key)!.usage!.session!.percent; + const week = () => t.entries.get(key)!.usage!.weekly!.percent; + t.noteRateLimit(key, { rateLimitType: 'five_hour', utilization: 0.5, resetsAt: Date.parse('2026-09-28T05:00:00Z') / 1000 }); + assert.equal(sess(), 50, 'a fraction, as the headers carry it'); + assert.equal(t.entries.get(key)!.usage!.session!.resetsAt, '2026-09-28T05:00:00.000Z'); + t.noteRateLimit(key, { rateLimitType: 'five_hour', utilization: 0.2 }); + assert.equal(sess(), 50, 'never lowered between polls'); + t.noteRateLimit(key, { rateLimitType: 'seven_day', utilization: 85 }); + assert.equal(week(), 85, 'a value above 1 is a percent'); + for (const bad of [{ rateLimitType: 'seven_day_opus', utilization: 0.99 }, { rateLimitType: 'five_hour' }, { rateLimitType: 'five_hour', utilization: NaN }, { utilization: 0.99 }]) t.noteRateLimit(key, bad); + assert.deepEqual([sess(), week()], [50, 85]); + t.noteRateLimit(HOST_LOGIN, { rateLimitType: 'five_hour', utilization: 0.99 }); + t.noteRateLimit(undefined, { rateLimitType: 'five_hour', utilization: 0.99 }); + t.noteRateLimit('token:unknown', { rateLimitType: 'five_hour', utilization: 0.99 }); + assert.equal(t.entries.get(HOST_LOGIN)!.usage!.session?.percent, 0, "a login's meters are not touched"); + await t.refresh(); + assert.equal(sess(), 33, 'the poll replaces the floor'); +}); diff --git a/server/usage.ts b/server/usage.ts index 83ee3e38..e5c6f89d 100644 --- a/server/usage.ts +++ b/server/usage.ts @@ -479,9 +479,12 @@ export const USAGE_URL = 'https://api.anthropic.com/api/oauth/usage'; /** A token's usage request that failed. `retryAfterMs`: how long the endpoint asked us to wait (HTTP 429). */ export class UsageFetchError extends Error { readonly retryAfterMs?: number; - constructor(message: string, retryAfterMs?: number) { + /** The HTTP status of the failed request, when there was a reply (403: the token lacks user:profile). */ + readonly status?: number; + constructor(message: string, retryAfterMs?: number, status?: number) { super(message); this.retryAfterMs = retryAfterMs; + this.status = status; } } @@ -518,7 +521,7 @@ export async function fetchTokenUsage(token: string, fetchImpl: typeof fetch = f : res.status === 403 ? 'the token may not read usage' : retryAfterMs ? `the usage endpoint rate-limits this token, next try in ${Math.ceil(retryAfterMs / 60_000)} min` : 'the usage endpoint failed'; - throw new UsageFetchError(`${what} (HTTP ${res.status}${detail ? `: ${scrub(detail).slice(0, 150)}` : ''})`, retryAfterMs); + throw new UsageFetchError(`${what} (HTTP ${res.status}${detail ? `: ${scrub(detail).slice(0, 150)}` : ''})`, retryAfterMs, res.status); } let body: UsageReply['rate_limits']; try { @@ -591,6 +594,12 @@ const brief = (u: PlanUsage) => const ANSWER_MS = 75_000; /** A poll still marked running after this long is taken as stuck: the next one goes ahead (it cannot happen while ANSWER_MS holds). */ const STUCK_MS = 5 * 60_000; +/** A token the endpoint refused for its scope is asked again after this long (a re-issued token may carry it). */ +const NO_SCOPE_RETRY_MS = 6 * 3_600_000; +/** A token near a pool limit is polled alone this often, between the regular polls (w739): 5-hour at 60% or weekly at 90%. */ +export const NEAR_POLL_MS = 2 * 60_000; +export const NEAR_SESSION_PERCENT = 60; +export const NEAR_WEEKLY_PERCENT = 90; /** A new token (set_app_config) is polled this soon, and a manual refresh no sooner than this after the last poll. */ const SOON_MS = 5_000; @@ -642,6 +651,13 @@ export class UsageTracker { private readonly answerMs: number; /** Per token key: no request before this time (epoch ms), the endpoint's Retry-After. */ private readonly retryAt = new Map(); + /** + * Per token key: when the usage endpoint answered 403 (the token lacks the user:profile scope, as every `claude + * setup-token` token does), epoch ms. Such a token is read from the API's rate-limit headers (FFBox does the same, + * ffbox scripts/claude_keys.py), and the endpoint is asked again only after NO_SCOPE_RETRY_MS. + */ + private readonly noScope = new Map(); + private nearTimer?: NodeJS.Timeout; constructor(cfg: Config, changed: () => void, deps: UsageDeps = {}) { this.cfg = cfg; @@ -706,11 +722,53 @@ export class UsageTracker { setInterval(() => { if (this.tokensKey() !== this.tokenSeen) this.poke(); }, 10_000).unref(); + // A token near a pool limit is read more often than the regular poll (w739): the meters are the pool's only eyes. + this.nearTimer = setInterval(() => void this.refreshNear(), NEAR_POLL_MS); + this.nearTimer.unref?.(); + } + + /** The tokens that are near a pool limit now (5-hour at NEAR_SESSION_PERCENT or weekly at NEAR_WEEKLY_PERCENT), or have no numbers. */ + nearTokens(): { token: string; label: string }[] { + return this.tokens().filter((x) => { + const u = this.entries.get(tokenKey(x.token))?.usage; + return !u?.available || (u.session?.percent ?? 0) >= NEAR_SESSION_PERCENT || (u.weekly?.percent ?? 0) >= NEAR_WEEKLY_PERCENT; + }); + } + + /** One poll of just the near tokens, between the regular polls; skipped while a regular poll runs. */ + async refreshNear() { + if (this.inFlightSince) return; + const near = this.nearTokens().filter((x) => this.entries.has(tokenKey(x.token))); + if (!near.length) return; + await Promise.all(near.map((x) => this.refreshToken(x.token, x.label))); + this.save(); + this.changed(); + } + + /** + * A running session's rate_limit_event (the SDK's, from the API's rate-limit headers on its own requests): raises the + * token's meter at once, never lowers it (the next poll replaces it). `account` is the session's recorded account key. + * `utilization` is a fraction (the header's scale; a value above 1 is read as a percent). A guess at the event's + * frequency: it is not relied on, only used as a floor between polls. + */ + noteRateLimit(account: string | undefined, info: { rateLimitType?: string; utilization?: number; resetsAt?: number }) { + if (!account?.startsWith('token:')) return; + const entry = this.entries.get(account); + const u = entry?.usage; + if (!entry || !u?.available || typeof info.utilization !== 'number' || !Number.isFinite(info.utilization) || info.utilization < 0) return; + const field = info.rateLimitType === 'five_hour' ? 'session' : info.rateLimitType === 'seven_day' ? 'weekly' : undefined; + const meter = field ? u[field] : undefined; + if (!field || !meter) return; + const percent = info.utilization <= 1 ? info.utilization * 100 : info.utilization; + if (percent <= meter.percent) return; + u[field] = { ...meter, percent: Math.round(percent * 100) / 100, ...(info.resetsAt ? { resetsAt: new Date(info.resetsAt * 1000).toISOString() } : {}) }; + this.changed(); } /** Stop the timers (tests, shutdown). */ stop() { this.started = false; + clearInterval(this.nearTimer); clearTimeout(this.next); clearTimeout(this.soon); this.soon = undefined; @@ -868,6 +926,9 @@ export class UsageTracker { const reason = (e: unknown) => scrub(e instanceof Error ? e.message : String(e)).slice(0, 200); // The usage endpoint's answer, or why it rate-limits this token (now, or still within its Retry-After). const endpoint = async (): Promise => { + const refused = this.noScope.get(key); + if (refused && Date.now() - refused < NO_SCOPE_RETRY_MS) return { limited: 'the usage endpoint needs the user:profile scope, which this token lacks' }; + this.noScope.delete(key); const wait = this.retryAt.get(key); if (wait && wait > Date.now()) return { limited: `the usage endpoint rate-limits this token until ${new Date(wait).toISOString()}` }; this.retryAt.delete(key); @@ -876,6 +937,11 @@ export class UsageTracker { if (!u.available) u.why = `the usage endpoint gave no plan limits for this token (${u.why ?? 'unknown'})`; return u; } catch (e) { + // 403: no user:profile scope (a setup-token): the headers answer instead, and the endpoint is not asked every poll. + if (e instanceof UsageFetchError && e.status === 403) { + this.noScope.set(key, Date.now()); + return { limited: reason(e) }; + } const after = e instanceof UsageFetchError ? e.retryAfterMs : undefined; if (!after) return { available: false, asOf, models: [], why: `usage unknown: ${reason(e)}` }; this.retryAt.set(key, Date.now() + after); diff --git a/server/vault.test.ts b/server/vault.test.ts index 056ca2c1..550ae599 100644 --- a/server/vault.test.ts +++ b/server/vault.test.ts @@ -4,7 +4,7 @@ import { spawnSync } from 'node:child_process'; import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; -import { FULL_PERCENT, NIGHTLY_SENTRY, UNATTRIBUTED_DEFAULTS, Vault, tokenPersonForWork, unattributedKind, unattributedPerson, eligible, type KeySource, envNameProblem, fingerprintOf, headroom, keySource, newKeyText, pickClaude, readKey, valueProblem, vaultStatusLine, type VaultEntryMeta, type VaultContext } from './vault.ts'; +import { NIGHTLY_SENTRY, UNATTRIBUTED_DEFAULTS, Vault, tokenPersonForWork, unattributedKind, unattributedPerson, eligible, type KeySource, envNameProblem, fingerprintOf, keySource, newKeyText, readKey, valueProblem, vaultStatusLine, type VaultEntryMeta, type VaultContext } from './vault.ts'; import { SECRET_ENV, addSecretValues, claudeFromVault, machineRunEnv, redactSecrets, registerSecretValues } from './secrets.ts'; import { GITHUB_HELPER, githubCredentialEnv } from './launch.ts'; import { buildAccounts, tokenKey } from './usage.ts'; @@ -66,14 +66,14 @@ test('vault: what each kind accepts; names and env variables are checked; duplic assert.match(envNameProblem('PATH') ?? '', /ends in _TOKEN/); assert.match(envNameProblem('GIT_ASKPASS_TOKEN') ?? '', /GIT_ variables/); assert.match(envNameProblem('CLAUDE_CODE_OAUTH_TOKEN') ?? '', /kind claude/); - assert.throws(() => v.add({ name: 'Bad Name', kind: 'claude', value: A, share: 'anyone' }), /a name is/); + assert.throws(() => v.add({ name: 'Bad Name', kind: 'claude', value: A, owner: 'ben', share: 'anyone' }), /a name is/); assert.throws(() => v.add({ name: 'x', kind: 'claude', value: A }), /needs --owner/); assert.throws(() => v.add({ name: 'x', kind: 'env', value: DISCORD, share: 'anyone' }), /variable name/); - assert.throws(() => v.add({ name: 'x', kind: 'claude', value: A, share: 'anyone', roles: ['orchestrator' as never] }), /roles are/); - assert.throws(() => v.add({ name: 'x', kind: 'claude', value: A, share: 'anyone', machines: ['M 3'] }), /machines are/); - v.add({ name: 'a', kind: 'claude', value: A, share: 'anyone' }); - assert.throws(() => v.add({ name: 'a', kind: 'claude', value: B, share: 'anyone' }), /exists; rotate/); - assert.throws(() => v.add({ name: 'a2', kind: 'claude', value: A, share: 'anyone' }), /already in the vault as a/); + assert.throws(() => v.add({ name: 'x', kind: 'claude', value: A, owner: 'ben', share: 'anyone', roles: ['orchestrator' as never] }), /roles are/); + assert.throws(() => v.add({ name: 'x', kind: 'claude', value: A, owner: 'ben', share: 'anyone', machines: ['M 3'] }), /machines are/); + v.add({ name: 'a', kind: 'claude', value: A, owner: 'ben', share: 'anyone' }); + assert.throws(() => v.add({ name: 'a', kind: 'claude', value: B, owner: 'ben', share: 'anyone' }), /exists; rotate/); + assert.throws(() => v.add({ name: 'a2', kind: 'claude', value: A, owner: 'ben', share: 'anyone' }), /already in the vault as a/); // No error message ever quotes a value. try { v.add({ name: 'z', kind: 'github', value: A, share: 'anyone' }); @@ -85,19 +85,21 @@ test('vault: what each kind accepts; names and env variables are checked; duplic test('vault: rotate, grant and remove; a rotated value is the one handed out', (t) => { const { make } = setup(t); const v = make(); - v.add({ name: 'a', kind: 'claude', value: A, share: 'anyone' }); + v.add({ name: 'a', kind: 'claude', value: A, owner: 'ben', share: 'anyone' }); const r = v.rotate('a', B); assert.equal(r.fingerprint, fingerprintOf(B)); assert.ok(r.rotatedAt); - const run = { machineId: 'm3', role: 'workers' as const }; + const run = { machineId: 'm3', role: 'workers' as const, userId: 'ben' }; assert.equal(v.forRun(run, { claude: true }).claude?.token, B); const g = v.update('a', { machines: ['M5'], roles: ['standing'] }); assert.deepEqual(g.machines, ['m5']); assert.equal(v.forRun(run, { claude: true }).claude, undefined, 'no longer granted to m3 workers'); - assert.equal(v.forRun({ machineId: 'm5', role: 'standing' }, { claude: true }).claude?.token, B); + assert.equal(v.forRun({ machineId: 'm5', role: 'standing', userId: 'ben' }, { claude: true }).claude?.token, B); v.update('a', { disabled: true }); - assert.equal(v.forRun({ machineId: 'm5', role: 'standing' }, { claude: true }).claude, undefined, 'disabled'); - assert.throws(() => v.update('a', { share: 'owner' }), /needs an owner/); + assert.equal(v.forRun({ machineId: 'm5', role: 'standing', userId: 'ben' }, { claude: true }).claude, undefined, 'disabled'); + v.add({ name: 'shared-gh', kind: 'github', value: GH, share: 'anyone' }); + assert.throws(() => v.update('shared-gh', { share: 'owner' }), /needs an owner/); + v.remove('shared-gh'); v.remove('a'); assert.deepEqual(v.list(), []); assert.throws(() => v.remove('a'), /no vault entry named a/); @@ -122,34 +124,34 @@ test('vault: the key lives outside data/, readable by its owner only; a missing assert.throws(() => readKey(path.join(dir, 'short.key')), /not 32 bytes/); const v = make(); - v.add({ name: 'a', kind: 'claude', value: A, share: 'anyone' }); + v.add({ name: 'a', kind: 'claude', value: A, owner: 'ben', share: 'anyone' }); v.add({ name: 'gh', kind: 'github', value: GH, share: 'anyone' }); const none = make(() => ({ why: 'no vault key: test' })); assert.equal(none.status().key, 'missing'); assert.deepEqual(none.list().map((e) => e.name), ['a', 'gh'], 'listing needs no key'); - const r = none.forRun({ machineId: 'm3', role: 'workers' }, { claude: true }); + const r = none.forRun({ machineId: 'm3', role: 'workers', userId: 'ben' }, { claude: true }); assert.equal(r.claude, undefined); assert.deepEqual(r.env, {}); assert.match(r.problems[0], /cannot open a, gh: no vault key/); - assert.throws(() => none.add({ name: 'b', kind: 'claude', value: B, share: 'anyone' }), /cannot seal/); + assert.throws(() => none.add({ name: 'b', kind: 'claude', value: B, owner: 'ben', share: 'anyone' }), /cannot seal/); const other = path.join(dir, 'other.key'); fs.writeFileSync(other, newKeyText(), { mode: 0o600 }); fs.chmodSync(other, 0o600); const wrong = make(() => ({ file: other })); assert.equal(wrong.status().key, 'wrong'); - assert.equal(wrong.forRun({ machineId: 'm3', role: 'workers' }, { claude: true }).claude, undefined); + assert.equal(wrong.forRun({ machineId: 'm3', role: 'workers', userId: 'ben' }, { claude: true }).claude, undefined); }); test('vault: forRun gives the granted secrets as environment, and a Claude token only when asked', (t) => { const { make } = setup(t); const v = make(); - v.add({ name: 'a', kind: 'claude', value: A, share: 'anyone' }); + v.add({ name: 'a', kind: 'claude', value: A, owner: 'ben', share: 'anyone' }); v.add({ name: 'gh', kind: 'github', value: GH, share: 'anyone', roles: ['workers'] }); v.add({ name: 'max', kind: 'env', env: 'FFDISCORD_APP_TOKEN', value: DISCORD, share: 'anyone', machines: ['lothdesktop'] }); const m3 = v.forRun({ machineId: 'm3', role: 'workers' }, { claude: false }); assert.deepEqual(m3.env, { GH_TOKEN: GH }); assert.equal(m3.claude, undefined); - const loth = v.forRun({ machineId: 'lothdesktop', role: 'workers' }, { claude: true }); + const loth = v.forRun({ machineId: 'lothdesktop', role: 'workers', userId: 'ben' }, { claude: true }); assert.deepEqual(loth.env, { GH_TOKEN: GH, FFDISCORD_APP_TOKEN: DISCORD }); assert.equal(loth.claude?.token, A); assert.deepEqual(v.forRun({ machineId: 'm3', role: 'standing' }, { claude: false }).env, {}, 'gh is for workers only'); @@ -159,41 +161,33 @@ test('vault: forRun gives the granted secrets as environment, and a Claude token const usage = (session: number, weekly: number): PlanUsage => ({ available: true, asOf: '', models: [], session: { label: 's', percent: session }, weekly: { label: 'w', percent: weekly } }); const meta = (name: string, over: Partial = {}): VaultEntryMeta => ({ id: name, name, kind: 'claude', share: 'anyone', roles: ['workers', 'standing'], machines: ['*'], fingerprint: name, last4: 'xxxx', createdAt: '', updatedAt: '', ...over }); -test('pickClaude: the person own first, then room, the most headroom, the fewest live sessions; sticky while it has room', () => { - const run = { machineId: 'm3', role: 'workers' as const, userId: 'ben' }; - const u: Record = { a: usage(10, 20), b: usage(50, 10), c: usage(96, 10), own: usage(80, 80) }; - const of = (fp: string) => u[fp]; - const all = [meta('a'), meta('b'), meta('c'), meta('own', { owner: 'ben', share: 'owner' }), meta('loth', { owner: 'lothsahn', share: 'owner' })]; - assert.equal(pickClaude(all, run, of)?.name, 'own', "ben's own token first"); - assert.equal(pickClaude(all, { ...run, userId: 'lothsahn' }, of)?.name, 'loth'); - const shared = all.filter((e) => e.share === 'anyone'); - assert.equal(pickClaude(shared, { ...run, userId: 'x' }, of)?.name, 'a', 'most headroom: a has 80, b 50'); - assert.equal(headroom(u.c), 4); - assert.equal(pickClaude([meta('c'), meta('d')], run, of)?.name, 'd', `c is over ${FULL_PERCENT}%; d (no numbers) counts as 50`); - assert.equal(pickClaude([meta('c')], run, of)?.name, 'c', 'a full token still beats none'); - u.b = usage(20, 20); - assert.equal(pickClaude([meta('a'), meta('b')], run, of, (fp) => (fp === 'a' ? 3 : 0))?.name, 'b', 'equal room: the fewest live sessions'); - assert.equal(pickClaude([meta('a'), meta('b')], run, of, () => 0, 'b')?.name, 'b', 'sticky'); - assert.equal(pickClaude([meta('a'), meta('c')], run, of, () => 0, 'c')?.name, 'a', 'sticky, unless out of room'); - assert.equal(pickClaude([meta('a', { kind: 'github' })], run, of), undefined, 'only claude entries'); -}); test('eligible: role, machine and share', () => { const run = { machineId: 'm5', role: 'standing' as const, userId: 'Ben' }; - assert.equal(eligible(meta('a'), run), true); - assert.equal(eligible(meta('a', { roles: ['workers'] }), run), false); - assert.equal(eligible(meta('a', { machines: ['m3'] }), run), false); - assert.equal(eligible(meta('a', { machines: ['m3', 'M5'] }), run), true); - assert.equal(eligible(meta('a', { share: 'owner', owner: 'ben' }), run), true, 'user ids compare without case'); - assert.equal(eligible(meta('a', { share: 'owner', owner: 'loth' }), run), false); - assert.equal(eligible(meta('a', { share: 'owner', owner: 'loth' }), { ...run, userId: undefined }), false); - assert.equal(eligible(meta('a', { disabled: true }), run), false); + const gh = (over: Partial = {}) => meta('a', { kind: 'github', ...over }); + assert.equal(eligible(gh(), run), true); + assert.equal(eligible(gh({ roles: ['workers'] }), run), false); + assert.equal(eligible(gh({ machines: ['m3'] }), run), false); + assert.equal(eligible(gh({ machines: ['m3', 'M5'] }), run), true); + // A Claude token serves only its owner (w739): shared and other people's tokens are in nobody's pool. + assert.equal(eligible(meta('a'), run), false, 'a Claude token with no owner serves nobody'); + assert.equal(eligible(meta('a', { owner: 'ben' }), run), true); + assert.equal(eligible(meta('a', { owner: 'loth', share: 'anyone' }), run), false, "another person's token, shared or not"); + assert.equal(eligible(meta('a', { owner: 'ben', roles: ['workers'] }), run), false, 'the standing role is not granted'); + // The ops worker takes the workers' grants on no machine; a person's orchestrator takes no grant at all. + assert.equal(eligible(meta('a', { owner: 'ben', roles: ['workers'], machines: ['m3'] }), { ...run, role: 'ops' }), true); + assert.equal(eligible(meta('a', { owner: 'ben', roles: [], machines: [] }), { ...run, role: 'orchestrator' }), true); + assert.equal(eligible(meta('a', { owner: 'loth', roles: [], machines: [] }), { ...run, role: 'orchestrator' }), false); + assert.equal(eligible(gh({ share: 'owner', owner: 'ben' }), run), true, 'user ids compare without case'); + assert.equal(eligible(gh({ share: 'owner', owner: 'loth' }), run), false); + assert.equal(eligible(gh({ share: 'owner', owner: 'loth' }), { ...run, userId: undefined }), false); + assert.equal(eligible(gh({ disabled: true }), run), false); }); test('machineRunEnv: off by default; on, a vault token alone with the daemon credentials dropped; a person own token still wins', (t) => { const { make } = setup(t); const v = make(); - v.add({ name: 'shared', kind: 'claude', value: A, share: 'anyone' }); + v.add({ name: 'shared', kind: 'claude', value: A, owner: 'ben', share: 'anyone' }); v.add({ name: 'gh', kind: 'github', value: GH, share: 'anyone' }); const problems: string[] = []; const ctx: VaultContext = { vault: v, payer: () => 'ben', onProblem: (l) => problems.push(l) }; @@ -286,7 +280,7 @@ test('machine credentials: issued once as a hash, revoked by id', (t) => { test('vaultStatusLine: counts and the key, never a value', (t) => { const { make } = setup(t); const v = make(); - v.add({ name: 'a', kind: 'claude', value: A, share: 'anyone' }); + v.add({ name: 'a', kind: 'claude', value: A, owner: 'ben', share: 'anyone' }); v.add({ name: 'gh', kind: 'github', value: GH, share: 'anyone' }); const line = vaultStatusLine(v, ['m3']) ?? ''; assert.match(line, /2 entries \(1 claude, 1 github\); key loaded; Claude tokens from it on m3/); @@ -349,14 +343,14 @@ test('addSecretValues: a daemon learns the secret values of a launch spec withou test('vault: after a new key, the entries sealed with the old one are named, and rotating each makes it usable again', (t) => { const { dir, make } = setup(t); - make().add({ name: 'a', kind: 'claude', value: A, share: 'anyone' }); + make().add({ name: 'a', kind: 'claude', value: A, owner: 'ben', share: 'anyone' }); make().add({ name: 'gh', kind: 'github', value: GH, share: 'anyone' }); const other = path.join(dir, 'new.key'); fs.writeFileSync(other, newKeyText(), { mode: 0o600 }); fs.chmodSync(other, 0o600); const v = make(() => ({ file: other })); assert.equal(v.status().key, 'wrong'); - const r = v.forRun({ machineId: 'm3', role: 'workers' }, { claude: true }); + const r = v.forRun({ machineId: 'm3', role: 'workers', userId: 'ben' }, { claude: true }); assert.equal(r.claude, undefined); assert.deepEqual(r.env, {}); assert.match(r.problems.join('\n'), /gh does not open with this key \(rotate it\)/); @@ -364,8 +358,8 @@ test('vault: after a new key, the entries sealed with the old one are named, and const half = v.status(); assert.equal(half.key, 'loaded'); assert.match(half.why ?? '', /sealed with another key, rotate: gh/); - assert.equal(v.forRun({ machineId: 'm3', role: 'workers' }, { claude: true }).claude?.token, B); - v.add({ name: 'c', kind: 'claude', value: C, share: 'anyone' }); + assert.equal(v.forRun({ machineId: 'm3', role: 'workers', userId: 'ben' }, { claude: true }).claude?.token, B); + v.add({ name: 'c', kind: 'claude', value: C, owner: 'ben', share: 'anyone' }); v.remove('gh'); assert.equal(v.status().why, undefined, 'every entry opens again'); }); @@ -380,7 +374,7 @@ test('vault: one value per variable, the run person own entry first, then by nam assert.equal(v.forRun({ machineId: 'm3', role: 'workers', userId: 'loth' }, { claude: false }).env.GH_TOKEN, ghTok('a'), 'then by name'); // An entry named like another's id: grants are looked up by id, never by that name. const owned = v.add({ name: 'x', kind: 'claude', value: A, owner: 'loth', share: 'owner' }); - v.add({ name: owned.id, kind: 'claude', value: B, share: 'anyone', machines: ['m9'] }); + v.add({ name: owned.id, kind: 'claude', value: B, owner: 'ben', share: 'anyone', machines: ['m9'] }); assert.equal(v.forRun({ machineId: 'm3', role: 'workers', userId: 'loth' }, { claude: true }).claude?.token, A); assert.equal(v.forRun({ machineId: 'm9', role: 'workers', userId: 'ben' }, { claude: true }).claude?.token, B); }); diff --git a/server/vault.ts b/server/vault.ts index aa8841d2..58e02825 100644 --- a/server/vault.ts +++ b/server/vault.ts @@ -4,6 +4,7 @@ import path from 'node:path'; import { checkObject, readJsonDurable } from './durable.ts'; import { withFileLock, writeJsonOwned } from './machineTokens.ts'; import type { PlanUsage, VaultEntryMeta, VaultKind, VaultRole, VaultShare, VaultStatus, WorkItem } from '../shared/types.ts'; +import { POOL_DEFAULTS, pickPool, judge, type PoolLimits, type PoolToken, type PoolView } from './tokenPool.ts'; /** * The token vault (docs/vault.md, w512): the secrets worker runs need, kept by the portal, encrypted value by value with a @@ -43,10 +44,8 @@ const GITHUB = /^(?:gh[pousr]_[A-Za-z0-9]{30,}|github_pat_[A-Za-z0-9_]{30,})$/; const ENV_NAME = /^[A-Z][A-Z0-9_]{1,63}$/; const ENV_SUFFIX = /_(?:TOKEN|KEY|SECRET|PASSWORD)$/; const ENV_RESERVED = /^(?:CLAUDE_|ANTHROPIC_|GH_|GITHUB_|GIT_)/; -/** A meter at this percent or more counts as out of room (section 4 of docs/vault.md). */ -export const FULL_PERCENT = 95; -/** The headroom assumed for a token with no numbers yet. */ -const UNKNOWN_HEADROOM = 50; +/** How long a pick counts as live before the session's process shows in the live counts (a start in progress). */ +const PENDING_MS = 20_000; export const fingerprintOf = (value: string) => createHash('sha256').update(value).digest('hex').slice(0, 12); const keyIdOf = (key: Buffer) => createHash('sha256').update(key).digest('hex').slice(0, 8); @@ -159,10 +158,16 @@ function unseal(key: Buffer, e: VaultEntry): string { // ---------------------------------------------------------------- choosing a Claude token -/** A run on a machine, as the vault needs it. */ +/** + * The kinds of run the vault serves: a worker or a standing agent on a machine, the ops worker for a person, and a + * person's own orchestrator on the portal. The last two take no machine grant; the orchestrator takes no role grant. + */ +export type RunRole = VaultRole | 'orchestrator' | 'ops'; + +/** A run, as the vault needs it. */ export interface VaultRun { machineId: string; - role: VaultRole; + role: RunRole; /** The person the run is for (its requester, else the system payer). */ userId?: string; sessionId?: string; @@ -170,50 +175,39 @@ export interface VaultRun { const same = (a: string | undefined, b: string | undefined) => !!a && !!b && a.toLowerCase() === b.toLowerCase(); -/** Whether `e` may serve `run` at all (enabled, granted to its role and machine, shared or the run's person's own). */ +/** + * Whether `e` may serve `run` at all (enabled and granted to the run's role and machine). A Claude token serves only + * its owner's runs (w739: a person's pool, no shared Claude token and no one else's); every other kind is the run's + * person's own or shared. The ops worker takes the workers' grants; a person's orchestrator takes none: the account + * setting claudeAccounts.orchestrator is its consent. + */ export function eligible(e: VaultEntryMeta, run: VaultRun): boolean { if (e.disabled) return false; - if (!e.roles.includes(run.role)) return false; - if (!e.machines.includes('*') && !e.machines.some((m) => same(m, run.machineId))) return false; + if (run.role !== 'orchestrator') { + if (!e.roles.includes(run.role === 'ops' ? 'workers' : run.role)) return false; + if (run.role !== 'ops' && !e.machines.includes('*') && !e.machines.some((m) => same(m, run.machineId))) return false; + } + if (e.kind === 'claude') return same(e.owner, run.userId); return e.share === 'anyone' || same(e.owner, run.userId); } -/** How much room a token's account has left: 100 minus its fullest 5-hour or weekly meter, or undefined when unknown. */ -export function headroom(u: PlanUsage | undefined): number | undefined { - if (!u?.available) return undefined; - const used = [u.session?.percent, u.weekly?.percent].filter((p): p is number => typeof p === 'number'); - return used.length ? 100 - Math.max(...used) : undefined; -} - /** - * The Claude entry a run gets (docs/vault.md, section 4): among the eligible ones, the run's person's own first, then - * those with room (under FULL_PERCENT on both meters), the most headroom, the fewest live sessions, the name. `sticky`: - * the entry the session ran on last, kept while it is eligible and has room. + * A run that may not start now because its person's Claude pool has no token that may serve it (every one over its caps or + * used up): a worker or the ops worker waits (queued with this reason), an orchestrator stops. `next` is when the first + * token frees up (ISO). Never holds a token's value. */ -export function pickClaude( - entries: readonly VaultEntryMeta[], - run: VaultRun, - usageOf: (fingerprint: string) => PlanUsage | undefined = () => undefined, - liveOn: (fingerprint: string) => number = () => 0, - sticky?: string, -): VaultEntryMeta | undefined { - const pool = entries.filter((e) => e.kind === 'claude' && eligible(e, run)); - const room = (e: VaultEntryMeta) => headroom(usageOf(e.fingerprint)); - const full = (e: VaultEntryMeta) => { - const h = room(e); - return h !== undefined && h <= 100 - FULL_PERCENT; - }; - const kept = sticky ? pool.find((e) => e.id === sticky) : undefined; - if (kept && !full(kept)) return kept; - const score = (e: VaultEntryMeta): [number, number, number, number] => [same(e.owner, run.userId) ? 0 : 1, full(e) ? 1 : 0, -(room(e) ?? UNKNOWN_HEADROOM), liveOn(e.fingerprint)]; - return [...pool].sort((a, b) => { - const x = score(a); - const y = score(b); - for (let i = 0; i < x.length; i++) if (x[i] !== y[i]) return x[i] - y[i]; - return a.name.localeCompare(b.name); - })[0]; +export class PoolHeldError extends Error { + readonly next?: string; + constructor(message: string, next?: string) { + super(message); + this.name = 'PoolHeldError'; + this.next = next; + } } +/** The limits a pick applies, read when it is made; the defaults without a context (tests). */ +export type PoolLimitsOf = () => PoolLimits; + // ---------------------------------------------------------------- the store export interface AddInput { @@ -231,7 +225,14 @@ export type UpdateInput = Partial; /** Entries granted to the run that could not be opened (no key, a wrong key): named, never their values. */ problems: string[]; @@ -265,6 +266,8 @@ export class Vault { private mtime = -1; /** session id → the Claude entry it ran on last (pickClaude's sticky). */ private readonly lastPick = new Map(); + /** token fingerprint → session id → when it was picked: a process not yet in the live counts (PENDING_MS). */ + private readonly recent = new Map>(); constructor(o: VaultOptions) { this.o = o; @@ -442,11 +445,50 @@ export class Vault { .map(({ e, token }) => ({ token, label: `vault: ${e.name} …${e.last4}`, fingerprint: e.fingerprint, where: `the token vault: ${e.roles.join(', ')} on ${e.machines.includes('*') ? 'every machine' : e.machines.join(', ')}${e.share === 'owner' ? `, ${e.owner}'s own work` : ''}` })); } + /** The pick's view of one Claude entry: its meters, and the processes live on it other than `except`'s. */ + private poolToken(e: VaultEntry, usageOf: ((fp: string) => PlanUsage | undefined) | undefined, liveOn: ((fp: string, except?: string) => number) | undefined, except: string | undefined, now: number): PoolToken { + const r = this.recent.get(e.fingerprint); + let pending = 0; + if (r) { + for (const [sid, at] of r) { + if (now - at > PENDING_MS) r.delete(sid); + else if (sid !== except) pending++; + } + } + // A start in progress is not in the live counts yet; whichever is more, never both (the same process). + return { id: e.id, name: e.name, fingerprint: e.fingerprint, owner: e.owner, usage: usageOf?.(e.fingerprint), others: Math.max(liveOn?.(e.fingerprint, except) ?? 0, pending) }; + } + /** - * What a run gets (docs/vault.md, section 3): with `claude`, one Claude token chosen by pickClaude; and every other + * Every person's Claude pool as the rules read it now (system_status, `fffctl vault list`): each enabled token with its + * state and how many processes run on it. The state is the worker's reading (the strictest). + */ + pools(o: { usageOf?: (fp: string) => PlanUsage | undefined; liveOn?: (fp: string, except?: string) => number; limits?: PoolLimitsOf; now?: () => number }): Map { + this.reload(); + const limits = o.limits?.() ?? POOL_DEFAULTS; + const now = o.now?.() ?? Date.now(); + const out = new Map(); + for (const e of this.data.entries) { + if (e.kind !== 'claude' || e.disabled || !e.owner) continue; + const t = this.poolToken(e, o.usageOf, o.liveOn, undefined, now); + const list = out.get(e.owner.toLowerCase()) ?? []; + list.push({ entry: this.meta(e), view: judge(t, limits), live: t.others }); + out.set(e.owner.toLowerCase(), list); + } + return out; + } + + /** The fingerprint of the Claude entry a session ran on last (this portal run), for attributing its stopped session. */ + lastPickOf(sessionId: string): string | undefined { + const id = this.lastPick.get(sessionId); + return id ? this.byId(id)?.fingerprint : undefined; + } + + /** + * What a run gets (docs/vault.md, section 3): with `claude`, one Claude token chosen by the pool rules (tokenPool.ts); and every other * entry granted to the run's role and machine, as environment. Nothing when the key is missing (the problems say so). */ - forRun(run: VaultRun, o: { claude: boolean; usageOf?: (fingerprint: string) => PlanUsage | undefined; liveOn?: (fingerprint: string) => number }): RunSecrets { + forRun(run: VaultRun, o: { claude: boolean; usageOf?: (fingerprint: string) => PlanUsage | undefined; liveOn?: (fingerprint: string, exceptSession?: string) => number; limits?: PoolLimitsOf; now?: () => number }): RunSecrets { this.reload(); const out: RunSecrets = { env: {}, problems: [] }; const granted = this.data.entries.filter((e) => eligible(e, run)); @@ -472,31 +514,36 @@ export class Vault { } if (o.claude) { const sticky = run.sessionId ? this.lastPick.get(run.sessionId) : undefined; + const limits = o.limits?.() ?? POOL_DEFAULTS; + const now = o.now?.() ?? Date.now(); const tried = new Set(); + out.poolSize = granted.filter((e) => e.kind === 'claude').length; // A token that does not open (sealed by another key) is skipped for the next best. for (;;) { - const pick = pickClaude( - granted.filter((e) => !tried.has(e.id)), - run, - o.usageOf, - o.liveOn, - sticky, - ); - if (!pick) break; - const e = this.byId(pick.id)!; + const tokens = granted.filter((e) => e.kind === 'claude' && !tried.has(e.id)).map((e) => this.poolToken(e, o.usageOf, o.liveOn, run.sessionId, now)); + if (!tokens.length) break; + const pick = pickPool(tokens, { orchestrator: run.role === 'orchestrator', limits, sticky }); + if (!pick.token) { + out.claudeHold = pick.held; + break; + } + const e = this.byId(pick.token.id)!; const v = this.open(key, e); if (v) { if (run.sessionId) { this.lastPick.delete(run.sessionId); - this.lastPick.set(run.sessionId, pick.id); + this.lastPick.set(run.sessionId, e.id); // Oldest first: a portal that runs for months keeps the last few thousand sessions' picks only. if (this.lastPick.size > 5000) this.lastPick.delete(this.lastPick.keys().next().value!); + const r = this.recent.get(e.fingerprint) ?? new Map(); + r.set(run.sessionId, now); + this.recent.set(e.fingerprint, r); } - out.claude = { entry: this.meta(e), token: v }; + out.claude = { entry: this.meta(e), token: v, how: pick.how ?? 'normal' }; break; } - tried.add(pick.id); - out.problems.push(`vault entry ${pick.name} does not open with this key (rotate it)`); + tried.add(pick.token.id); + out.problems.push(`vault entry ${pick.token.name} does not open with this key (rotate it)`); } } return out; @@ -511,7 +558,10 @@ export class Vault { export interface VaultContext { vault: Vault; usageOf?: (fingerprint: string) => PlanUsage | undefined; - liveOn?: (fingerprint: string) => number; + /** Live processes on a token, not counting `exceptSession`'s own. */ + liveOn?: (fingerprint: string, exceptSession?: string) => number; + /** The system-wide pool limits in effect (config vault.pool), read at each pick. */ + limits?: PoolLimitsOf; payer?: () => string | undefined; /** A run that wanted a vault Claude token and fell back, or an entry that would not open: for the log and system_status. */ onProblem?: (line: string) => void; diff --git a/server/vaultCli.ts b/server/vaultCli.ts index 033ceb9a..e66dbb45 100644 --- a/server/vaultCli.ts +++ b/server/vaultCli.ts @@ -23,6 +23,8 @@ import path from 'node:path'; import { parseArgs } from 'node:util'; import { loadConfig } from './config.ts'; import { enrolledMachines, issueMachineToken, machineTokensFile, revokeMachineToken } from './machineTokens.ts'; +import type { PlanUsage } from '../shared/types.ts'; +import { clock, poolLimits } from './tokenPool.ts'; import { VAULT_FILE, Vault, fingerprintOf, keySource, newKeyText, type VaultEntryMeta, type VaultKind, type VaultRole, type VaultShare } from './vault.ts'; const { values: o, positionals: args } = parseArgs({ @@ -107,6 +109,44 @@ if (cmd === 'machine-credential') { process.exit(0); } +/** + * Each person's Claude pool as the portal's rules read it (w739, docs/vault.md section 4): per token its last four + * characters, the plan meters the portal last polled (data/usage.json) with their resets, the state (ok, held, + * one-at-a-time, retired, exhausted) and how many agents ran on it when the portal last saved its state (state.json). Read + * only, no token and no key needed. The portal's own system_status is the live view. + */ +function poolSection() { + let usageFile: { entries?: Record } = {}; + try { + usageFile = JSON.parse(fs.readFileSync(path.join(cfg.dataDir, 'usage.json'), 'utf8')) as typeof usageFile; + } catch { + // the portal has not polled yet + } + let sessions: { account?: string; status?: string }[] = []; + try { + sessions = (JSON.parse(fs.readFileSync(path.join(cfg.dataDir, 'state.json'), 'utf8')) as { sessions?: typeof sessions }).sessions ?? []; + } catch { + // no state file + } + const live = (fp: string) => sessions.filter((x) => x.account === `token:${fp}` && x.status !== 'stopped' && x.status !== 'error').length; + const pools = vault.pools({ usageOf: (fp) => usageFile.entries?.[`token:${fp}`]?.usage, liveOn: (fp) => live(fp), limits: () => poolLimits(cfg) }); + // A Claude entry with no owner is in nobody's pool: the portal no longer hands out a shared Claude token (w739). + const orphans = vault.list().filter((e) => e.kind === 'claude' && !e.disabled && !e.owner); + if (orphans.length) console.log(` +not in any pool (a Claude token with no owner is used by nobody; grant it to a person with --owner): ${orphans.map((e) => `${e.name} …${e.last4}`).join(', ')}`); + if (!pools.size) return; + const lim = poolLimits(cfg); + console.log(` +Claude token pools (5-hour >= ${lim.sessionHold}%: held; weekly >= ${lim.onePerWeekly}%: one at a time; >= ${lim.retireWeekly}%: retired; soonest weekly reset first):`); + for (const [person, list] of [...pools].sort((a, b) => a[0].localeCompare(b[0]))) { + console.log(` ${person}`); + for (const t of list.sort((a, b) => (a.view.weeklyResetsAt ?? '9').localeCompare(b.view.weeklyResetsAt ?? '9'))) { + const v = t.view; + console.log(` ${t.entry.name.padEnd(24)} …${t.entry.last4} ${v.state.padEnd(13)} 5-hour ${v.known ? Math.round(v.session) + '%' : '?'}${v.sessionResetsAt ? ` (resets ${clock(v.sessionResetsAt)})` : ''}, weekly ${v.known ? Math.round(v.weekly) + '%' : '?'}${v.weeklyResetsAt ? ` (resets ${clock(v.weeklyResetsAt)})` : ''}; ${t.live} agent${t.live === 1 ? '' : 's'} running`); + } + } +} + const keyFile = process.env.FFF_VAULT_KEY_FILE; const vault = new Vault({ file: path.join(cfg.dataDir, VAULT_FILE), @@ -126,6 +166,7 @@ try { console.log(`key: ${s.key}${s.keyFile ? ` (${s.keyFile})` : ''}${s.why ? `: ${s.why}` : ''}`); const all = vault.list(); console.log(all.length ? all.map(show).join('\n') : 'no entries'); + poolSection(); break; } case 'add': { diff --git a/server/vaultPool.test.ts b/server/vaultPool.test.ts new file mode 100644 index 00000000..d069f701 --- /dev/null +++ b/server/vaultPool.test.ts @@ -0,0 +1,416 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { PoolHeldError, Vault, fingerprintOf, newKeyText, type KeySource, type VaultContext } from './vault.ts'; +import { hostAccount, hostProcessEnv, machineRunEnv, poolRunEnv, reserveLines, reservedCredentials } from './secrets.ts'; +import { checkAccountConfig } from './config.ts'; +import { OWNER_ONLY_KEYS, normalizeSetting, setAppConfig } from './appConfig.ts'; +import { poolLimits } from './tokenPool.ts'; +import { tokenKey } from './usage.ts'; +import type { PlanUsage } from '../shared/types.ts'; +import type { Config } from './config.ts'; + +const NOW = Date.parse('2026-10-09T12:00:00Z'); +const DAY = 86_400_000; +const iso = (ms: number) => new Date(ms).toISOString(); +const claudeTok = (c: string) => `sk-ant-oat01-${c.repeat(48)}`; +const B1 = claudeTok('b'); +const B2 = claudeTok('c'); +const L1 = claudeTok('d'); +const SHARED = claudeTok('e'); +const FILE = claudeTok('f'); // the token file: the dispatcher's +const HOST = claudeTok('g'); // config claudeEnv: the host token +const API_KEY = ['sk-ant-api03-', 'z'.repeat(40)].join(''); + +const usage = (session: number, weekly: number, o: { weeklyDays?: number; sessionHours?: number } = {}): PlanUsage => ({ + available: true, + asOf: iso(NOW), + models: [], + session: { label: '5-hour', percent: session, ...(o.sessionHours !== undefined ? { resetsAt: iso(NOW + o.sessionHours * 3_600_000) } : {}) }, + weekly: { label: 'weekly', percent: weekly, ...(o.weeklyDays !== undefined ? { resetsAt: iso(NOW + o.weeklyDays * DAY) } : {}) }, +}); + +function setup(t: { after: (fn: () => void) => void }) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ffsb-pool-')); + t.after(() => fs.rmSync(dir, { recursive: true, force: true })); + const data = path.join(dir, 'data'); + fs.mkdirSync(data); + const keyFile = path.join(dir, 'vault.key'); + fs.writeFileSync(keyFile, newKeyText(), { mode: 0o600 }); + fs.chmodSync(keyFile, 0o600); + const tokenFile = path.join(dir, 'claude-oauth-token'); + fs.writeFileSync(tokenFile, `${FILE}\n`, { mode: 0o600 }); + const make = (key: () => KeySource = () => ({ file: keyFile })) => new Vault({ file: path.join(data, 'vault.json'), key }); + return { dir, data, keyFile, tokenFile, make }; +} + +/** A vault with ben's two tokens, lothsahn's one, and a Claude entry nobody owns. */ +function poolVault(make: ReturnType['make']) { + const v = make(); + v.add({ name: 'ben-a', kind: 'claude', value: B1, owner: 'ben', share: 'owner' }); + v.add({ name: 'ben-b', kind: 'claude', value: B2, owner: 'ben', share: 'owner' }); + v.add({ name: 'loth-a', kind: 'claude', value: L1, owner: 'lothsahn', share: 'owner' }); + v.add({ name: 'nobody', kind: 'claude', value: SHARED, share: 'anyone' }); + return v; +} + +/** Meters by fingerprint, and live counts by fingerprint, the way server/index.ts supplies them. */ +function world(meters: Record, live: Record = {}) { + return { + usageOf: (fp: string) => meters[fp], + liveOn: (fp: string) => live[fp] ?? 0, + now: () => NOW, + }; +} +const fp = (v: string) => fingerprintOf(v); +const run = (userId: string | undefined, role: 'workers' | 'standing' | 'ops' | 'orchestrator' = 'workers', sessionId?: string) => ({ machineId: role === 'ops' || role === 'orchestrator' ? 'portal' : 'm3', role, userId, ...(sessionId ? { sessionId } : {}) }); + +test("pool: a person's runs use that person's own tokens only: no shared token, no one else's", (t) => { + const { make } = setup(t); + const v = poolVault(make); + const w = world({}); + for (const role of ['workers', 'standing', 'ops', 'orchestrator'] as const) { + const ben = v.forRun(run('ben', role), { claude: true, ...w }); + assert.ok([B1, B2].includes(ben.claude?.token ?? ''), `ben's ${role} run is on one of ben's tokens`); + assert.equal(ben.poolSize, 2); + const loth = v.forRun(run('lothsahn', role), { claude: true, ...w }); + assert.equal(loth.claude?.token, L1); + } + // A person with no token of their own gets nothing from the vault, whatever else it holds: today's account (the caller's). + const nobody = v.forRun(run('sam'), { claude: true, ...w }); + assert.equal(nobody.claude, undefined); + assert.equal(nobody.claudeHold, undefined); + assert.equal(nobody.poolSize ?? 0, 0); + const unasked = v.forRun(run(undefined), { claude: true, ...w }); + assert.equal(unasked.claude, undefined, 'a run with no person has no pool'); + // The tokens that nobody owns are listed for the meters but never picked. + assert.equal(v.claudeTokens().length, 4); +}); + +test('pool: the token whose weekly window resets first is used first; a held or retired one is skipped', (t) => { + const { make } = setup(t); + const v = poolVault(make); + const m = { + [fp(B1)]: usage(10, 30, { weeklyDays: 5, sessionHours: 3 }), + [fp(B2)]: usage(10, 60, { weeklyDays: 2, sessionHours: 3 }), + }; + assert.equal(v.forRun(run('ben'), { claude: true, ...world(m) }).claude?.entry.name, 'ben-b', 'resets sooner'); + m[fp(B2)] = usage(85, 60, { weeklyDays: 2, sessionHours: 3 }); + assert.equal(v.forRun(run('ben'), { claude: true, ...world(m) }).claude?.entry.name, 'ben-a', 'ben-b is at 85% of its 5-hour window: held'); + m[fp(B2)] = usage(10, 99.4, { weeklyDays: 2 }); + assert.equal(v.forRun(run('ben'), { claude: true, ...world(m) }).claude?.entry.name, 'ben-a', 'ben-b retired until its week resets'); + // The limits are the config's (system-wide), read at each pick. + m[fp(B2)] = usage(60, 60, { weeklyDays: 2 }); + const strict = () => poolLimits({ vault: { pool: { sessionHoldPercent: 50 } } }); + assert.equal(v.forRun(run('ben'), { claude: true, ...world(m), limits: strict }).claude?.entry.name, 'ben-a', 'held at 50% with the changed limit'); + assert.equal(v.forRun(run('ben'), { claude: true, ...world(m) }).claude?.entry.name, 'ben-b', 'and not at the default 80%'); +}); + +test('pool: one process at a time at 95% weekly, counting processes starting now and not the asking session', (t) => { + const { make } = setup(t); + const v = poolVault(make); + const m = { [fp(B1)]: usage(10, 96, { weeklyDays: 1 }), [fp(B2)]: usage(10, 96, { weeklyDays: 3 }) }; + // Idle: the first run takes ben-a (resets first). + const s1 = v.forRun(run('ben', 'workers', 's1'), { claude: true, ...world(m) }); + assert.equal(s1.claude?.entry.name, 'ben-a'); + // A start in progress counts: the second run takes ben-b, the third finds both taken. + const s2 = v.forRun(run('ben', 'workers', 's2'), { claude: true, ...world(m) }); + assert.equal(s2.claude?.entry.name, 'ben-b'); + const s3 = v.forRun(run('ben', 'workers', 's3'), { claude: true, ...world(m) }); + assert.equal(s3.claude, undefined); + assert.match(s3.claudeHold?.why ?? '', /2 tokens: all over their caps or used up/); + // Live counts from the portal do the same once the processes show. + const live = { [fp(B1)]: 1, [fp(B2)]: 1 }; + assert.equal(v.forRun(run('ben', 'workers', 's9'), { claude: true, ...world(m, live) }).claude, undefined); + // The session that is on ben-a does not count itself: it may start again on it (a resume). + const mine = { [fp(B1)]: 0, [fp(B2)]: 1 }; + assert.equal(v.forRun(run('ben', 'workers', 's1'), { claude: true, ...world(m, mine) }).claude?.entry.name, 'ben-a', 'sticky, and alone on it'); +}); + +test('pool: every token over its caps holds a worker and the ops worker; the orchestrator uses the pool up; used up stops it too', (t) => { + const { make } = setup(t); + const v = poolVault(make); + const m: Record = { + [fp(B1)]: usage(90, 40, { weeklyDays: 3, sessionHours: 2 }), + [fp(B2)]: usage(85, 30, { weeklyDays: 5, sessionHours: 4 }), + }; + for (const role of ['workers', 'standing', 'ops'] as const) { + const r = v.forRun(run('ben', role), { claude: true, ...world(m) }); + assert.equal(r.claude, undefined, role); + assert.equal(r.poolSize, 2); + assert.match(r.claudeHold?.why ?? '', /all over their caps or used up, the first frees up Fri 14:00Z/, role); + assert.equal(r.claudeHold?.next, iso(NOW + 2 * 3_600_000)); + } + const orch = v.forRun(run('ben', 'orchestrator'), { claude: true, ...world(m) }); + assert.equal(orch.claude?.entry.name, 'ben-b', 'the most room (85 against 90)'); + assert.equal(orch.claude?.how, 'over-cap'); + // Both used up: nobody, the orchestrator included. + m[fp(B1)] = usage(100, 40, { sessionHours: 2 }); + m[fp(B2)] = usage(10, 100, { weeklyDays: 4 }); + const o2 = v.forRun(run('ben', 'orchestrator'), { claude: true, ...world(m) }); + assert.equal(o2.claude, undefined); + assert.match(o2.claudeHold?.why ?? '', /all used up/); + // lothsahn's pool is her own: ben's being held does not touch it. + assert.equal(v.forRun(run('lothsahn'), { claude: true, ...world(m) }).claude?.token, L1); +}); + +test('pool: the ops worker takes the workers grants; the orchestrator takes none; a granted-nowhere token is not a worker token', (t) => { + const { make } = setup(t); + const v = make(); + v.add({ name: 'ben-a', kind: 'claude', value: B1, owner: 'ben', share: 'owner', roles: ['standing'], machines: ['m5'] }); + const w = world({}); + assert.equal(v.forRun(run('ben', 'workers'), { claude: true, ...w }).claude, undefined, 'granted to standing agents on m5 only'); + assert.equal(v.forRun(run('ben', 'standing'), { claude: true, ...w }).claude, undefined, 'standing on m3: not granted'); + assert.equal(v.forRun(run('ben', 'orchestrator'), { claude: true, ...w }).claude?.token, B1, "the orchestrator's account setting is its consent"); + v.update('ben-a', { roles: ['workers'], machines: ['*'] }); + assert.equal(v.forRun(run('ben', 'ops'), { claude: true, ...w }).claude?.token, B1); +}); + +test("pool: lastPickOf names the token a session ran on, for the stopped session's meter; a removed token is forgotten", (t) => { + const { make } = setup(t); + const v = poolVault(make); + const r = v.forRun(run('ben', 'orchestrator', 'orch-ben'), { claude: true, ...world({}) }); + assert.equal(v.lastPickOf('orch-ben'), r.claude?.entry.fingerprint); + assert.equal(v.lastPickOf('nobody-at-all'), undefined); + v.remove(r.claude!.entry.name); + assert.equal(v.lastPickOf('orch-ben'), undefined); +}); + +test('pools(): every person pool as the rules read it, no token in it', (t) => { + const { make } = setup(t); + const v = poolVault(make); + const m = { [fp(B1)]: usage(85, 20, { sessionHours: 1, weeklyDays: 3 }), [fp(B2)]: usage(10, 96, { weeklyDays: 2 }) }; + const pools = v.pools({ ...world(m, { [fp(B2)]: 1 }), now: () => NOW }); + assert.deepEqual([...pools.keys()].sort(), ['ben', 'lothsahn'], 'the owner-less token is in no pool'); + const ben = pools.get('ben')!; + assert.deepEqual(ben.map((x) => [x.entry.name, x.view.state]), [['ben-a', 'held'], ['ben-b', 'one-at-a-time']]); + assert.equal(ben.find((x) => x.entry.name === 'ben-b')!.live, 1); + assert.ok(!JSON.stringify([...pools]).includes(B1.slice(13, 40)), 'no value in it'); + assert.equal(pools.get('lothsahn')![0].view.state, 'ok'); +}); + +// ---------------------------------------------------------------- machineRunEnv and poolRunEnv + +function ctxOf(v: Vault, meters: Record = {}, live: Record = {}): VaultContext & { problems: string[] } { + const problems: string[] = []; + return { vault: v, usageOf: (f) => meters[f], liveOn: (f) => live[f] ?? 0, payer: () => 'lothsahn', onProblem: (l) => problems.push(l), problems, limits: () => poolLimits(undefined) }; +} +const cfgOf = (extra: Record = {}) => + ({ claudeEnv: { CLAUDE_CODE_OAUTH_TOKEN: HOST, ANTHROPIC_API_KEY: API_KEY, CLAUDE_CONFIG_DIR: '/x' }, userClaudeEnv: {}, machines: { claudeFromVault: true }, ...extra }) as never; + +test('machineRunEnv: a worker waits (PoolHeldError) when its person pool is over its caps, and falls back to no one', (t) => { + const { make } = setup(t); + const v = poolVault(make); + const m = { [fp(B1)]: usage(90, 40, { weeklyDays: 3, sessionHours: 2 }), [fp(B2)]: usage(95, 30, { weeklyDays: 5, sessionHours: 4 }) }; + const ctx = ctxOf(v, m); + const ben = { userId: 'ben', displayName: 'Ben' }; + assert.throws( + () => machineRunEnv(cfgOf(), { id: 'm3' }, { role: 'workers', requestedBy: ben, sessionId: 'w1' }, ctx), + (e: unknown) => e instanceof PoolHeldError && /ben's Claude token pool is held: 2 tokens: all over their caps or used up, the first frees up Fri 14:00Z/.test(e.message) && e.next === iso(NOW + 2 * 3_600_000) && !e.message.includes(B1.slice(13, 40)), + ); + // Never the host token, the machine's login, nor another person's token (the dispatcher's token file is not a worker's at all). + assert.equal(ctx.problems.length, 0, 'held is not a fallback: nothing reported as one'); + // Ben's pool frees up: he gets his token, alone, with the daemon's other credentials dropped. + m[fp(B1)] = usage(30, 40, { weeklyDays: 3 }); + const ok = machineRunEnv(cfgOf(), { id: 'm3' }, { role: 'workers', requestedBy: ben, sessionId: 'w1' }, ctx); + assert.equal(ok.env.CLAUDE_CODE_OAUTH_TOKEN, B1); + assert.equal(ok.env.ANTHROPIC_API_KEY, undefined); + assert.equal(ok.login, true); + assert.match(ok.account, /^vault token ben-a …bbbb \(ben's pool, docs\/vault.md\)$/); + // Work nobody asked for runs on the system payer's pool (lothsahn here), not on a shared token. + const unasked = machineRunEnv(cfgOf(), { id: 'm3' }, { role: 'workers', sessionId: 'w2' }, ctx); + assert.equal(unasked.env.CLAUDE_CODE_OAUTH_TOKEN, L1); +}); + +test("the transition: a person with no vault tokens keeps today's account, and so does one whose vault cannot be opened", (t) => { + const { make } = setup(t); + const v = poolVault(make); + const ctx = ctxOf(v); + const sam = { userId: 'sam', displayName: 'Sam' }; + const r = machineRunEnv(cfgOf(), { id: 'm3' }, { role: 'workers', requestedBy: sam, sessionId: 'w1' }, ctx); + assert.equal(r.env.CLAUDE_CODE_OAUTH_TOKEN, HOST, "the host token, as before the pool"); + assert.equal(r.login, false); + assert.match(ctx.problems.at(-1) ?? '', /no vault Claude token for a workers run on m3 for sam; it runs on host token/); + // Ben has tokens but the vault key is gone: not a hold (nothing the person can wait for), today's account with a warning. + const nokey = ctxOf(make(() => ({ why: 'no vault key: test' }))); + const r2 = machineRunEnv(cfgOf(), { id: 'm3' }, { role: 'workers', requestedBy: { userId: 'ben', displayName: 'Ben' }, sessionId: 'w3' }, nokey); + assert.equal(r2.env.CLAUDE_CODE_OAUTH_TOKEN, HOST); + assert.match(nokey.problems.join('\n'), /the vault cannot open ben-a, ben-b: no vault key: test/); +}); + +test('poolRunEnv: on "vault" the orchestrator and the ops worker run on that person token alone; every other Claude credential is gone', (t) => { + const { make, tokenFile } = setup(t); + const v = poolVault(make); + const ctx = ctxOf(v, { [fp(B1)]: usage(10, 50, { weeklyDays: 4 }), [fp(B2)]: usage(10, 50, { weeklyDays: 2 }) }); + const cfg = cfgOf({ claudeAccounts: { orchestrator: 'vault' }, claudeTokenFile: tokenFile }); + const fallback = () => hostProcessEnv(cfg, 'orchestrator', { PATH: '/bin', ANTHROPIC_API_KEY: API_KEY, CLAUDE_CODE_OAUTH_TOKEN: 'server-own' }); + for (const role of ['orchestrator', 'ops'] as const) { + const r = poolRunEnv(cfg, { role, personId: 'ben', sessionId: `s-${role}` }, fallback, ctx); + assert.equal(r.env.CLAUDE_CODE_OAUTH_TOKEN, B2, `${role}: ben's token that resets first`); + assert.equal(r.env.ANTHROPIC_API_KEY, undefined, 'an API key would outrank the token: gone'); + assert.equal(r.env.CLAUDE_CONFIG_DIR, '/x', 'settings that are not credentials stay'); + assert.match(r.account, /^vault token ben-b …cccc \(ben's pool/); + assert.ok(!JSON.stringify(r.account).includes(B2.slice(13, 40))); + } + assert.equal(poolRunEnv(cfg, { role: 'orchestrator', personId: 'lothsahn' }, fallback, ctx).env.CLAUDE_CODE_OAUTH_TOKEN, L1, "each person's orchestrator on their own"); + // A new token in the vault applies to the next session start, no restart: ben-c resets sooner than anything. + v.add({ name: 'ben-c', kind: 'claude', value: claudeTok('h'), owner: 'ben', share: 'owner' }); + const meters = { [fp(claudeTok('h'))]: usage(10, 50, { weeklyDays: 1 }), [fp(B1)]: usage(10, 50, { weeklyDays: 4 }), [fp(B2)]: usage(10, 50, { weeklyDays: 2 }) }; + const ctx2 = ctxOf(v, meters); + assert.equal(poolRunEnv(cfg, { role: 'orchestrator', personId: 'ben' }, fallback, ctx2).env.CLAUDE_CODE_OAUTH_TOKEN, claudeTok('h')); +}); + +test('poolRunEnv: no pool token yet (the transition) is the token file; not on "vault" it is the fallback whatever the vault holds', (t) => { + const { make, tokenFile } = setup(t); + const v = poolVault(make); + const ctx = ctxOf(v); + const cfg = cfgOf({ claudeAccounts: { orchestrator: 'vault' }, claudeTokenFile: tokenFile }); + const fallback = () => hostProcessEnv(cfg, 'orchestrator', { PATH: '/bin' }); + const r = poolRunEnv(cfg, { role: 'orchestrator', personId: 'sam' }, fallback, ctx); + assert.equal(r.env.CLAUDE_CODE_OAUTH_TOKEN, FILE, 'the token file, as today'); + assert.equal(r.env.ANTHROPIC_API_KEY, undefined); + assert.equal(r.account, 'token file'); + assert.equal(poolRunEnv(cfg, { role: 'ops', personId: undefined }, fallback, ctx).env.CLAUDE_CODE_OAUTH_TOKEN, FILE, 'the ops worker with no caller'); + // Another account setting: the pool is not used. + const tokenAccount = cfgOf({ claudeAccounts: { orchestrator: 'token' } }); + assert.equal(poolRunEnv(tokenAccount, { role: 'orchestrator', personId: 'ben' }, () => ({ CLAUDE_CODE_OAUTH_TOKEN: HOST }), ctx).env.CLAUDE_CODE_OAUTH_TOKEN, HOST); + // The vault cannot open ben's tokens: the fallback, with the problem named (never a value). + const nokey = ctxOf(make(() => ({ why: 'no vault key: test' }))); + const r2 = poolRunEnv(cfg, { role: 'orchestrator', personId: 'ben' }, fallback, nokey); + assert.equal(r2.env.CLAUDE_CODE_OAUTH_TOKEN, FILE); + assert.match(nokey.problems.join('\n'), /no vault key: test/); +}); + +test("poolRunEnv: the orchestrator stops when its pool is used up and the error shows the reset, never a token; the ops worker is held when capped", (t) => { + const { make, tokenFile } = setup(t); + const v = poolVault(make); + const cfg = cfgOf({ claudeAccounts: { orchestrator: 'vault' }, claudeTokenFile: tokenFile }); + const fallback = () => hostProcessEnv(cfg, 'orchestrator', {}); + const used = ctxOf(v, { [fp(B1)]: usage(100, 40, { sessionHours: 2 }), [fp(B2)]: usage(10, 100, { weeklyDays: 3 }) }); + assert.throws( + () => poolRunEnv(cfg, { role: 'orchestrator', personId: 'ben' }, fallback, used), + (e: unknown) => e instanceof PoolHeldError && /ben's Claude token pool is used up: 2 tokens: all used up, the first frees up Fri 14:00Z/.test(e.message) && !e.message.includes(B1.slice(13, 40)) && !e.message.includes(FILE.slice(13, 40)), + ); + const capped = ctxOf(v, { [fp(B1)]: usage(90, 40, { sessionHours: 2 }), [fp(B2)]: usage(85, 30, { weeklyDays: 3, sessionHours: 4 }) }); + assert.throws(() => poolRunEnv(cfg, { role: 'ops', personId: 'ben' }, fallback, capped), (e: unknown) => e instanceof PoolHeldError && /ben's Claude token pool is held/.test(e.message)); + // The orchestrator on the same pool still runs: over its cap, on the token with the most room. + assert.equal(poolRunEnv(cfg, { role: 'orchestrator', personId: 'ben' }, fallback, capped).env.CLAUDE_CODE_OAUTH_TOKEN, B2); + // Neither falls to the dispatcher's token file: the stop is the stop. +}); + +// ---------------------------------------------------------------- the dispatcher and its reserve + +test('hostAccount: "vault" is the orchestrators only; the dispatcher that inherits it stays on the token file (else the host token)', () => { + const cfg = { claudeAccounts: { orchestrator: 'vault' }, claudeTokenFile: '/srv/fff/secrets/t', claudeEnv: {} } as never; + assert.equal(hostAccount(cfg, 'orchestrator'), 'vault'); + assert.equal(hostAccount(cfg, 'dispatcher'), 'tokenfile'); + assert.equal(hostAccount({ claudeAccounts: { orchestrator: 'vault' }, claudeEnv: {} } as never, 'dispatcher'), 'token'); + assert.equal(hostAccount({ claudeAccounts: { orchestrator: 'vault', dispatcher: 'tokenfile' }, claudeTokenFile: '/x', claudeEnv: {} } as never, 'dispatcher'), 'tokenfile'); +}); + +test('the reserve follows the dispatcher account and covers the host token; one credential is the dispatcher', (t) => { + const { tokenFile } = setup(t); + const base = { claudeEnv: { CLAUDE_CODE_OAUTH_TOKEN: HOST }, claudeTokenFile: tokenFile } as const; + // The token file is the dispatcher's, the host token is reserved for its own roles. + const a = reservedCredentials({ ...base, claudeAccounts: { orchestrator: 'vault', dispatcher: 'tokenfile' } } as never); + assert.deepEqual(a.map((c) => [c.role, c.key]), [['dispatcher', tokenKey(FILE)], ['host-token', tokenKey(HOST)]]); + assert.match(a[0].what, /^the dispatcher's token file …ffff$/); + assert.match(a[1].what, /^the host token …gggg$/); + // Switch the dispatcher's account to the host token: the reserve moves and the two are one credential, the dispatcher's. + const b = reservedCredentials({ ...base, claudeAccounts: { dispatcher: 'token' } } as never); + assert.deepEqual(b.map((c) => [c.role, c.key]), [['dispatcher', tokenKey(HOST)]]); + assert.match(b[0].what, /^the dispatcher's host token …gggg$/); + // The dispatcher on this host's login. + const c = reservedCredentials({ ...base, claudeAccounts: { dispatcher: 'login' } } as never); + assert.deepEqual(c.map((x) => [x.role, x.key]), [['dispatcher', 'host:login'], ['host-token', tokenKey(HOST)]]); + // No token file readable: no reserve on it (nothing to name). + assert.deepEqual(reservedCredentials({ claudeEnv: {}, claudeAccounts: { dispatcher: 'tokenfile' }, claudeTokenFile: '/nonexistent/x' } as never), []); +}); + +test("reserveLines: the dispatcher's token shows its reserve; runs inside it raise the warning; none without readings or users", (t) => { + const { tokenFile } = setup(t); + const cfg = { claudeEnv: { CLAUDE_CODE_OAUTH_TOKEN: HOST }, claudeTokenFile: tokenFile, claudeAccounts: { orchestrator: 'vault', dispatcher: 'tokenfile' } } as never; + const file = tokenKey(FILE); + const meters: Record = { [file]: usage(12, 41, { weeklyDays: 5 }), [tokenKey(HOST)]: usage(5, 10, { weeklyDays: 5 }) }; + const none = (_: string) => 0; + const calm = reserveLines(cfg, (k) => meters[k], none, NOW); + assert.equal(calm[0].line, "dispatcher's token file …ffff: weekly 41%, reserve 25% (5 days to reset); 5-hour 12%, reserve 20%"); + assert.equal(calm[0].warning, undefined); + assert.match(calm[1].line, /^host token …gggg: weekly 10%, reserve 25% \(5 days to reset\)/); + // Inside the buffer with another person's orchestrator on it (a person with no vault token yet): a warning, not a refusal. + meters[file] = usage(12, 83, { weeklyDays: 5 }); + const busy = reserveLines(cfg, (k) => meters[k], (k, role) => (k === file && role === 'dispatcher' ? 2 : 0), NOW); + assert.match(busy[0].line, /inside its reserve, kept for the dispatcher; 2 other runs using it/); + assert.match(busy[0].warning ?? '', /dispatcher's token file …ffff is the last token with room for 2 runs and is inside the buffer kept for the dispatcher \(weekly 83%, reserve 25%, resets \w{3} \d\d:\d\dZ\)/); + // Inside the buffer and nobody else on it: the dispatcher alone uses it, as intended: no warning. + const alone = reserveLines(cfg, (k) => meters[k], none, NOW); + assert.match(alone[0].line, /inside its reserve, kept for the dispatcher$/); + assert.equal(alone[0].warning, undefined); + // No reading yet: counted as inside its reserve. + const unread = reserveLines(cfg, () => undefined, none, NOW); + assert.match(unread[0].line, /no reading yet \(counted as inside its reserve\)/); + // 5 days to the reset gives 25; the figure per day is the system-wide config. + const slow = reserveLines({ ...(cfg as object), vault: { pool: { reservePerDayPercent: 2 } } } as never, (k) => meters[k], none, NOW); + assert.match(slow[0].line, /reserve 10% \(5 days to reset\)/); + assert.ok(![calm, busy, alone, unread].some((x) => x.some((l) => l.line.includes(FILE.slice(13, 40)))), 'a line shows the last four characters only'); +}); + +test("the dispatcher is never held by its own reserve: its environment is the token file whatever the meters say", (t) => { + const { tokenFile } = setup(t); + const cfg = { claudeEnv: { CLAUDE_CODE_OAUTH_TOKEN: HOST }, claudeTokenFile: tokenFile, claudeAccounts: { orchestrator: 'vault', dispatcher: 'tokenfile' } } as never; + // hostProcessEnv consults no meter: at 99.9% weekly and 100% of the 5-hour window the dispatcher still starts on it. + const env = hostProcessEnv(cfg, 'dispatcher', { PATH: '/bin' }); + assert.equal(env.CLAUDE_CODE_OAUTH_TOKEN, FILE); + // And the reserve is a line and a warning, never an exception: reserveLines returns even at 100%. + const full = reserveLines(cfg, () => usage(100, 99.9, { weeklyDays: 0.1 }), () => 0, NOW); + assert.equal(full.length, 2); + assert.match(full[0].line, /inside its reserve, kept for the dispatcher/); +}); + +// ---------------------------------------------------------------- config + +test('config: "vault" is the orchestrator account; the dispatcher and workers refuse it; vault.pool is checked at load', () => { + checkAccountConfig({ claudeAccounts: { orchestrator: 'vault' } } as never); + checkAccountConfig({ claudeAccounts: { orchestrator: 'vault', dispatcher: 'tokenfile' }, claudeTokenFile: '/x' } as never); + assert.throws(() => checkAccountConfig({ claudeAccounts: { dispatcher: 'vault' } } as never), /claudeAccounts\.dispatcher cannot be "vault"/); + assert.throws(() => checkAccountConfig({ claudeAccounts: { workers: 'vault' } } as never), /claudeAccounts\.workers cannot be "vault"/); + checkAccountConfig({ vault: { pool: { sessionHoldPercent: 70, reservePerDayPercent: 0 } } } as never); + assert.throws(() => checkAccountConfig({ vault: { pool: { sessionHoldPercent: 101 } } } as never), /percent from 0 to 100/); + assert.throws(() => checkAccountConfig({ vault: { pool: { onePerWeeklyPercent: 99.9 } } } as never), /above retireWeeklyPercent/); + assert.throws(() => checkAccountConfig({ vault: { pool: { retireWeeklyPercent: 90, onePerWeeklyPercent: 95 } } } as never), /above retireWeeklyPercent/); +}); + +test('set_app_config: "vault" for the orchestrator only; the pool limits are anyone\'s, checked, applied live and cleared by null', (t) => { + assert.equal(normalizeSetting('claudeAccounts.orchestrator', 'vault'), 'vault'); + assert.throws(() => normalizeSetting('claudeAccounts.dispatcher', 'vault'), /cannot be "vault"/); + assert.throws(() => normalizeSetting('claudeAccounts.workers', 'vault'), /cannot be "vault"/); + // Not owner-only keys: the pool limits are one system-wide set that any person's orchestrator may change. + for (const k of ['vault.pool.sessionHoldPercent', 'vault.pool.onePerWeeklyPercent', 'vault.pool.retireWeeklyPercent', 'vault.pool.reservePerDayPercent', 'vault.pool.reserveSessionPercent'] as const) { + assert.equal(OWNER_ONLY_KEYS.has(k), false, k); + } + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'pool-cfg-')); + t.after(() => fs.rmSync(dir, { recursive: true, force: true })); + const file = path.join(dir, 'config.json'); + fs.writeFileSync(file, JSON.stringify({ port: 8790 })); + const cfg = { voice: { vocabulary: [], ttsVoice: 'af_heart', device: 'auto' } } as unknown as Config; + assert.deepEqual(setAppConfig(file, cfg, 'vault.pool.sessionHoldPercent', 70), { before: undefined, after: 70 }); + assert.equal(poolLimits(cfg).sessionHold, 70, 'live: the next pick reads it'); + setAppConfig(file, cfg, 'vault.pool.reservePerDayPercent', '8'); + assert.equal(poolLimits(cfg).reservePerDay, 8, 'a number given as text'); + assert.deepEqual(JSON.parse(fs.readFileSync(file, 'utf8')).vault.pool, { sessionHoldPercent: 70, reservePerDayPercent: 8 }, 'written to config.json'); + // Invalid values are refused; the other limit in the pair is read from the config. + for (const bad of [101, -1, 'lots', NaN, '']) assert.throws(() => setAppConfig(file, cfg, 'vault.pool.sessionHoldPercent', bad as never), /percent from 0 to 100|is a percent/, String(bad)); + assert.throws(() => setAppConfig(file, cfg, 'vault.pool.onePerWeeklyPercent', 99.5), /cannot be above vault.pool.retireWeeklyPercent \(99\)/); + assert.throws(() => setAppConfig(file, cfg, 'vault.pool.retireWeeklyPercent', 90), /cannot be below vault.pool.onePerWeeklyPercent \(95\)/); + setAppConfig(file, cfg, 'vault.pool.onePerWeeklyPercent', 85); + assert.equal(poolLimits(cfg).onePerWeekly, 85); + setAppConfig(file, cfg, 'vault.pool.retireWeeklyPercent', 90); + assert.throws(() => setAppConfig(file, cfg, 'vault.pool.onePerWeeklyPercent', 95), /cannot be above vault.pool.retireWeeklyPercent \(90\)/); + assert.throws(() => setAppConfig(file, cfg, 'vault.pool.retireWeeklyPercent', 80), /cannot be below vault.pool.onePerWeeklyPercent \(85\)/); + // null puts a limit back to its default. + setAppConfig(file, cfg, 'vault.pool.sessionHoldPercent', null); + assert.equal(poolLimits(cfg).sessionHold, 80); + assert.equal(JSON.parse(fs.readFileSync(file, 'utf8')).vault.pool.sessionHoldPercent, undefined); +}); diff --git a/shared/types.ts b/shared/types.ts index 204afb8f..a30cdfdc 100644 --- a/shared/types.ts +++ b/shared/types.ts @@ -1085,6 +1085,17 @@ export interface HostStatus { pathHealth?: PathHealth; /** What the VM's unit watchdog (fff-health) did about critical systemd units (server/unitWatchdog.ts). */ unitWatchdog?: UnitWatchdog; + /** Claude token pools that need a person's eye, and the dispatcher's reserve in use (server/index.ts tokenWarnings, w739). */ + tokenWarnings?: TokenWarning[]; +} + +/** One banner about the Claude token pools. No token, only its last four characters. */ +export interface TokenWarning { + /** "pool:" or "reserve:": a banner's identity, so a dismissed one returns when its text changes. */ + id: string; + /** exhausted: the person's orchestrator has stopped; held: workers wait; reserve: runs are inside the dispatcher's buffer. */ + kind: 'exhausted' | 'held' | 'reserve'; + text: string; } /** One layer of the path from the internet to the portal (deploy/vm/host/pathwatch.sh). */ diff --git a/web/src/App.tsx b/web/src/App.tsx index ff8f8545..6a6bfc7b 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -162,7 +162,8 @@ function HostBanner({ host, app }: { host?: HostStatus; app: AppState }) { const gaveUp = wd?.units.filter((u) => u.state === 'gave-up') ?? []; const restarts = (wd?.events ?? []).filter((e) => e.action !== 'gave-up' && Date.now() - Date.parse(e.at) <= 24 * 3_600_000); const wdShown = gaveUp.length > 0 || wd?.silentMinutes !== undefined || restarts.length > 0; - if (!host || (!host.elevated && !host.drain && !drive && !disk && !host.dryRun && !pathShown && !wdShown)) return null; + const tokenWarnings = host?.tokenWarnings ?? []; + if (!host || (!host.elevated && !host.drain && !drive && !disk && !host.dryRun && !pathShown && !wdShown && !tokenWarnings.length)) return null; const low = h?.disks.filter((d) => d.level !== 'ok').map((d) => `${d.path} ${d.freeBytes === undefined ? '?' : fmtBytes(d.freeBytes)} free`).join(', '); const title = (id: string) => app.sessions.find((s) => s.id === id)?.title ?? id; const bars: { key: string; kind: 'warn' | 'error'; lead: string; rest: string; fixed?: boolean }[] = []; @@ -237,6 +238,11 @@ function HostBanner({ host, app }: { host?: HostStatus; app: AppState }) { rest: `${list}${restarts.length > 3 ? `; ${restarts.length - 3} more` : ''}.`, }); } + // The Claude token pools (w739): a person's tokens used up or over their caps, and runs inside the dispatcher's reserve. + // The text names no token but its last four characters; a banner returns when its text changes. + for (const w of tokenWarnings) { + bars.push({ key: `token-${w.id}`, kind: w.kind === 'exhausted' ? 'error' : 'warn', lead: w.kind === 'reserve' ? 'The dispatcher\'s Claude token buffer is in use.' : w.kind === 'exhausted' ? 'A Claude token pool is used up.' : 'A Claude token pool is over its caps.', rest: w.text }); + } if (host.drain) { const d = host.drain; bars.push({ From a282f753f84107f6e183def5b9ca98fb6f352de5 Mon Sep 17 00:00:00 2001 From: Final Factory <481770+bryding@users.noreply.github.com> Date: Thu, 8 Oct 2026 22:21:09 -0600 Subject: [PATCH 2/2] w739: the pool test of the credential file uses an owner; the watchdog test waits up to 30 s for its test socket (a slow runner) Co-Authored-By: Claude Sonnet 5.5 --- deploy/vm/test/fff-watchdog.test.sh | 2 +- server/vault.test.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/deploy/vm/test/fff-watchdog.test.sh b/deploy/vm/test/fff-watchdog.test.sh index 67782cac..0871fc1a 100755 --- a/deploy/vm/test/fff-watchdog.test.sh +++ b/deploy/vm/test/fff-watchdog.test.sh @@ -77,7 +77,7 @@ command -v node >/dev/null || fail "node is needed" node -e 'require("net").createServer().listen(process.argv[1]); setInterval(() => {}, 1e6)' "$tmp/claude.sock" & NODE_PID=$! trap 'kill $NODE_PID 2>/dev/null || true; rm -rf "$tmp"' EXIT -for _ in $(seq 50); do [ -S "$tmp/claude.sock" ] && break; sleep 0.1; done +for _ in $(seq 300); do [ -S "$tmp/claude.sock" ] && break; sleep 0.1; done [ -S "$tmp/claude.sock" ] || fail "node did not make the test socket" mkunit() { # UNIT ACTIVE UNITFILESTATE [RESULT] [SINCE] diff --git a/server/vault.test.ts b/server/vault.test.ts index 249017e0..0ab00f29 100644 --- a/server/vault.test.ts +++ b/server/vault.test.ts @@ -160,8 +160,8 @@ test('vault: systemd credential file (0440 by the ACL mask) opens, in the creden assert.equal(readKey(cred, { credentialsDir: creds }).length, 32); const v = make(() => src); assert.equal(v.status().key, 'loaded', v.status().why ?? ''); - v.add({ name: 'a', kind: 'claude', value: A, share: 'anyone' }); - assert.equal(v.forRun({ machineId: 'm3', role: 'workers' }, { claude: true }).claude?.token, A); + v.add({ name: 'a', kind: 'claude', value: A, owner: 'ben', share: 'owner' }); + assert.equal(v.forRun({ machineId: 'm3', role: 'workers', userId: 'ben' }, { claude: true }).claude?.token, A); // The same mode anywhere else is refused: through config vault.keyFile, and as a bare path. const loose = path.join(dir, 'etc', 'loose.key');