From c55def6ee7a92e12afb64cb914a601d8a612e21b Mon Sep 17 00:00:00 2001 From: John Fawcett Date: Tue, 21 Jul 2026 13:28:42 -0500 Subject: [PATCH 01/10] docs(gastown): specify usage-based container billing --- .kilo/skills/specs/SKILL.md | 1 + .specs/gastown-usage-based-billing.md | 473 ++++++++++++++++++++++++++ 2 files changed, 474 insertions(+) create mode 100644 .specs/gastown-usage-based-billing.md diff --git a/.kilo/skills/specs/SKILL.md b/.kilo/skills/specs/SKILL.md index e4217c84e2c..0e5a14065ca 100644 --- a/.kilo/skills/specs/SKILL.md +++ b/.kilo/skills/specs/SKILL.md @@ -13,6 +13,7 @@ to the user if instructions or changes will cause deviations from the original i | Spec | Governs | |---|---| +| `.specs/gastown-usage-based-billing.md` | Gastown container pricing, payer attribution, metering lifecycle, and budget enforcement | | `.specs/kiloclaw-billing.md` | KiloClaw billing, pricing, invoicing, usage metering, payment flows | | `.specs/kiloclaw-billing-lifecycle.md` | KiloClaw billing lifecycle — credit-renewal orchestration safety | | `.specs/kiloclaw-composio.md` | KiloClaw Composio credential provisioning, injection, and sharing | diff --git a/.specs/gastown-usage-based-billing.md b/.specs/gastown-usage-based-billing.md new file mode 100644 index 00000000000..008f3984a14 --- /dev/null +++ b/.specs/gastown-usage-based-billing.md @@ -0,0 +1,473 @@ +# Gastown Usage-Based Billing + +## Role of This Document + +This spec defines how Gastown charges Kilo credits for Cloudflare Container usage. It is the +source of truth for pricing, payer attribution, metering lifecycle, budget enforcement, and +user-visible behavior. The `@kilocode/container-usage` interface described here is proposed +and is not yet available in production. + +## Status + +Draft -- created 2026-07-21. + +## Conventions + +The key words "MUST", "MUST NOT", "SHOULD", and "MAY" are to be interpreted as described in +BCP 14 when they appear in all capitals. + +## Overview + +Gastown will charge the owner of a town for the time its Cloudflare Container is awake. Charges +settle continuously against the owner's Kilo credit balance through the future +`@kilocode/container-usage` service. Gastown remains responsible for reporting lifecycle events +and enforcing budget verdicts; the metering service owns usage calculation, pricing, ledger +debits, idempotency, and balance evaluation. + +The initial price is **three times the attributable Cloudflare Container cost**. This is a +usage-based charge, not a subscription or flat-rate entitlement. + +## Definitions + +- **Interval**: One continuous runtime of a town's container, from a successful + `recordStart` through `recordStop`. +- **Awake time**: Seconds during which Cloudflare reports the container as running. Sleeping or + stopped time is not billable. +- **Payer**: The Kilo user or organization whose credits are debited. +- **Actor**: The user or Gastown automation that caused the container to run. +- **SKU**: The metering catalog entry for the configured Cloudflare Container instance type. + +## Pricing + +1. The customer charge MUST equal `3 x attributable Cloudflare Container cost` for the measured + awake time. +2. The attributable cost MUST be calculated by the metering service from a versioned SKU catalog. + Gastown MUST send the catalog SKU and MUST NOT embed Cloudflare prices in application code. +3. The initial production SKU MUST represent Gastown's configured `standard-4` container. A + container-size change MUST use a corresponding new or updated catalog entry before rollout. +4. Charges MUST settle in Kilo credits. One dollar of calculated charge uses one dollar of Kilo + credit balance under the existing credit conversion rules. +5. The initial scope includes Cloudflare Container compute and memory costs represented by the + SKU. Token inference, external model usage, network egress, and durable workspace storage are + outside this meter unless later added to the catalog explicitly. + +## Payer Attribution + +1. An org-owned town MUST debit `{ type: "org", id: owner_id }`. +2. A user-owned town MUST debit `{ type: "user", id: owner_id }`. +3. Legacy user towns without `owner_id` MAY fall back to `owner_user_id`. A town with no reliable + payer MUST NOT start billable work. +4. Interactive work SHOULD identify the authenticated user as the actor. Autonomous scheduling + MUST use a stable Gastown bot identity and MAY include the initiating user as `onBehalfOf` when + known. +5. Payer, SKU, and interval start time MUST remain immutable for an open interval. An ownership or + SKU change takes effect on the next container runtime. + +## Metering Lifecycle + +Gastown's `TownContainerDO` is the producer for `service: "gastown"`. It MUST durably retain enough +interval state to retry every call after a Durable Object restart. + +1. **Start:** Before billable work is admitted, Gastown MUST complete the admission check described + below and await an acknowledged `recordStart`. The context MUST include the Town Container DO ID + as `instanceId`, the town ID as `sessionId`, the configured SKU, payer, actor, and + `metadata.townId`. Failure of both Postgres and the failover buffer MUST block admission and be + retried; `durable: "buffer"` is a successful ack. +2. **Heartbeat:** Approximately every five minutes while `getState()` confirms the container is + running, Gastown MUST await `recordHeartbeat` with monotonic `seq`, measured + `usageSinceLast`, and the original context for self-healing. Gastown MUST NOT infer billable + time merely from the existence of a Town or Durable Object. +3. **Stop:** Every observed container stop MUST eventually produce an acknowledged `recordStop`. + Gastown MUST retry it durably until acknowledged, including after eviction, idle shutdown, + explicit destruction, health recovery, or budget enforcement. +4. All idempotency keys MUST follow the metering contract. Retries and `dedup: true` responses MUST + never create duplicate charges or intervals. +5. Gastown's existing agent heartbeats MUST NOT substitute for container metering heartbeats: a + town may contain many agents, but it has one billable container interval. + +Stop reasons map as follows: + +| Gastown event | Meter reason | +|---|---| +| Natural container exit | `exit` | +| Idle timeout or `sleepAfter` | `activity_expired` | +| Budget stop, explicit stop, eviction, destroy, or watchdog restart | `runtime_signal` | + +## Technical Architecture + +### Responsibility split + +| Component | Responsibility | +|---|---| +| UBB / `@kilocode/container-usage` | SKU catalog, admission decision, reservations, usage ledger, credit debit, balance floor, idempotency, and reconciliation | +| `TownDO` | Resolve payer and actor, gate every dispatch path, expose billing state to the UI, stop scheduling after a budget stop, and request usage heartbeats from the container DO | +| `TownContainerDO` | Own the durable interval state, guard cold starts, call `recordStart`/`recordHeartbeat`/`recordStop`, observe actual container runtime, and stop the container when instructed | +| Gastown tRPC | Verify town access, pass the authenticated actor to `TownDO`, return stable billing errors/status, and never make an independent billing decision | +| Web UI | Display estimates and meter status, prevent known-invalid actions, link to the correct credit owner, and recover after top-up | + +Gastown MUST receive the metering WorkerEntrypoint through a Cloudflare service binding. Production, +development, and test Wrangler configurations MUST bind the same typed interface; tests MAY use an +in-memory fake ledger. + +### Required admission contract + +The proposed three recording calls are not sufficient by themselves to prevent a cold start: +`recordStart` returns no budget verdict, while `recordHeartbeat` is defined only after +`getState()` confirms that the container is running. UBB therefore MUST provide an upstream +admission operation before paid rollout. This is a fourth operation, but it is not a usage-recording +call. + +Conceptually: + +```ts +type AuthorizeStartResult = + | { + verdict: 'allow'; + authorizationId: string; + expiresAt: number; + remaining?: number; + minimumRequired: number; + } + | { + verdict: 'deny'; + remaining: number; + minimumRequired: number; + }; +``` + +1. `authorizeStart` MUST atomically evaluate the payer's current available balance and reserve the + configured minimum-start amount. This prevents several towns owned by one payer from passing the + same balance check concurrently. +2. The reservation SHOULD cover at least one maximum-price heartbeat window. The exact minimum is + catalog configuration, not a Gastown constant. +3. Gastown MUST include the `authorizationId` in `recordStart` metadata. Opening the interval MUST + consume the reservation or associate it with that interval. An expired authorization MUST be + re-authorized, not reused. +4. A denied authorization MUST return `remaining` and `minimumRequired` so Gastown can produce an + actionable response without querying a second balance source. +5. If UBB chooses not to implement reservations initially, it MUST document the bounded race: two + concurrent cold starts may both pass against the same credits and can consume up to one + heartbeat window before being stopped. This weaker mode is acceptable only during shadow rollout. + +An equivalent design MAY add an atomic budget verdict and reservation to `recordStart`, provided it +can complete before the Cloudflare container is started. A browser-only check against +`user.getContextBalance` is useful for UX but MUST NOT be the authoritative admission gate because +autonomous scheduling and direct service calls bypass the browser. + +### Durable interval state + +`TownContainerDO` SHOULD persist one state record with these conceptual fields: + +```ts +type ContainerUsageState = + | { phase: 'idle' } + | { + phase: 'starting' | 'running' | 'stopping'; + context: UsageContext; + authorizationId: string; + startEpochMs: number; + seq: number; + lastReportedAt: number; + pendingHeartbeat?: { + seq: number; + observedAt: number; + usageSinceLast: number; + idempotencyKey: string; + }; + stopReason?: 'exit' | 'runtime_signal' | 'activity_expired'; + }; +``` + +The concrete layout MAY differ, but it MUST provide these guarantees: + +1. Exactly one open interval exists per `TownContainerDO`. +2. The heartbeat sequence and payload are persisted before the RPC. A retry after a timeout or DO + restart sends the identical sequence, usage, observation time, and idempotency key. +3. `lastReportedAt` advances only after an ack. A lost response therefore retries rather than + dropping usage. +4. A pending stop is retained until `recordStop` is acknowledged. Starting a new runtime creates a + new `startEpochMs`; it never reopens or reuses the old interval. +5. Heartbeat, stop, and restart transitions are serialized so the final usage slice cannot race a + normal heartbeat. + +### Reporting awake time + +1. `TownContainerDO` lifecycle observations are the source of truth for billable runtime. Agent + count, PTY connections, bead status, and browser presence MUST NOT directly produce usage. +2. Awake time begins when the Cloudflare container runtime starts and ends when it stops. The ten + minute `sleepAfter` period and any other time the runtime remains awake are billable because they + incur Cloudflare cost. +3. The existing `TownDO` alarm loop SHOULD call a rate-limited + `TownContainerDO.recordUsageHeartbeat()` operation. That operation MUST no-op until approximately + five minutes have passed, then confirm runtime state and report usage. This avoids introducing an + alarm implementation that may conflict with the `Container` base class's sleep behavior. +4. While agents are active, the Town alarm currently runs every five seconds; while idle, it runs + every five minutes. Both paths MUST eventually invoke the same rate-limited usage operation. +5. `onStart` and `onStop` SHOULD enqueue lifecycle work with `ctx.waitUntil`. `onStop` MUST report the + final partial window even when fewer than five minutes elapsed. +6. Usage MUST be measured from persisted runtime timestamps, rounded according to the UBB contract, + and submitted as awake seconds. Gastown MUST NOT derive cost locally. +7. If an open interval is found after a restart but the runtime is stopped, Gastown MUST close it + using the last reliable observation and flag it for reconciliation rather than inventing an + unbounded runtime duration. + +### Cold-start gate + +All paths that can wake a container MUST converge on one gate. Today these include eager mayor +startup, mayor messages, bead dispatch, alarm-driven `ensureContainerReady`, health recovery, +terminal/stream proxy requests, and direct container fetches. + +The gate behaves as follows: + +1. If the runtime is already running and has an open interval, allow the request. Admission is not + repeated for every agent because the town has one shared container. +2. If the town is in `billing_stopping` or `billing_blocked`, reject new dispatch and do not call a + container proxy that would wake the runtime. +3. For a cold start, resolve payer and actor, call `authorizeStart`, persist the authorization and + context in `TownContainerDO`, await `recordStart`, and only then call `startAndWaitForPorts` or a + proxy operation that can boot the container. +4. If authorization is denied, return the stable domain code `INSUFFICIENT_CREDITS` with payer type, + remaining balance, and minimum required balance. Do not mutate a bead or agent to `working`. +5. If admission infrastructure is unavailable, return `BILLING_UNAVAILABLE` and fail closed for the + cold start. An already-running interval continues and fails open as described under Failure + Handling. +6. If container startup fails after `recordStart`, close the zero- or near-zero-usage interval with + `runtime_signal` and release unused admission reservation. +7. `TownContainerDO` MUST defend the boundary as well as `TownDO`: a request that could cold-start + the runtime without prepared metering context MUST be rejected. This prevents a new proxy route + from accidentally bypassing billing. + +The scheduler currently marks a bead and agent as active before asynchronous dispatch. Billing +admission MUST happen before those transitions, or a denied start MUST synchronously restore the +previous states. The preferred implementation is to move admission before the state mutation. + +### Heartbeat and low-balance stop + +The response to `recordHeartbeat` is authoritative for a running interval: + +1. On `continue`, persist the ack, publish the new remaining balance to `TownDO`, and schedule the + next report. +2. On `warn`, do the same and persist a `billing_warning` state. Existing work continues and new work + MAY still dispatch while the shared container remains running. Gastown MUST de-duplicate user + notifications for the same low-balance episode. +3. On `stop`, atomically persist `billing_stopping` before any network operation. The Town scheduler + MUST stop dispatching new agents and `ensureContainerReady` MUST not restart the runtime. +4. Gastown MUST trigger the existing graceful eviction/drain path so active agents save checkpoints, + database snapshots, and WIP branches. Shutdown receives a configurable deadline; after it expires, + Gastown MAY force-stop the container to cap additional spend. +5. After the drain, call `container.stop()`, report the final usage slice, and retry `recordStop` until + acknowledged. Then persist `billing_blocked` with the latest `remaining` value. +6. Container `onStop` and the explicit budget-stop path MUST be idempotent. Either may observe the + stop first, but together they produce one final usage segment and one metering stop record. +7. The ordinary alarm reconciler and health watchdog MUST treat `billing_stopping` and + `billing_blocked` as intentional downtime. They MUST NOT classify it as a failed container or + auto-restart it. +8. A later explicit user action or pending scheduler action MAY re-run admission. On success, + Gastown clears `billing_blocked`, opens a new interval, restores saved work, and resumes. A balance + query alone MUST NOT wake the container. + +### Gastown API changes + +Gastown SHOULD expose a typed billing status through tRPC and the Town status WebSocket: + +```ts +type GastownBillingStatus = { + state: 'idle' | 'starting' | 'running' | 'warning' | 'stopping' | 'blocked' | 'degraded'; + payer: { type: 'user' | 'org'; id: string }; + remaining?: number; + minimumRequired?: number; + estimatedHourlyCharge?: number; + intervalStartedAt?: number; + lastReportedAt?: number; +}; +``` + +1. A `getBillingStatus` query MUST verify town access and return only the caller-safe status. It MUST + NOT expose ledger internals, admission IDs, idempotency keys, or another payer's data. +2. The Town status WebSocket SHOULD publish state transitions so warning and stop UX does not wait + for polling. +3. Interactive mutations that can cold-start a town MUST return stable error codes for + `INSUFFICIENT_CREDITS` and `BILLING_UNAVAILABLE`. The frontend MUST NOT parse human-readable error + strings to choose behavior. +4. The authenticated user ID from tRPC MUST be passed as the actor for interactive starts. Alarm and + reconciler starts use a stable bot actor. +5. Admin access MUST not shift the charge to the admin. Support actions use the town's configured + payer and identify the admin only as actor. + +### Expected code changes + +| Area | Expected change | +|---|---| +| `services/gastown/wrangler.jsonc` and test config | Add the UBB WorkerEntrypoint service binding and rollout flags. | +| `services/gastown/worker-configuration.d.ts` | Add the typed admission and recording RPC contract until generated binding types provide it. | +| `services/gastown/package.json` | Add `@kilocode/container-usage` once the package exists. | +| `services/gastown/src/dos/TownContainer.do.ts` | Persist interval state, guard cold starts, report lifecycle calls, expose a rate-limited usage heartbeat, and make stop handling idempotent. | +| `services/gastown/src/dos/Town.do.ts` | Resolve billing context, retain public billing state, invoke usage reporting from the existing alarm, and suppress restart while billing-blocked. | +| `services/gastown/src/dos/town/scheduling.ts` | Run admission before mutating beads and agents to active states. | +| `services/gastown/src/dos/town/container-dispatch.ts` | Route all cold starts through the metered gate and carry the actor/context. | +| `services/gastown/src/trpc/schemas.ts` and `router.ts` | Add billing status outputs, stable billing errors, and authenticated actor propagation. | +| `apps/web/src/components/gastown/TerminalBar.tsx` and `MayorChat.tsx` | Remove mount-time `ensureMayor`, render billing states, and stop terminal reconnect loops while blocked. | +| Gastown town page shell | Add the compact usage indicator, warning/paused banner, and payer-aware credit action. | + +The exact module split MAY change during implementation, but no route or scheduler path may retain a +direct, unmetered cold start. + +## Budget Enforcement + +1. Gastown MUST enforce the `budget.verdict` returned by each heartbeat. +2. On `continue`, Gastown MUST keep running and schedule the next heartbeat. This includes the + metering service's fail-open response when its ledger is temporarily unreadable. +3. On `warn`, Gastown MUST keep running and SHOULD notify connected users that the town is nearing + its credit limit. The warning SHOULD include `remaining` when supplied. +4. On `stop`, Gastown MUST stop dispatching new work, request the container's existing graceful + drain/save behavior, then stop the container and close the interval. Work already in a + non-interruptible save step MAY finish before shutdown. +5. A budget stop MUST preserve the town's durable control state and recoverable agent work to the + same standard as a platform eviction. Adding credits MUST allow the next request or scheduler + action to start a new interval and continue the town. +6. Gastown MUST NOT independently calculate a balance floor or override a `stop` verdict based on + a cached balance. + +## User Experience + +### Do not bill for viewing a town + +The current Mayor UI calls `ensureMayor` on mount, and `ensureMayor` eagerly starts an idle container. +Paid rollout MUST remove this behavior. Loading a town page, polling status, or expanding the +terminal MUST NOT by itself create container usage. + +The page SHOULD first show the saved town state and a stopped terminal. A container starts only when +the user sends work, explicitly selects **Start Mayor**, or autonomous queued work is eligible to +run. If product requirements retain eager prewarming, the UI MUST disclose that opening the town +starts billable usage; the default remains no eager start. + +### Status and controls + +The UI SHOULD use the server's `GastownBillingStatus` rather than calculate state from a separately +cached balance. A personal balance query or `user.getContextBalance` MAY prefetch display data, but +server admission remains authoritative. + +| State | UI behavior | +|---|---| +| `idle` | Show the estimated active hourly charge near the start control. Do not show a running cost counter. | +| `starting` | Disable duplicate starts and show `Starting Gas Town...` | +| `running` | Show a compact active-usage indicator with elapsed runtime, estimated charge so far, and remaining credits when available. | +| `warning` | Show a persistent warning banner and keep the terminal usable. Primary action: **Add credits**. | +| `stopping` | Disable new work and show `Saving work and pausing Gas Town...` Do not show a retry action until save/stop completes. | +| `blocked` | Replace reconnect loops with a paused state explaining the required balance. Primary action: **Add credits**; after funding, **Resume Gas Town**. | +| `degraded` | Keep running. Avoid alarming the user for a transient buffered write; show service degradation only if action is required or the delay is prolonged. | + +Recommended copy: + +- Before start: `Container usage is billed at an estimated {rate}/hour while Gas Town is running.` +- Admission denied: `Gas Town needs at least {minimum} in credits to start. {remaining} is available.` +- Warning: `Credits are running low. Gas Town will save its work and pause when the balance reaches the limit.` +- Stopping: `Saving work and pausing Gas Town...` +- Paused: `Gas Town is paused. Your work was saved. Add credits, then resume when you are ready.` +- Billing unavailable: `Gas Town cannot verify billing right now. Try starting it again in a minute.` + +The warning UI MUST use the warning status domain, while a completed budget pause is informational, +not destructive: the user's town and work have not been deleted. **Add credits** is the single +primary CTA. Amounts and usage counters SHOULD use tabular numerals. + +### Personal and organization payers + +1. Personal towns link **Add credits** to `/credits` and preserve a return URL to the town. +2. Organization towns link users with billing permission to the organization's credit-management + surface and preserve the return URL. +3. Organization members without billing permission SHOULD see who can resolve the issue and an + action such as **View organization billing** rather than a purchase control they cannot use. +4. Every balance message MUST name the payer context, for example `Your credits` or + `Acme organization credits`, so users understand which wallet controls the town. + +### Estimates and receipts + +1. Estimates MUST be labeled as estimates because the final debit comes from measured awake time. +2. The displayed hourly estimate MUST come from the same versioned SKU price used by UBB, including + the `3x` multiplier. The frontend MUST NOT duplicate the rate card. +3. The running cost estimate MAY be calculated from the interval start and hourly estimate for + responsiveness, but it MUST be replaced by settled usage when available and MUST NOT be presented + as an exact ledger balance. +4. Customer billing history SHOULD group debits by town and interval, showing town name, runtime, + customer rate, total charge, start/stop timestamps, and stop reason. Base Cloudflare cost and the + internal multiplier belong in staff-only reconciliation tooling, not customer receipts. +5. Warning and pause states MUST work on mobile, remain keyboard accessible, and be announced through + an appropriate live region without repeatedly announcing every balance refresh. + +## Failure Handling + +1. A metering ack with `durable: "buffer"` MUST be treated as success and monitored as degraded + operation. +2. If a heartbeat throws because both persistence paths are unavailable, Gastown SHOULD fail open + for the running interval, retry with the same idempotency key, and alert. It MUST NOT terminate + customer work solely because the billing service is temporarily unreachable. +3. If `recordStop` cannot be acknowledged during shutdown, the pending stop MUST survive and retry + later. A missing stop ack MUST NOT cause the next runtime to reuse the previous interval. +4. Metering errors and retries MUST NOT log credit balances, tokens, or other credentials. + +## Rollout And Acceptance + +Rollout SHOULD proceed in independently reversible stages: + +1. **Contract and catalog:** Ship the UBB service binding, `standard-4` SKU, `3x` customer rate, and + admission operation behind disabled flags. +2. **Shadow reporting:** Emit complete intervals and calculated charges without reserving or + debiting credits and without enforcing `warn`/`stop`. +3. **Admission and UX:** Enable authoritative cold-start admission and all blocked/warning UI while + usage debits remain shadowed. Exercise personal and organization payer flows. +4. **Charging:** Enable ledger debits while heartbeat `stop` remains observable but unenforced for a + short canary period. +5. **Enforcement:** Enable graceful budget stops for internal users, then a percentage rollout, then + all paid Gastown traffic. + +Before charging customers, shadow data MUST demonstrate that starts and stops reconcile with +Cloudflare runtime observations, duplicate RPCs do not duplicate charges, and aggregate base cost is +reasonably consistent with the Cloudflare invoice. Admission, debits, warnings, and stops MUST have +separate kill switches so metering can continue while a control behavior is disabled. + +Dashboards MUST expose awake seconds, base Cloudflare cost, customer charge, gross multiplier, +buffered writes, unclosed intervals, admission denials, RPC failures, graceful-stop duration, forced +stops, and counts of `warn` and `stop` verdicts. + +### Required verification + +1. Unit tests MUST cover every durable interval transition, identical retry payloads, dedup acks, + stop/heartbeat races, ownership mapping, and billing-blocked scheduler behavior. +2. Worker integration tests MUST use a fake UBB binding to cover PG-buffer acks, total failure, + allow/deny admission, all three heartbeat verdicts, lost responses, DO restart between calls, + idle stop, watchdog restart, and final partial-window settlement. +3. Concurrency tests MUST prove two towns cannot consume the same minimum-start reservation for one + payer. +4. End-to-end tests MUST cover a personal town and an organization town through start, warning, + graceful pause, top-up, and resume. They MUST also prove that merely opening the town page creates + no interval and no Cloudflare container start. +5. Reconciliation tests MUST compare sampled intervals with Cloudflare runtime data and verify the + configured customer charge is exactly three times catalog base cost, subject only to documented + rounding. +6. UX verification MUST cover desktop and mobile layouts, keyboard operation, live-region + announcements, organization members without billing permission, and a prolonged billing-service + outage. + +## Open Decisions + +1. Confirm which Cloudflare cost components belong in the initial SKU, especially disk and egress. +2. Choose the exact town-shell placement for the active estimate, warning banner, and interval + receipt link. +3. Define the maximum graceful-drain duration after a `stop` verdict. +4. Confirm whether legacy towns with ambiguous ownership are migrated or blocked from paid use. +5. Finalize whether admission is a separate `authorizeStart` RPC or an atomic extension to + `recordStart`. +6. Set the minimum-start reservation and `warn` threshold from the final SKU rate and desired + graceful-save window. + +## Changelog + +### 2026-07-21 -- Technical design expansion + +- Added admission, durable metering state, reporting, low-balance shutdown, API, UI, rollout, and + verification design. + +### 2026-07-21 -- Initial draft + +- Defined `3x` Cloudflare Container cost pricing and the proposed + `@kilocode/container-usage` integration for Gastown. From 0d9e31d8a8005ca68e59d6acd65aa6b4905f1706 Mon Sep 17 00:00:00 2001 From: John Fawcett Date: Wed, 22 Jul 2026 09:01:01 -0500 Subject: [PATCH 02/10] feat(gastown): add usage-based container billing --- .specs/gastown-usage-based-billing.md | 15 +- ENVIRONMENT.md | 2 + .../[townId]/TownOverviewPageClient.tsx | 18 + .../src/app/(app)/gastown/[townId]/page.tsx | 8 +- .../[id]/gastown/[townId]/page.tsx | 8 +- apps/web/src/components/gastown/MayorChat.tsx | 10 +- .../src/components/gastown/TerminalBar.tsx | 137 ++++- .../web/src/components/gastown/useXtermPty.ts | 7 +- apps/web/src/lib/config.server.ts | 2 + apps/web/src/lib/gastown/types/router.d.ts | 78 +++ .../src/billing/ContainerBilling.error.ts | 10 + .../container-usage-state.billing.test.ts | 73 +++ .../billing/container-usage-state.billing.ts | 73 +++ .../billing/container-usage.billing.test.ts | 60 ++ .../src/billing/container-usage.billing.ts | 144 +++++ services/gastown/src/dos/Town.do.ts | 163 +++++- services/gastown/src/dos/TownContainer.do.ts | 536 +++++++++++++++++- services/gastown/src/dos/town/scheduling.ts | 2 + services/gastown/src/trpc/router.ts | 45 +- services/gastown/src/trpc/schemas.ts | 13 + .../test/integration/billing-disabled.test.ts | 15 + services/gastown/tsconfig.types.json | 1 + services/gastown/worker-configuration.d.ts | 5 + services/gastown/wrangler.jsonc | 2 + services/gastown/wrangler.test.jsonc | 1 + 25 files changed, 1385 insertions(+), 43 deletions(-) create mode 100644 services/gastown/src/billing/ContainerBilling.error.ts create mode 100644 services/gastown/src/billing/container-usage-state.billing.test.ts create mode 100644 services/gastown/src/billing/container-usage-state.billing.ts create mode 100644 services/gastown/src/billing/container-usage.billing.test.ts create mode 100644 services/gastown/src/billing/container-usage.billing.ts create mode 100644 services/gastown/test/integration/billing-disabled.test.ts diff --git a/.specs/gastown-usage-based-billing.md b/.specs/gastown-usage-based-billing.md index 008f3984a14..4eb3355cd9d 100644 --- a/.specs/gastown-usage-based-billing.md +++ b/.specs/gastown-usage-based-billing.md @@ -298,7 +298,7 @@ type GastownBillingStatus = { | Area | Expected change | |---|---| -| `services/gastown/wrangler.jsonc` and test config | Add the UBB WorkerEntrypoint service binding and rollout flags. | +| `services/gastown/wrangler.jsonc` and test config | Add the default-off `GASTOWN_BILLING_ENABLED` flag; add the production UBB binding when that Worker is provisioned. | | `services/gastown/worker-configuration.d.ts` | Add the typed admission and recording RPC contract until generated binding types provide it. | | `services/gastown/package.json` | Add `@kilocode/container-usage` once the package exists. | | `services/gastown/src/dos/TownContainer.do.ts` | Persist interval state, guard cold starts, report lifecycle calls, expose a rate-limited usage heartbeat, and make stop handling idempotent. | @@ -407,10 +407,11 @@ primary CTA. Amounts and usage counters SHOULD use tabular numerals. ## Rollout And Acceptance -Rollout SHOULD proceed in independently reversible stages: +Rollout SHOULD proceed through a single default-off backend flag: -1. **Contract and catalog:** Ship the UBB service binding, `standard-4` SKU, `3x` customer rate, and - admission operation behind disabled flags. +1. **Contract and catalog:** Ship the `standard-4` SKU, `3x` customer rate, and admission operation + behind `GASTOWN_BILLING_ENABLED=false`; provision the production UBB service binding before the + flag can be enabled. 2. **Shadow reporting:** Emit complete intervals and calculated charges without reserving or debiting credits and without enforcing `warn`/`stop`. 3. **Admission and UX:** Enable authoritative cold-start admission and all blocked/warning UI while @@ -422,8 +423,10 @@ Rollout SHOULD proceed in independently reversible stages: Before charging customers, shadow data MUST demonstrate that starts and stops reconcile with Cloudflare runtime observations, duplicate RPCs do not duplicate charges, and aggregate base cost is -reasonably consistent with the Cloudflare invoice. Admission, debits, warnings, and stops MUST have -separate kill switches so metering can continue while a control behavior is disabled. +reasonably consistent with the Cloudflare invoice. `GASTOWN_BILLING_ENABLED` controls metering, +admission, debits, warnings, and stops together. A separate default-off +`GASTOWN_BILLING_ANNOUNCEMENT_ENABLED` web flag MAY show advance notice while actual billing remains +disabled; it MUST NOT alter backend billing behavior. Dashboards MUST expose awake seconds, base Cloudflare cost, customer charge, gross multiplier, buffered writes, unclosed intervals, admission denials, RPC failures, graceful-stop duration, forced diff --git a/ENVIRONMENT.md b/ENVIRONMENT.md index 152ed701c1b..20d8d46ca35 100644 --- a/ENVIRONMENT.md +++ b/ENVIRONMENT.md @@ -205,6 +205,8 @@ Manage shared web env var additions and rotations with `pnpm web:env set = { @@ -94,6 +97,7 @@ export function TownOverviewPageClient({ townId, basePath: basePathOverride, organizationId, + billingAnnouncementEnabled, }: TownOverviewPageClientProps) { const townBasePath = basePathOverride ?? `/gastown/${townId}`; const router = useRouter(); @@ -215,6 +219,20 @@ export function TownOverviewPageClient({
+ {billingAnnouncementEnabled && ( + + + + + Usage-based container billing is coming + + Charges will use Kilo credits only while the town container is running. We'll share + more details before billing is activated. + + + + )}
{/* Top bar — sticky */}
diff --git a/apps/web/src/app/(app)/gastown/[townId]/page.tsx b/apps/web/src/app/(app)/gastown/[townId]/page.tsx index 677d6c1e367..50a25e1f2e9 100644 --- a/apps/web/src/app/(app)/gastown/[townId]/page.tsx +++ b/apps/web/src/app/(app)/gastown/[townId]/page.tsx @@ -1,6 +1,7 @@ import { getUserFromAuthOrRedirect } from '@/lib/user/server'; import { notFound } from 'next/navigation'; import { isGastownEnabled } from '@/lib/gastown/feature-flags'; +import { GASTOWN_BILLING_ANNOUNCEMENT_ENABLED } from '@/lib/config.server'; import { TownOverviewPageClient } from './TownOverviewPageClient'; export default async function TownOverviewPage({ @@ -15,5 +16,10 @@ export default async function TownOverviewPage({ return notFound(); } - return ; + return ( + + ); } diff --git a/apps/web/src/app/(app)/organizations/[id]/gastown/[townId]/page.tsx b/apps/web/src/app/(app)/organizations/[id]/gastown/[townId]/page.tsx index e1aef4452b6..249b38daf03 100644 --- a/apps/web/src/app/(app)/organizations/[id]/gastown/[townId]/page.tsx +++ b/apps/web/src/app/(app)/organizations/[id]/gastown/[townId]/page.tsx @@ -1,5 +1,6 @@ import { OrganizationByPageLayout } from '@/components/organizations/OrganizationByPageLayout'; import { TownOverviewPageClient } from '@/app/(app)/gastown/[townId]/TownOverviewPageClient'; +import { GASTOWN_BILLING_ANNOUNCEMENT_ENABLED } from '@/lib/config.server'; export default async function OrgTownOverviewPage({ params, @@ -13,7 +14,12 @@ export default async function OrgTownOverviewPage({ params={params} fullBleed render={() => ( - + )} /> ); diff --git a/apps/web/src/components/gastown/MayorChat.tsx b/apps/web/src/components/gastown/MayorChat.tsx index f774f71cad0..c1640f82ed0 100644 --- a/apps/web/src/components/gastown/MayorChat.tsx +++ b/apps/web/src/components/gastown/MayorChat.tsx @@ -18,6 +18,7 @@ export function MayorChat({ townId }: MayorChatProps) { const trpc = useGastownTRPC(); const queryClient = useQueryClient(); const [collapsed, setCollapsed] = useState(false); + const billingQuery = useQuery(trpc.gastown.getBillingStatus.queryOptions({ townId })); // Eagerly ensure mayor agent + container on mount const ensureMayor = useMutation( @@ -48,10 +49,11 @@ export function MayorChat({ townId }: MayorChatProps) { // Reset on townId change so ensureMayor fires for each town const ensuredTownRef = useRef(null); useEffect(() => { + if (!billingQuery.isSuccess || billingQuery.data.enabled) return; if (ensuredTownRef.current === townId) return; ensuredTownRef.current = townId; ensureMayor.mutate({ townId }); - }, [townId]); + }, [billingQuery.data, billingQuery.isSuccess, townId]); // Poll mayor status to get agentId const statusQuery = useQuery({ @@ -69,6 +71,12 @@ export function MayorChat({ townId }: MayorChatProps) { const { terminalRef, connected, status, fitAddonRef } = useXtermPty({ townId, agentId: mayorAgentId, + enabled: + billingQuery.isSuccess && + (!billingQuery.data.enabled || + billingQuery.data.state === 'running' || + billingQuery.data.state === 'warning' || + billingQuery.data.state === 'starting'), retries: 10, retryDelay: 3_000, }); diff --git a/apps/web/src/components/gastown/TerminalBar.tsx b/apps/web/src/components/gastown/TerminalBar.tsx index 5eb7dc0d53d..12011242b89 100644 --- a/apps/web/src/components/gastown/TerminalBar.tsx +++ b/apps/web/src/components/gastown/TerminalBar.tsx @@ -3,6 +3,7 @@ import { useCallback, useEffect, useRef, useState } from 'react'; import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'; import { useRouter } from 'next/navigation'; +import Link from 'next/link'; import { useGastownTRPC, gastownWsUrl, type GastownOutputs } from '@/lib/gastown/trpc'; import { useSidebar } from '@/components/ui/sidebar'; @@ -38,6 +39,7 @@ import { Bug, Github, MessageCircle, + CircleDollarSign, } from 'lucide-react'; import { motion, AnimatePresence } from 'motion/react'; import styles from './TerminalBar.module.css'; @@ -48,6 +50,8 @@ type TerminalBarProps = { basePath?: string; }; +type BillingStatus = GastownOutputs['gastown']['getBillingStatus']; + /** * Unified terminal bar. Always shows a Mayor tab (non-closeable). * Agent terminal tabs are opened/closed via TerminalBarContext. @@ -358,6 +362,7 @@ export function TerminalBar({ townId, basePath: basePathOverride }: TerminalBarP setCollapsed={setCollapsed} setPosition={setPosition} closeTab={closeTab} + billing={alarmWs.data?.billing} /> {!collapsed && (
void; setPosition: (position: TerminalPosition) => void; closeTab: (tabId: string) => void; + billing?: BillingStatus; }) { const borderClass = horizontal ? 'border-b border-white/[0.06]' : 'border-r border-white/[0.06]'; @@ -593,6 +603,38 @@ function TabBar({
+ {horizontal && billing?.enabled && billing.state !== 'idle' && ( +
+ + {billing.state === 'warning' + ? 'Credits running low' + : billing.state === 'blocked' || billing.state === 'stopping' + ? 'Usage paused' + : billing.estimatedHourlyCharge === undefined + ? 'Usage billing active' + : `$${billing.estimatedHourlyCharge.toFixed(2)}/hr active`} + {billing.state === 'warning' && ( + + Add credits + + )} +
+ )} + {/* Bug report dropdown */} {horizontal && (
@@ -738,11 +780,19 @@ function TerminalContent({ className={`overflow-hidden ${horizontal ? '' : 'h-full'} ${fullscreen ? 'h-full' : ''}`} > {activeTab.kind === 'mayor' ? ( - + ) : activeTab.kind === 'status' ? ( ) : ( - + )} @@ -770,6 +820,7 @@ type AlarmStatus = { recentEvents: Array<{ time: string; type: string; message: string }>; draining?: boolean; drainStartedAt?: string; + billing: BillingStatus; }; type AgentStatusEvent = { @@ -1193,7 +1244,15 @@ function TerminalStatusBadge({ const FIRST_TASK_STORAGE_PREFIX = 'gastown_first_task_'; -function MayorTerminalPane({ townId, collapsed }: { townId: string; collapsed: boolean }) { +function MayorTerminalPane({ + townId, + collapsed, + billing, +}: { + townId: string; + collapsed: boolean; + billing?: BillingStatus; +}) { const trpc = useGastownTRPC(); const queryClient = useQueryClient(); @@ -1224,10 +1283,11 @@ function MayorTerminalPane({ townId, collapsed }: { townId: string; collapsed: b const ensuredTownRef = useRef(null); useEffect(() => { + if (!billing || billing.enabled) return; if (ensuredTownRef.current === townId) return; ensuredTownRef.current = townId; ensureMayor.mutate({ townId }); - }, [townId]); + }, [billing, townId]); const statusQuery = useQuery({ ...trpc.gastown.getMayorStatus.queryOptions({ townId }), @@ -1240,6 +1300,16 @@ function MayorTerminalPane({ townId, collapsed }: { townId: string; collapsed: b }); const mayorAgentId = statusQuery.data?.session?.agentId ?? null; + const billingPaused = + billing?.enabled && (billing.state === 'blocked' || billing.state === 'stopping'); + const terminalEnabled = + billing !== undefined && + (!billing.enabled || + billing.state === 'running' || + billing.state === 'warning' || + billing.state === 'starting'); + const creditsHref = + billing?.payer?.type === 'org' ? `/organizations/${billing.payer.id}` : '/credits'; // Send a queued first task from the onboarding wizard via the sendMessage // tRPC procedure (goes through the SDK's session.prompt API, not PTY stdin). @@ -1262,6 +1332,7 @@ function MayorTerminalPane({ townId, collapsed }: { townId: string; collapsed: b const { terminalRef, connectionStatus, status, fitAddonRef } = useXtermPty({ townId, agentId: mayorAgentId, + enabled: terminalEnabled, retries: 10, retryDelay: 3_000, }); @@ -1280,16 +1351,72 @@ function MayorTerminalPane({ townId, collapsed }: { townId: string; collapsed: b
+ {billing?.enabled && !terminalEnabled && ( +
+
+ +

+ {billingPaused ? 'Gas Town is paused' : 'Start the Mayor when you are ready'} +

+

+ {billingPaused + ? 'Add credits to the billing account, then resume. Your town state is preserved.' + : billing.estimatedHourlyCharge === undefined + ? 'Container usage is charged only while Gas Town is running.' + : `Estimated container charge: $${billing.estimatedHourlyCharge.toFixed(2)}/hour while running.`} +

+
+ {billingPaused && ( + + Add credits + + )} + +
+ {ensureMayor.error && ( +

+ {ensureMayor.error.message} +

+ )} +
+
+ )}
); } // ── Agent Terminal Pane ────────────────────────────────────────────────── -function AgentTerminalPane({ townId, agentId }: { townId: string; agentId: string }) { +function AgentTerminalPane({ + townId, + agentId, + billing, +}: { + townId: string; + agentId: string; + billing?: BillingStatus; +}) { + const enabled = + billing !== undefined && + (!billing.enabled || billing.state === 'running' || billing.state === 'warning'); const { terminalRef, connectionStatus, status } = useXtermPty({ townId, agentId, + enabled, }); return ( diff --git a/apps/web/src/components/gastown/useXtermPty.ts b/apps/web/src/components/gastown/useXtermPty.ts index 538f0b4f4cc..3e157bea2bc 100644 --- a/apps/web/src/components/gastown/useXtermPty.ts +++ b/apps/web/src/components/gastown/useXtermPty.ts @@ -54,6 +54,8 @@ export function attachKittyEnterHandler(term: Terminal, wsRef?: React.RefObject< type XtermPtyOptions = { townId: string; agentId: string | null; + /** Whether PTY creation and reconnects are allowed. */ + enabled?: boolean; /** Number of retry attempts for PTY creation (default: 1, no retries). */ retries?: number; /** Delay in ms between retries (default: 3000). */ @@ -99,6 +101,7 @@ const RECONNECT_MAX_DELAY_MS = 8_000; export function useXtermPty({ townId, agentId, + enabled = true, retries = 1, retryDelay = 3_000, onStatusChange, @@ -140,7 +143,7 @@ export function useXtermPty({ const connectedAgentRef = useRef(null); useEffect(() => { - if (!agentId || agentId === connectedAgentRef.current) return; + if (!enabled || !agentId || agentId === connectedAgentRef.current) return; const capturedAgentId = agentId; connectedAgentRef.current = capturedAgentId; @@ -416,7 +419,7 @@ export function useXtermPty({ ptyRef.current = null; connectedAgentRef.current = null; }; - }, [agentId, townId]); + }, [agentId, enabled, townId]); return { terminalRef, connected, connectionStatus, status, fitAddonRef }; } diff --git a/apps/web/src/lib/config.server.ts b/apps/web/src/lib/config.server.ts index 22a3a84c036..8bd337eb710 100644 --- a/apps/web/src/lib/config.server.ts +++ b/apps/web/src/lib/config.server.ts @@ -253,6 +253,8 @@ export const AGENT_ENV_VARS_PUBLIC_KEY = getEnvVariable('AGENT_ENV_VARS_PUBLIC_K export const GASTOWN_SERVICE_URL = getEnvVariable('GASTOWN_SERVICE_URL') || (process.env.NODE_ENV === 'production' ? 'https://gastown.kiloapps.io' : null); +export const GASTOWN_BILLING_ANNOUNCEMENT_ENABLED = + getEnvVariable('GASTOWN_BILLING_ANNOUNCEMENT_ENABLED') === 'true'; export const GASTOWN_CF_ACCESS_CLIENT_ID = getEnvVariable('GASTOWN_SERVICE_CF_ACCESS_CLIENT_ID'); export const GASTOWN_CF_ACCESS_CLIENT_SECRET = getEnvVariable( 'GASTOWN_SERVICE_CF_ACCESS_CLIENT_SECRET' diff --git a/apps/web/src/lib/gastown/types/router.d.ts b/apps/web/src/lib/gastown/types/router.d.ts index 530a9c6ff56..c2f0652743d 100644 --- a/apps/web/src/lib/gastown/types/router.d.ts +++ b/apps/web/src/lib/gastown/types/router.d.ts @@ -550,6 +550,38 @@ export declare const gastownRouter: import('@trpc/server').TRPCBuiltRouter< type: string; message: string; }[]; + billing: { + enabled: boolean; + state: 'blocked' | 'degraded' | 'idle' | 'running' | 'starting' | 'stopping' | 'warning'; + payer?: { + type: 'org' | 'user'; + id: string; + } | undefined; + remaining?: number | undefined; + minimumRequired?: number | undefined; + estimatedHourlyCharge?: number | undefined; + intervalStartedAt?: number | undefined; + lastReportedAt?: number | undefined; + }; + }; + meta: object; + }>; + getBillingStatus: import('@trpc/server').TRPCQueryProcedure<{ + input: { + townId: string; + }; + output: { + enabled: boolean; + state: 'blocked' | 'degraded' | 'idle' | 'running' | 'starting' | 'stopping' | 'warning'; + payer?: { + type: 'org' | 'user'; + id: string; + } | undefined; + remaining?: number | undefined; + minimumRequired?: number | undefined; + estimatedHourlyCharge?: number | undefined; + intervalStartedAt?: number | undefined; + lastReportedAt?: number | undefined; }; meta: object; }>; @@ -2184,6 +2216,52 @@ export declare const wrappedGastownRouter: import('@trpc/server').TRPCBuiltRoute type: string; message: string; }[]; + billing: { + enabled: boolean; + state: + | 'blocked' + | 'degraded' + | 'idle' + | 'running' + | 'starting' + | 'stopping' + | 'warning'; + payer?: { + type: 'org' | 'user'; + id: string; + } | undefined; + remaining?: number | undefined; + minimumRequired?: number | undefined; + estimatedHourlyCharge?: number | undefined; + intervalStartedAt?: number | undefined; + lastReportedAt?: number | undefined; + }; + }; + meta: object; + }>; + getBillingStatus: import('@trpc/server').TRPCQueryProcedure<{ + input: { + townId: string; + }; + output: { + enabled: boolean; + state: + | 'blocked' + | 'degraded' + | 'idle' + | 'running' + | 'starting' + | 'stopping' + | 'warning'; + payer?: { + type: 'org' | 'user'; + id: string; + } | undefined; + remaining?: number | undefined; + minimumRequired?: number | undefined; + estimatedHourlyCharge?: number | undefined; + intervalStartedAt?: number | undefined; + lastReportedAt?: number | undefined; }; meta: object; }>; diff --git a/services/gastown/src/billing/ContainerBilling.error.ts b/services/gastown/src/billing/ContainerBilling.error.ts new file mode 100644 index 00000000000..3ce6e004c29 --- /dev/null +++ b/services/gastown/src/billing/ContainerBilling.error.ts @@ -0,0 +1,10 @@ +export class ContainerBillingError extends Error { + constructor( + readonly code: 'INSUFFICIENT_CREDITS' | 'BILLING_UNAVAILABLE', + message: string, + readonly details?: { remaining?: number; minimumRequired?: number } + ) { + super(message); + this.name = 'ContainerBillingError'; + } +} diff --git a/services/gastown/src/billing/container-usage-state.billing.test.ts b/services/gastown/src/billing/container-usage-state.billing.test.ts new file mode 100644 index 00000000000..88414e6f077 --- /dev/null +++ b/services/gastown/src/billing/container-usage-state.billing.test.ts @@ -0,0 +1,73 @@ +import { describe, expect, it } from 'vitest'; +import { toBillingStatus, type StoredUsageState } from './container-usage-state.billing'; +import type { UsageContext } from './container-usage.billing'; + +const context: UsageContext = { + service: 'gastown', + instanceId: 'container-1', + sku: 'cloudflare-container-standard-4', + subject: { type: 'org', id: 'org-1' }, + actor: { type: 'user', id: 'user-1' }, + sessionId: 'town-1', + metadata: { townId: 'town-1' }, +}; + +describe('toBillingStatus', () => { + it('hides persisted state while billing is disabled', () => { + expect( + toBillingStatus(false, { + phase: 'idle', + context, + blocked: true, + latestBudget: { verdict: 'stop', remaining: 0 }, + }) + ).toEqual({ enabled: false, state: 'idle' }); + }); + + it('returns a payer-aware blocked state', () => { + expect( + toBillingStatus(true, { + phase: 'idle', + context, + blocked: true, + latestBudget: { verdict: 'stop', remaining: 0.25 }, + }) + ).toEqual({ + enabled: true, + state: 'blocked', + payer: { type: 'org', id: 'org-1' }, + remaining: 0.25, + }); + }); + + it.each([ + ['continue', 'running'], + ['warn', 'warning'], + ['stop', 'stopping'], + ] as const)('maps %s budget verdicts to %s', (verdict, expectedState) => { + const state: StoredUsageState = { + phase: 'running', + context, + authorizationId: 'auth-1', + authorizationKey: 'authorize-1', + startEpochMs: 1_000, + startRecorded: true, + seq: 2, + lastReportedAt: 2_000, + latestBudget: { verdict, remaining: 4 }, + minimumRequired: 1, + estimatedHourlyCharge: 1.2, + }; + + expect(toBillingStatus(true, state)).toMatchObject({ + enabled: true, + state: expectedState, + payer: context.subject, + remaining: 4, + minimumRequired: 1, + estimatedHourlyCharge: 1.2, + intervalStartedAt: 1_000, + lastReportedAt: 2_000, + }); + }); +}); diff --git a/services/gastown/src/billing/container-usage-state.billing.ts b/services/gastown/src/billing/container-usage-state.billing.ts new file mode 100644 index 00000000000..b46161dd8d8 --- /dev/null +++ b/services/gastown/src/billing/container-usage-state.billing.ts @@ -0,0 +1,73 @@ +import type { BudgetVerdict, GastownBillingStatus, UsageContext } from './container-usage.billing'; + +export type PendingHeartbeat = { + seq: number; + observedAt: number; + usageSinceLast: number; + idempotencyKey: string; +}; + +export type OpenUsageInterval = { + phase: 'starting' | 'running' | 'stopping'; + context: UsageContext; + authorizationId: string; + authorizationKey: string; + authorizationExpiresAt?: number; + startEpochMs: number; + startRecorded: boolean; + seq: number; + lastReportedAt: number; + pendingHeartbeat?: PendingHeartbeat; + latestBudget?: BudgetVerdict; + minimumRequired?: number; + estimatedHourlyCharge?: number; + stopReason?: 'exit' | 'runtime_signal' | 'activity_expired'; + stopObservedAt?: number; + finalUsageCaptured?: boolean; +}; + +export type StoredUsageState = + | { phase: 'idle'; context?: UsageContext; blocked?: boolean; latestBudget?: BudgetVerdict } + | OpenUsageInterval; + +export function toBillingStatus(enabled: boolean, state: StoredUsageState): GastownBillingStatus { + if (!enabled) return { enabled: false, state: 'idle' }; + + const payer = state.context?.subject; + if (state.phase === 'idle') { + return { + enabled: true, + state: state.blocked ? 'blocked' : 'idle', + ...(payer ? { payer } : {}), + ...(state.latestBudget?.remaining === undefined + ? {} + : { remaining: state.latestBudget.remaining }), + }; + } + + const publicState = + state.phase === 'starting' + ? 'starting' + : state.phase === 'stopping' + ? 'stopping' + : state.latestBudget?.verdict === 'stop' + ? 'stopping' + : state.latestBudget?.verdict === 'warn' + ? 'warning' + : 'running'; + + return { + enabled: true, + state: publicState, + payer, + ...(state.latestBudget?.remaining === undefined + ? {} + : { remaining: state.latestBudget.remaining }), + ...(state.minimumRequired === undefined ? {} : { minimumRequired: state.minimumRequired }), + ...(state.estimatedHourlyCharge === undefined + ? {} + : { estimatedHourlyCharge: state.estimatedHourlyCharge }), + intervalStartedAt: state.startEpochMs, + lastReportedAt: state.lastReportedAt, + }; +} diff --git a/services/gastown/src/billing/container-usage.billing.test.ts b/services/gastown/src/billing/container-usage.billing.test.ts new file mode 100644 index 00000000000..64ad70656e4 --- /dev/null +++ b/services/gastown/src/billing/container-usage.billing.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, it } from 'vitest'; +import { getContainerUsageService, isGastownBillingEnabled } from './container-usage.billing'; +import type { ContainerUsageService } from './container-usage.billing'; + +function envFixture(overrides: Partial): Env { + return overrides as Env; +} + +describe('Gastown billing configuration', () => { + it('enables billing only for the exact true value', () => { + expect(isGastownBillingEnabled(envFixture({ GASTOWN_BILLING_ENABLED: 'true' }))).toBe(true); + expect(isGastownBillingEnabled(envFixture({ GASTOWN_BILLING_ENABLED: 'false' }))).toBe(false); + expect(isGastownBillingEnabled(envFixture({}))).toBe(false); + }); + + it('uses the no-charge development stub when the binding is absent', async () => { + const service = getContainerUsageService(envFixture({ ENVIRONMENT: 'development' })); + const authorization = await service.authorizeStart({ + context: { + service: 'gastown', + instanceId: 'container-1', + sku: 'cloudflare-container-standard-4', + subject: { type: 'user', id: 'user-1' }, + actor: { type: 'user', id: 'user-1' }, + sessionId: 'town-1', + metadata: { townId: 'town-1' }, + }, + idempotencyKey: 'authorize-1', + observedAt: 1_000, + }); + + expect(authorization).toMatchObject({ verdict: 'allow', minimumRequired: 1 }); + }); + + it('prefers the configured WorkerEntrypoint binding', () => { + const binding = { + authorizeStart: async () => ({ + verdict: 'deny' as const, + remaining: 0, + minimumRequired: 1, + }), + recordStart: async () => ({ intervalId: 'test', durable: 'pg' as const, dedup: false }), + recordHeartbeat: async () => ({ + intervalId: 'test', + durable: 'pg' as const, + dedup: false, + budget: { verdict: 'continue' as const }, + }), + recordStop: async () => ({ intervalId: 'test', durable: 'pg' as const, dedup: false }), + } satisfies ContainerUsageService; + + expect(getContainerUsageService(envFixture({ CONTAINER_USAGE: binding }))).toBe(binding); + }); + + it('fails closed in production when the binding is absent', () => { + expect(() => getContainerUsageService(envFixture({ ENVIRONMENT: 'production' }))).toThrow( + 'CONTAINER_USAGE binding is required' + ); + }); +}); diff --git a/services/gastown/src/billing/container-usage.billing.ts b/services/gastown/src/billing/container-usage.billing.ts new file mode 100644 index 00000000000..ee8423fc9f4 --- /dev/null +++ b/services/gastown/src/billing/container-usage.billing.ts @@ -0,0 +1,144 @@ +export type UsageSubject = { type: 'user' | 'org'; id: string }; +export type UsageActor = { type: 'user' | 'bot'; id: string }; + +export type UsageContext = { + service: 'gastown'; + instanceId: string; + sku: string; + subject: UsageSubject; + actor: UsageActor; + onBehalfOf?: UsageSubject; + sessionId: string; + region?: string; + metadata: Record; +}; + +export type RecordAck = { + intervalId: string; + durable: 'pg' | 'buffer'; + dedup: boolean; +}; + +export type BudgetVerdict = { + verdict: 'continue' | 'warn' | 'stop'; + remaining?: number; +}; + +export type HeartbeatAck = RecordAck & { + budget: BudgetVerdict; +}; + +export type AuthorizeStartResult = + | { + verdict: 'allow'; + authorizationId: string; + expiresAt: number; + remaining?: number; + minimumRequired: number; + estimatedHourlyCharge?: number; + } + | { + verdict: 'deny'; + remaining: number; + minimumRequired: number; + estimatedHourlyCharge?: number; + }; + +export type ContainerUsageService = { + authorizeStart(input: { + context: UsageContext; + idempotencyKey: string; + observedAt: number; + }): Promise; + recordStart( + input: UsageContext & { + idempotencyKey: string; + startEpochMs: number; + observedAt: number; + } + ): Promise; + recordHeartbeat(input: { + service: 'gastown'; + instanceId: string; + startEpochMs: number; + idempotencyKey: string; + seq: number; + observedAt: number; + usageSinceLast?: number; + context?: UsageContext; + }): Promise; + recordStop(input: { + service: 'gastown'; + instanceId: string; + startEpochMs: number; + idempotencyKey: string; + observedAt: number; + reason: 'exit' | 'runtime_signal' | 'activity_expired'; + exitCode?: number; + }): Promise; +}; + +export type GastownBillingStatus = { + enabled: boolean; + state: 'idle' | 'starting' | 'running' | 'warning' | 'stopping' | 'blocked' | 'degraded'; + payer?: { type: 'user' | 'org'; id: string }; + remaining?: number; + minimumRequired?: number; + estimatedHourlyCharge?: number; + intervalStartedAt?: number; + lastReportedAt?: number; +}; + +export const GASTOWN_CONTAINER_SKU = 'cloudflare-container-standard-4'; +export const USAGE_HEARTBEAT_INTERVAL_MS = 5 * 60_000; + +export function isGastownBillingEnabled(env: Env): boolean { + return env.GASTOWN_BILLING_ENABLED === 'true'; +} + +/** + * No-charge implementation for local development while the real WorkerEntrypoint + * package is under construction. Production never falls back to this service. + */ +function createDevelopmentContainerUsageStub(): ContainerUsageService { + return { + async authorizeStart({ idempotencyKey, observedAt }) { + return { + verdict: 'allow', + authorizationId: `dev:${idempotencyKey}`, + expiresAt: observedAt + 60_000, + remaining: 100, + minimumRequired: 1, + estimatedHourlyCharge: 1.2, + }; + }, + async recordStart(input) { + return { + intervalId: `${input.instanceId}:${input.startEpochMs}`, + durable: 'buffer', + dedup: false, + }; + }, + async recordHeartbeat(input) { + return { + intervalId: `${input.instanceId}:${input.startEpochMs}`, + durable: 'buffer', + dedup: false, + budget: { verdict: 'continue', remaining: 100 }, + }; + }, + async recordStop(input) { + return { + intervalId: `${input.instanceId}:${input.startEpochMs}`, + durable: 'buffer', + dedup: false, + }; + }, + }; +} + +export function getContainerUsageService(env: Env): ContainerUsageService { + if (env.CONTAINER_USAGE) return env.CONTAINER_USAGE; + if (env.ENVIRONMENT === 'development') return createDevelopmentContainerUsageStub(); + throw new Error('CONTAINER_USAGE binding is required when Gastown billing is enabled'); +} diff --git a/services/gastown/src/dos/Town.do.ts b/services/gastown/src/dos/Town.do.ts index b96d5da0fca..1a881fd801e 100644 --- a/services/gastown/src/dos/Town.do.ts +++ b/services/gastown/src/dos/Town.do.ts @@ -73,6 +73,13 @@ import { generateKiloApiToken } from '../util/kilo-token.util'; import { resolveSecret } from '../util/secret.util'; import { writeEvent, type GastownEventData } from '../util/analytics.util'; import { logger, withLogTags } from '../util/log.util'; +import { + GASTOWN_CONTAINER_SKU, + isGastownBillingEnabled, + type GastownBillingStatus, + type UsageActor, + type UsageContext, +} from '../billing/container-usage.billing'; import { BeadPriority } from '../types'; import type { TownConfig, @@ -275,6 +282,14 @@ export class TownDO extends DurableObject { getRigConfig: (rigId: string) => this.getRigConfig(rigId), resolveKilocodeToken: () => this.resolveKilocodeToken(), emitEvent: data => this.emitEvent(data), + prepareContainerBilling: async () => { + await this.prepareContainerBilling(); + if (isGastownBillingEnabled(this.env)) { + await getTownContainerStub(this.env, this.townId).warmUp(); + this._billingBlocked = false; + await this.ctx.storage.put('billing:blocked', false); + } + }, }; } @@ -764,6 +779,7 @@ export class TownDO extends DurableObject { this._draining = (await this.ctx.storage.get('town:draining')) ?? false; this._drainNonce = (await this.ctx.storage.get('town:drainNonce')) ?? null; this._drainStartedAt = (await this.ctx.storage.get('town:drainStartedAt')) ?? null; + this._billingBlocked = (await this.ctx.storage.get('billing:blocked')) ?? false; // All tables are now initialized via beads.initBeadTables(): // beads, bead_events, bead_dependencies, agent_metadata, review_metadata, @@ -820,6 +836,7 @@ export class TownDO extends DurableObject { private _draining = false; private _drainNonce: string | null = null; private _drainStartedAt: number | null = null; + private _billingBlocked = false; /** Instance UUID of the current container, set by the first heartbeat. */ private _containerInstanceId: string | null = null; @@ -930,9 +947,69 @@ export class TownDO extends DurableObject { async updateTownConfig(update: TownConfigUpdate): Promise { const result = await config.updateTownConfig(this.ctx.storage, update); this._ownerUserId = result.owner_user_id; + await this.prepareContainerBilling(undefined, result); return result; } + async getBillingStatus(): Promise { + if (!isGastownBillingEnabled(this.env)) return { enabled: false, state: 'idle' }; + await this.prepareContainerBilling(); + return getTownContainerStub(this.env, this.townId).getBillingStatus(); + } + + private async prepareContainerBilling( + actor?: UsageActor, + providedConfig?: TownConfig + ): Promise { + if (!isGastownBillingEnabled(this.env)) return; + + const townConfig = providedConfig ?? (await this.getTownConfig()); + const ownerId = townConfig.owner_id ?? townConfig.organization_id ?? townConfig.owner_user_id; + if (!ownerId) throw new Error('Town has no billing owner'); + + const subject = + townConfig.owner_type === 'org' + ? ({ type: 'org', id: ownerId } as const) + : ({ type: 'user', id: ownerId } as const); + const context: UsageContext = { + service: 'gastown', + instanceId: getTownContainerDoId(this.env, this.townId), + sku: GASTOWN_CONTAINER_SKU, + subject, + actor: actor ?? { type: 'bot', id: `gastown:${this.townId}` }, + sessionId: this.townId, + metadata: { townId: this.townId }, + ...(townConfig.created_by_user_id + ? { onBehalfOf: { type: 'user' as const, id: townConfig.created_by_user_id } } + : {}), + }; + await getTownContainerStub(this.env, this.townId).setBillingContext(context); + } + + private async updateContainerBilling(): Promise { + if (!isGastownBillingEnabled(this.env)) return { enabled: false, state: 'idle' }; + + await this.prepareContainerBilling(); + const container = getTownContainerStub(this.env, this.townId); + const status = await container.recordUsageHeartbeat(); + const blocked = status.state === 'blocked' || status.state === 'stopping'; + + if (blocked !== this._billingBlocked) { + this._billingBlocked = blocked; + await this.ctx.storage.put('billing:blocked', blocked); + } + + if (status.state === 'stopping') { + await container.stopForBilling().catch(error => { + logger.warn('billing: graceful container stop failed', { + error: error instanceof Error ? error.message : String(error), + }); + }); + } + + return status; + } + /** * Force-refresh the container token, bypassing the 1-hour throttle. * Called from the user-facing tRPC mutation so operators can manually @@ -2743,25 +2820,33 @@ export class TownDO extends DurableObject { async sendMayorMessage( message: string, _model?: string, - uiContext?: string + uiContext?: string, + actorUserId?: string ): Promise<{ agentId: string; sessionStatus: 'idle' | 'active' | 'starting'; }> { return withLogTags({ source: 'Town.do', tags: { townId: this.townId } }, () => - this._sendMayorMessage(message, _model, uiContext) + this._sendMayorMessage(message, _model, uiContext, actorUserId) ); } private async _sendMayorMessage( message: string, _model?: string, - uiContext?: string + uiContext?: string, + actorUserId?: string ): Promise<{ agentId: string; sessionStatus: 'idle' | 'active' | 'starting'; }> { const townId = this.townId; + await this.prepareContainerBilling(actorUserId ? { type: 'user', id: actorUserId } : undefined); + if (isGastownBillingEnabled(this.env)) { + await getTownContainerStub(this.env, townId).warmUp(); + this._billingBlocked = false; + await this.ctx.storage.put('billing:blocked', false); + } let mayor = agents.listAgents(this.sql, { role: 'mayor' })[0] ?? null; if (!mayor) { @@ -2956,20 +3041,26 @@ export class TownDO extends DurableObject { }; } - async ensureMayor(): Promise<{ + async ensureMayor(actorUserId?: string): Promise<{ agentId: string; sessionStatus: 'idle' | 'active' | 'starting'; }> { return withLogTags({ source: 'Town.do', tags: { townId: this.townId } }, () => - this._ensureMayor() + this._ensureMayor(actorUserId) ); } - private async _ensureMayor(): Promise<{ + private async _ensureMayor(actorUserId?: string): Promise<{ agentId: string; sessionStatus: 'idle' | 'active' | 'starting'; }> { const townId = this.townId; + await this.prepareContainerBilling(actorUserId ? { type: 'user', id: actorUserId } : undefined); + if (isGastownBillingEnabled(this.env)) { + await getTownContainerStub(this.env, townId).warmUp(); + this._billingBlocked = false; + await this.ctx.storage.put('billing:blocked', false); + } let mayor = agents.listAgents(this.sql, { role: 'mayor' })[0] ?? null; if (!mayor) { @@ -4144,8 +4235,20 @@ export class TownDO extends DurableObject { // Call once per tick — threaded to ensureContainerReady, maybeDispatchTriageAgent, and getAlarmStatus const rigList = rigs.listRigs(this.sql); const hasRigs = rigList.length > 0; + let billingStatus: GastownBillingStatus = { enabled: false, state: 'idle' }; + + if (isGastownBillingEnabled(this.env)) { + try { + billingStatus = await this.updateContainerBilling(); + } catch (err) { + billingStatus = { enabled: true, state: 'degraded' }; + logger.warn('alarm: container billing update failed', { + error: err instanceof Error ? err.message : String(err), + }); + } + } - if (hasRigs) { + if (hasRigs && !this._billingBlocked) { try { await this.ensureContainerReady(rigList); } catch (err) { @@ -4198,7 +4301,7 @@ export class TownDO extends DurableObject { ), ]); - if (workingAgentRows.length > 0) { + if (workingAgentRows.length > 0 && !this._billingBlocked) { const statusChecks = workingAgentRows.map(async row => { try { const containerInfo = await dispatch.checkAgentContainerStatus( @@ -4313,7 +4416,7 @@ export class TownDO extends DurableObject { const sideEffects: Array<() => Promise> = []; try { const actions = reconciler.reconcile(this.sql, { - draining: this._draining, + draining: this._draining || this._billingBlocked, townConfig, }); metrics.actionsEmitted = actions.length; @@ -4453,11 +4556,15 @@ export class TownDO extends DurableObject { error: err instanceof Error ? err.message : String(err), }) ), - this.maybeDispatchTriageAgent(cachedTriageCount, rigList).catch(err => - logger.warn('alarm: maybeDispatchTriageAgent failed', { - error: err instanceof Error ? err.message : String(err), - }) - ), + ...(this._billingBlocked + ? [] + : [ + this.maybeDispatchTriageAgent(cachedTriageCount, rigList).catch(err => + logger.warn('alarm: maybeDispatchTriageAgent failed', { + error: err instanceof Error ? err.message : String(err), + }) + ), + ]), // Prune processed reconciler events older than 7 days Promise.resolve().then(() => { try { @@ -4487,6 +4594,7 @@ export class TownDO extends DurableObject { const snapshot = await this.getAlarmStatus({ activeWork, triageCount: cachedTriageCount, + billingStatus, }); this.broadcastAlarmStatus(snapshot); } catch (err) { @@ -4546,12 +4654,18 @@ export class TownDO extends DurableObject { private async stopContainerIfIdle(): Promise { await _stopContainerIfIdle({ hasActiveWork: () => this.hasActiveWork(), - isDraining: () => this._draining, + isDraining: () => this._draining || this._billingBlocked, getMayor: () => agents.listAgents(this.sql, { role: 'mayor' })[0] ?? null, getTownId: () => this.townId, getLastIdleStopAt: () => this.ctx.storage.get('container:lastIdleStopAt'), setLastIdleStopAt: value => this.ctx.storage.put('container:lastIdleStopAt', value), - getContainerStub: townId => getTownContainerStub(this.env, townId), + getContainerStub: townId => { + const container = getTownContainerStub(this.env, townId); + return { + getState: () => container.getState(), + stop: () => container.stopForInactivity(), + }; + }, writeEventFn: data => writeEvent(this.env, data), now: () => Date.now(), }); @@ -4565,7 +4679,7 @@ export class TownDO extends DurableObject { townId, containerDoId: getTownContainerDoId(this.env, townId), outcome, - isDraining: () => this._draining, + isDraining: () => this._draining || this._billingBlocked, getConsecutiveHealthFailures: () => this.ctx.storage.get(CONTAINER_HEALTH_STORAGE_KEYS.consecutiveHealthFailures), setConsecutiveHealthFailures: value => @@ -4971,6 +5085,7 @@ export class TownDO extends DurableObject { cachedRigList?: rigs.RigRecord[], cachedActiveWork?: boolean ): Promise { + if (this._billingBlocked) return; const rigList = cachedRigList ?? rigs.listRigs(this.sql); if (rigList.length === 0) return; @@ -5263,7 +5378,11 @@ export class TownDO extends DurableObject { * Return a structured snapshot of the alarm loop and patrol state * for the dashboard Status tab. */ - async getAlarmStatus(cached?: { activeWork?: boolean; triageCount?: number }): Promise<{ + async getAlarmStatus(cached?: { + activeWork?: boolean; + triageCount?: number; + billingStatus?: GastownBillingStatus; + }): Promise<{ alarm: { nextFireAt: string | null; intervalMs: number; @@ -5298,6 +5417,7 @@ export class TownDO extends DurableObject { }>; draining?: boolean; drainStartedAt?: string; + billing: GastownBillingStatus; }> { const currentAlarm = await this.ctx.storage.getAlarm(); const active = cached?.activeWork ?? this.hasActiveWork(); @@ -5420,6 +5540,12 @@ export class TownDO extends DurableObject { message: formatEventMessage(row), })); + const billing = + cached?.billingStatus ?? + (isGastownBillingEnabled(this.env) + ? await getTownContainerStub(this.env, this.townId).getBillingStatus() + : { enabled: false, state: 'idle' as const }); + return { alarm: { nextFireAt: currentAlarm ? new Date(Number(currentAlarm)).toISOString() : null, @@ -5440,6 +5566,7 @@ export class TownDO extends DurableObject { drainStartedAt: this._drainStartedAt ? new Date(this._drainStartedAt).toISOString() : undefined, + billing, }; } diff --git a/services/gastown/src/dos/TownContainer.do.ts b/services/gastown/src/dos/TownContainer.do.ts index 2a29ea2504b..bb333302491 100644 --- a/services/gastown/src/dos/TownContainer.do.ts +++ b/services/gastown/src/dos/TownContainer.do.ts @@ -1,6 +1,21 @@ import { Container } from '@cloudflare/containers'; +import { ContainerBillingError } from '../billing/ContainerBilling.error'; +import { + getContainerUsageService, + isGastownBillingEnabled, + USAGE_HEARTBEAT_INTERVAL_MS, + type GastownBillingStatus, + type UsageContext, +} from '../billing/container-usage.billing'; +import { + toBillingStatus, + type OpenUsageInterval, + type StoredUsageState, +} from '../billing/container-usage-state.billing'; const TC_LOG = '[TownContainer.do]'; +const BILLING_CONTEXT_KEY = 'billing:context'; +const BILLING_STATE_KEY = 'billing:state'; /** * TownContainer — a Cloudflare Container per town. @@ -79,8 +94,27 @@ export class TownContainerDO extends Container { return registry ?? []; } - override onStart(): void { + override async onStart(): Promise { console.log(`${TC_LOG} container started for DO id=${this.ctx.id.toString()}`); + if (!isGastownBillingEnabled(this.env)) return; + + try { + const state = await this.ensureBillingInterval(); + if (state.phase === 'stopping') { + await this.stop(); + } else if (state.phase === 'starting' && state.startRecorded) { + await this.ctx.storage.put(BILLING_STATE_KEY, { + ...state, + phase: 'running', + lastReportedAt: Date.now(), + }); + } + } catch (error) { + console.error(`${TC_LOG} failed to open billing interval after container start`, error); + await this.stop().catch(stopError => + console.error(`${TC_LOG} failed to stop unmetered container`, stopError) + ); + } } /** @@ -92,6 +126,9 @@ export class TownContainerDO extends Container { * triggered a cold start. */ async warmUp(): Promise<{ coldStart: boolean; durationMs: number }> { + if (isGastownBillingEnabled(this.env)) { + await this.ensureBillingInterval(); + } if (this.ctx.container?.running === true) { // Runtime-level fast path only. TownDO's /health probe is the // application-level liveness source and may still recover a wedged @@ -99,24 +136,509 @@ export class TownContainerDO extends Container { return { coldStart: false, durationMs: 0 }; } const t0 = Date.now(); - await this.startAndWaitForPorts(); + try { + await this.startAndWaitForPorts(); + } catch (error) { + await this.closeBillingInterval('runtime_signal'); + throw error; + } return { coldStart: true, durationMs: Date.now() - t0 }; } - override onStop({ exitCode, reason }: { exitCode: number; reason: string }): void { + override async onStop({ exitCode, reason }: { exitCode: number; reason: string }): Promise { console.log( `${TC_LOG} container stopped: exitCode=${exitCode} reason=${reason} id=${this.ctx.id.toString()}` ); + if (!isGastownBillingEnabled(this.env)) return; + + const current = await this.getUsageState(); + const stopReason = + current.phase !== 'idle' && current.stopReason + ? current.stopReason + : reason.toLowerCase().includes('activity') + ? 'activity_expired' + : exitCode === 0 + ? 'exit' + : 'runtime_signal'; + await this.closeBillingInterval(stopReason); + } + + override async onActivityExpired(): Promise { + if (isGastownBillingEnabled(this.env)) { + const state = await this.getUsageState(); + if (state.phase !== 'idle' && state.phase !== 'stopping') { + await this.ctx.storage.put(BILLING_STATE_KEY, { + ...state, + phase: 'stopping', + stopReason: 'activity_expired', + stopObservedAt: Date.now(), + } satisfies OpenUsageInterval); + } + } + await super.onActivityExpired(); } override onError(error: unknown): void { console.error(`${TC_LOG} container error:`, error, `id=${this.ctx.id.toString()}`); } - // No fetch() override — the base Container class handles everything: - // - HTTP requests are proxied to port 8080 via containerFetch - // - WebSocket upgrades are proxied to port 8080 via containerFetch - // (the container's Bun.serve handles the WS upgrade natively) + override async fetch(request: Request): Promise { + if (isGastownBillingEnabled(this.env)) { + await this.ensureBillingInterval(); + } + return super.fetch(request); + } + + async setBillingContext(context: UsageContext): Promise { + await this.ctx.storage.put(BILLING_CONTEXT_KEY, context); + const state = await this.getUsageState(); + if (state.phase === 'idle') { + await this.ctx.storage.put(BILLING_STATE_KEY, { ...state, context }); + } + } + + async getBillingStatus(): Promise { + return toBillingStatus(isGastownBillingEnabled(this.env), await this.getUsageState()); + } + + async recordUsageHeartbeat(): Promise { + if (!isGastownBillingEnabled(this.env)) return { enabled: false, state: 'idle' }; + + let state = await this.getUsageState(); + if (state.phase === 'starting') { + state = await this.completeBillingStart(state); + } + const initialRuntimeState = state.phase === 'idle' ? await this.getState() : null; + if ( + state.phase === 'idle' && + (initialRuntimeState?.status === 'running' || initialRuntimeState?.status === 'healthy') + ) { + try { + state = await this.ensureBillingInterval(); + if (state.phase === 'starting' && state.startRecorded) { + state = { ...state, phase: 'running' }; + await this.ctx.storage.put(BILLING_STATE_KEY, state); + } + } catch (error) { + if (error instanceof ContainerBillingError && error.code === 'INSUFFICIENT_CREDITS') { + await this.stopForBilling(); + return this.getBillingStatus(); + } + throw error; + } + } + if (state.phase === 'idle') return toBillingStatus(true, state); + if (state.phase === 'stopping') { + await this.closeBillingInterval(state.stopReason ?? 'runtime_signal'); + return this.getBillingStatus(); + } + + const runtimeState = await this.getState(); + if (runtimeState.status !== 'running' && runtimeState.status !== 'healthy') { + await this.closeBillingInterval('runtime_signal', state.lastReportedAt); + return this.getBillingStatus(); + } + + const observedAt = Date.now(); + if (observedAt - state.lastReportedAt < USAGE_HEARTBEAT_INTERVAL_MS) { + return toBillingStatus(true, state); + } + + try { + const updated = await this.captureHeartbeat(state, observedAt); + return toBillingStatus(true, updated); + } catch (error) { + console.error(`${TC_LOG} billing heartbeat failed`, error); + const status = toBillingStatus(true, state); + return { ...status, state: 'degraded' }; + } + } + + async stopForBilling(): Promise { + if (!isGastownBillingEnabled(this.env)) return; + const state = await this.getUsageState(); + if (state.phase !== 'idle') { + if (state.phase !== 'stopping') { + await this.ctx.storage.put(BILLING_STATE_KEY, { + ...state, + phase: 'stopping', + stopReason: 'runtime_signal', + stopObservedAt: Date.now(), + } satisfies OpenUsageInterval); + } + } + await this.stop(); + } + + async stopForInactivity(): Promise { + if (isGastownBillingEnabled(this.env)) { + const state = await this.getUsageState(); + if (state.phase !== 'idle' && state.phase !== 'stopping') { + await this.ctx.storage.put(BILLING_STATE_KEY, { + ...state, + phase: 'stopping', + stopReason: 'activity_expired', + stopObservedAt: Date.now(), + } satisfies OpenUsageInterval); + } + } + await this.stop(); + } + + private async ensureBillingInterval(): Promise { + if (!isGastownBillingEnabled(this.env)) return { phase: 'idle' }; + + const existing = await this.getUsageState(); + if (existing.phase === 'running') return existing; + if (existing.phase === 'stopping') { + throw new ContainerBillingError( + 'BILLING_UNAVAILABLE', + 'Container billing is still settling the previous runtime' + ); + } + if (existing.phase === 'starting') return this.completeBillingStart(existing); + + const context = + existing.context ?? (await this.ctx.storage.get(BILLING_CONTEXT_KEY)); + if (!context) { + throw new ContainerBillingError( + 'BILLING_UNAVAILABLE', + 'Container billing context has not been configured' + ); + } + + const selected = await this.ctx.storage.transaction(async transaction => { + const latest = await transaction.get(BILLING_STATE_KEY); + if (latest && latest.phase !== 'idle') return latest; + + const observedAt = Date.now(); + const starting: OpenUsageInterval = { + phase: 'starting', + context: latest?.context ?? context, + authorizationId: '', + authorizationKey: `${context.instanceId}:authorize:${observedAt}`, + startEpochMs: observedAt, + startRecorded: false, + seq: 0, + lastReportedAt: observedAt, + }; + await transaction.put(BILLING_STATE_KEY, starting); + return starting; + }); + + if (selected.phase === 'running') return selected; + if (selected.phase === 'stopping') { + throw new ContainerBillingError( + 'BILLING_UNAVAILABLE', + 'Container billing is still settling the previous runtime' + ); + } + return this.completeBillingStart(selected); + } + + private async completeBillingStart(starting: OpenUsageInterval): Promise { + const usage = getContainerUsageService(this.env); + let current = starting; + + if ( + current.authorizationId && + !current.startRecorded && + current.authorizationExpiresAt !== undefined && + current.authorizationExpiresAt <= Date.now() + ) { + const selected = await this.ctx.storage.transaction(async transaction => { + const latest = await transaction.get(BILLING_STATE_KEY); + if (!latest || latest.phase !== 'starting') return latest; + if ( + !latest.authorizationId || + latest.startRecorded || + latest.authorizationExpiresAt === undefined || + latest.authorizationExpiresAt > Date.now() + ) { + return latest; + } + + const observedAt = Date.now(); + const refreshed: OpenUsageInterval = { + ...latest, + context: { + ...latest.context, + metadata: Object.fromEntries( + Object.entries(latest.context.metadata).filter(([key]) => key !== 'authorizationId') + ), + }, + authorizationId: '', + authorizationKey: `${latest.context.instanceId}:authorize:${observedAt}`, + authorizationExpiresAt: undefined, + startEpochMs: observedAt, + lastReportedAt: observedAt, + }; + await transaction.put(BILLING_STATE_KEY, refreshed); + return refreshed; + }); + + if (!selected || selected.phase === 'idle' || selected.phase === 'stopping') { + throw new ContainerBillingError( + 'BILLING_UNAVAILABLE', + 'Container billing state changed while refreshing admission' + ); + } + current = selected; + } + + if (!current.authorizationId) { + let authorization; + try { + authorization = await usage.authorizeStart({ + context: current.context, + idempotencyKey: current.authorizationKey, + observedAt: current.startEpochMs, + }); + } catch (error) { + throw new ContainerBillingError( + 'BILLING_UNAVAILABLE', + error instanceof Error ? error.message : 'Container billing is unavailable' + ); + } + + if (authorization.verdict === 'deny') { + const denied = await this.ctx.storage.transaction(async transaction => { + const latest = await transaction.get(BILLING_STATE_KEY); + if ( + !latest || + latest.phase !== 'starting' || + latest.startEpochMs !== current.startEpochMs || + latest.authorizationKey !== current.authorizationKey + ) { + return false; + } + await transaction.put(BILLING_STATE_KEY, { + phase: 'idle', + context: latest.context, + blocked: true, + latestBudget: { verdict: 'stop', remaining: authorization.remaining }, + } satisfies StoredUsageState); + return true; + }); + if (!denied) { + throw new ContainerBillingError( + 'BILLING_UNAVAILABLE', + 'Container billing state changed during admission' + ); + } + throw new ContainerBillingError( + 'INSUFFICIENT_CREDITS', + `Gas Town needs at least $${authorization.minimumRequired.toFixed(2)} in credits to start`, + { + remaining: authorization.remaining, + minimumRequired: authorization.minimumRequired, + } + ); + } + + const authorized = await this.ctx.storage.transaction(async transaction => { + const latest = await transaction.get(BILLING_STATE_KEY); + if ( + !latest || + latest.phase !== 'starting' || + latest.startEpochMs !== current.startEpochMs || + latest.authorizationKey !== current.authorizationKey + ) { + return null; + } + const updated: OpenUsageInterval = { + ...latest, + context: { + ...latest.context, + metadata: { + ...latest.context.metadata, + authorizationId: authorization.authorizationId, + }, + }, + authorizationId: authorization.authorizationId, + authorizationExpiresAt: authorization.expiresAt, + latestBudget: { + verdict: 'continue', + ...(authorization.remaining === undefined + ? {} + : { remaining: authorization.remaining }), + }, + minimumRequired: authorization.minimumRequired, + ...(authorization.estimatedHourlyCharge === undefined + ? {} + : { estimatedHourlyCharge: authorization.estimatedHourlyCharge }), + }; + await transaction.put(BILLING_STATE_KEY, updated); + return updated; + }); + if (!authorized) { + throw new ContainerBillingError( + 'BILLING_UNAVAILABLE', + 'Container billing state changed during admission' + ); + } + current = authorized; + } + + if (current.startRecorded) return current; + + try { + await usage.recordStart({ + ...current.context, + idempotencyKey: `${current.context.instanceId}:start:${current.startEpochMs}`, + startEpochMs: current.startEpochMs, + observedAt: current.startEpochMs, + }); + } catch (error) { + throw new ContainerBillingError( + 'BILLING_UNAVAILABLE', + error instanceof Error ? error.message : 'Unable to record container start' + ); + } + + const recorded = await this.ctx.storage.transaction(async transaction => { + const latest = await transaction.get(BILLING_STATE_KEY); + if ( + !latest || + latest.phase !== 'starting' || + latest.startEpochMs !== current.startEpochMs || + latest.authorizationId !== current.authorizationId + ) { + return null; + } + const updated: OpenUsageInterval = { ...latest, startRecorded: true }; + await transaction.put(BILLING_STATE_KEY, updated); + return updated; + }); + if (!recorded) { + throw new ContainerBillingError( + 'BILLING_UNAVAILABLE', + 'Container billing state changed while recording start' + ); + } + return recorded; + } + + private async captureHeartbeat( + state: OpenUsageInterval, + observedAt: number + ): Promise { + const selected = await this.ctx.storage.transaction(async transaction => { + const latest = await transaction.get(BILLING_STATE_KEY); + if (!latest || latest.phase === 'idle' || latest.startEpochMs !== state.startEpochMs) { + return null; + } + if (latest.pendingHeartbeat) { + return { interval: latest, pending: latest.pendingHeartbeat }; + } + + const pending = { + seq: latest.seq + 1, + observedAt, + usageSinceLast: Math.max(0, (observedAt - latest.lastReportedAt) / 1000), + idempotencyKey: `${latest.context.instanceId}:hb:${latest.seq + 1}`, + } satisfies NonNullable; + const interval = { ...latest, pendingHeartbeat: pending } satisfies OpenUsageInterval; + await transaction.put(BILLING_STATE_KEY, interval); + return { interval, pending }; + }); + + if (!selected) { + return this.getUsageState(); + } + const { interval, pending } = selected; + + const ack = await getContainerUsageService(this.env).recordHeartbeat({ + service: 'gastown', + instanceId: interval.context.instanceId, + startEpochMs: interval.startEpochMs, + idempotencyKey: pending.idempotencyKey, + seq: pending.seq, + observedAt: pending.observedAt, + usageSinceLast: pending.usageSinceLast, + context: interval.context, + }); + + const latest = await this.getUsageState(); + if (latest.phase === 'idle' || latest.startEpochMs !== interval.startEpochMs) return latest; + + const updated: OpenUsageInterval = { + ...latest, + phase: latest.phase === 'starting' ? 'running' : latest.phase, + seq: pending.seq, + lastReportedAt: pending.observedAt, + latestBudget: ack.budget, + }; + delete updated.pendingHeartbeat; + await this.ctx.storage.put(BILLING_STATE_KEY, updated); + return updated; + } + + private async closeBillingInterval( + reason: 'exit' | 'runtime_signal' | 'activity_expired', + observedAtOverride?: number + ): Promise { + const selected = await this.ctx.storage.transaction(async transaction => { + const latest = await transaction.get(BILLING_STATE_KEY); + if (!latest || latest.phase === 'idle') return null; + if (latest.phase === 'stopping') return latest; + + const stopping: OpenUsageInterval = { + ...latest, + phase: 'stopping', + stopReason: latest.stopReason ?? reason, + stopObservedAt: latest.stopObservedAt ?? observedAtOverride ?? Date.now(), + }; + await transaction.put(BILLING_STATE_KEY, stopping); + return stopping; + }); + if (!selected) return; + + const stopReason = selected.stopReason ?? reason; + const stopObservedAt = selected.stopObservedAt ?? observedAtOverride ?? Date.now(); + let stopping = selected; + + try { + if (!stopping.finalUsageCaptured && stopObservedAt > stopping.lastReportedAt) { + const captured = await this.captureHeartbeat(stopping, stopObservedAt); + if (captured.phase === 'idle') return; + stopping = captured; + stopping = { + ...stopping, + phase: 'stopping', + stopReason, + stopObservedAt, + finalUsageCaptured: true, + }; + await this.ctx.storage.put(BILLING_STATE_KEY, stopping); + } + + await getContainerUsageService(this.env).recordStop({ + service: 'gastown', + instanceId: stopping.context.instanceId, + startEpochMs: stopping.startEpochMs, + idempotencyKey: `${stopping.context.instanceId}:stop:${stopping.startEpochMs}`, + observedAt: stopObservedAt, + reason: stopReason, + }); + + const latest = await this.getUsageState(); + if (latest.phase !== 'idle' && latest.startEpochMs === stopping.startEpochMs) { + await this.ctx.storage.put(BILLING_STATE_KEY, { + phase: 'idle', + context: stopping.context, + blocked: stopping.latestBudget?.verdict === 'stop', + latestBudget: stopping.latestBudget, + } satisfies StoredUsageState); + } + } catch (error) { + console.error(`${TC_LOG} failed to close billing interval; will retry`, error); + } + } + + private async getUsageState(): Promise { + return (await this.ctx.storage.get(BILLING_STATE_KEY)) ?? { phase: 'idle' }; + } } export function getTownContainerStub(env: Env, townId: string) { diff --git a/services/gastown/src/dos/town/scheduling.ts b/services/gastown/src/dos/town/scheduling.ts index 2f9e47ff604..999956890b6 100644 --- a/services/gastown/src/dos/town/scheduling.ts +++ b/services/gastown/src/dos/town/scheduling.ts @@ -36,6 +36,7 @@ type SchedulingContext = { getRigConfig: (rigId: string) => Promise; resolveKilocodeToken: () => Promise; emitEvent: (data: Omit) => void; + prepareContainerBilling: () => Promise; }; type RigConfig = { @@ -77,6 +78,7 @@ export async function dispatchAgent( const townConfig = await ctx.getTownConfig(); const kilocodeToken = await ctx.resolveKilocodeToken(); + await ctx.prepareContainerBilling(); const convoyId = beadOps.getConvoyForBead(ctx.sql, bead.bead_id); const convoyFeatureBranch = convoyId ? beadOps.getConvoyFeatureBranch(ctx.sql, convoyId) : null; diff --git a/services/gastown/src/trpc/router.ts b/services/gastown/src/trpc/router.ts index 3d0427762e4..8a87134ca41 100644 --- a/services/gastown/src/trpc/router.ts +++ b/services/gastown/src/trpc/router.ts @@ -27,6 +27,7 @@ import { RpcBeadEventOutput, RpcMayorSendResultOutput, RpcMayorStatusOutput, + RpcBillingStatusOutput, RpcStreamTicketOutput, RpcPtySessionOutput, RpcSlingResultOutput, @@ -37,6 +38,7 @@ import { RpcMergeQueueDataOutput, } from './schemas'; import type { TRPCContext } from './init'; +import { ContainerBillingError } from '../billing/ContainerBilling.error'; // rpcSafe wrapper for TownConfigSchema (imported from ../types, not ./schemas) const RpcTownConfigSchema = z.any().pipe(TownConfigSchema); @@ -83,6 +85,24 @@ function userFromCtx(ctx: TRPCContext): { id: string; api_token_pepper: string | return { id: ctx.userId, api_token_pepper: ctx.apiTokenPepper }; } +function mapContainerBillingError(error: unknown): TRPCError | null { + let code: unknown; + if (error instanceof ContainerBillingError) { + code = error.code; + } else if (typeof error === 'object' && error !== null && 'code' in error) { + code = error.code; + } + + const message = error instanceof Error ? error.message : 'Container billing is unavailable'; + if (code === 'INSUFFICIENT_CREDITS' || message.includes('in credits to start')) { + return new TRPCError({ code: 'PRECONDITION_FAILED', message, cause: error }); + } + if (code === 'BILLING_UNAVAILABLE' || message.includes('billing')) { + return new TRPCError({ code: 'SERVICE_UNAVAILABLE', message, cause: error }); + } + return null; +} + /** Look up a user's membership for a specific org from the JWT claims. */ function getOrgMembership( memberships: JwtOrgMembership[], @@ -974,7 +994,16 @@ export const gastownRouter = router({ await verifyTownOwnership(ctx.env, ctx, input.townId); const townStub = getTownDOStub(ctx.env, input.townId); - return townStub.sendMayorMessage(input.message, input.model, input.uiContext); + try { + return await townStub.sendMayorMessage( + input.message, + input.model, + input.uiContext, + ctx.userId + ); + } catch (error) { + throw mapContainerBillingError(error) ?? error; + } }), getMayorStatus: gastownProcedure @@ -995,6 +1024,14 @@ export const gastownRouter = router({ return townStub.getAlarmStatus(); }), + getBillingStatus: gastownProcedure + .input(z.object({ townId: z.string().uuid() })) + .output(RpcBillingStatusOutput) + .query(async ({ ctx, input }) => { + await verifyTownOwnership(ctx.env, ctx, input.townId); + return getTownDOStub(ctx.env, input.townId).getBillingStatus(); + }), + ensureMayor: gastownProcedure .input(z.object({ townId: z.string().uuid() })) .output(RpcMayorSendResultOutput) @@ -1023,7 +1060,11 @@ export const gastownRouter = router({ } const townStub = getTownDOStub(ctx.env, input.townId); - return townStub.ensureMayor(); + try { + return await townStub.ensureMayor(ctx.userId); + } catch (error) { + throw mapContainerBillingError(error) ?? error; + } }), // ── Agent Streams ─────────────────────────────────────────────────── diff --git a/services/gastown/src/trpc/schemas.ts b/services/gastown/src/trpc/schemas.ts index e204f3b1bfc..44009b08260 100644 --- a/services/gastown/src/trpc/schemas.ts +++ b/services/gastown/src/trpc/schemas.ts @@ -102,6 +102,17 @@ export const MayorStatusOutput = z.object({ .nullable(), }); +export const BillingStatusOutput = z.object({ + enabled: z.boolean(), + state: z.enum(['idle', 'starting', 'running', 'warning', 'stopping', 'blocked', 'degraded']), + payer: z.object({ type: z.enum(['user', 'org']), id: z.string() }).optional(), + remaining: z.number().optional(), + minimumRequired: z.number().optional(), + estimatedHourlyCharge: z.number().optional(), + intervalStartedAt: z.number().optional(), + lastReportedAt: z.number().optional(), +}); + // StreamTicket export const StreamTicketOutput = z.object({ url: z.string(), @@ -184,6 +195,7 @@ export const RpcAgentOutput = rpcSafe(AgentOutput); export const RpcBeadEventOutput = rpcSafe(BeadEventOutput); export const RpcMayorSendResultOutput = rpcSafe(MayorSendResultOutput); export const RpcMayorStatusOutput = rpcSafe(MayorStatusOutput); +export const RpcBillingStatusOutput = rpcSafe(BillingStatusOutput); export const RpcStreamTicketOutput = rpcSafe(StreamTicketOutput); export const RpcPtySessionOutput = rpcSafe(PtySessionOutput); export const RpcConvoyOutput = rpcSafe(ConvoyOutput); @@ -224,6 +236,7 @@ const AlarmStatusOutput = z.object({ message: z.string(), }) ), + billing: BillingStatusOutput, }); export const RpcAlarmStatusOutput = rpcSafe(AlarmStatusOutput); export const RpcRigDetailOutput = rpcSafe(RigDetailOutput); diff --git a/services/gastown/test/integration/billing-disabled.test.ts b/services/gastown/test/integration/billing-disabled.test.ts new file mode 100644 index 00000000000..de94619959b --- /dev/null +++ b/services/gastown/test/integration/billing-disabled.test.ts @@ -0,0 +1,15 @@ +import { env } from 'cloudflare:test'; +import { describe, expect, it } from 'vitest'; + +describe('Gastown billing feature flag', () => { + it('keeps billing disabled in both dedicated and alarm status', async () => { + const townId = `billing-disabled-${crypto.randomUUID()}`; + const town = env.TOWN.get(env.TOWN.idFromName(townId)); + await town.setTownId(townId); + + await expect(town.getBillingStatus()).resolves.toEqual({ enabled: false, state: 'idle' }); + await expect(town.getAlarmStatus()).resolves.toMatchObject({ + billing: { enabled: false, state: 'idle' }, + }); + }); +}); diff --git a/services/gastown/tsconfig.types.json b/services/gastown/tsconfig.types.json index 14416e8d15f..78383858644 100644 --- a/services/gastown/tsconfig.types.json +++ b/services/gastown/tsconfig.types.json @@ -4,6 +4,7 @@ "declaration": true, "emitDeclarationOnly": true, "declarationDir": "./dist-types", + "rootDir": "./src", "noEmit": false, "skipLibCheck": true }, diff --git a/services/gastown/worker-configuration.d.ts b/services/gastown/worker-configuration.d.ts index d7d8e795bcc..d4567090434 100644 --- a/services/gastown/worker-configuration.d.ts +++ b/services/gastown/worker-configuration.d.ts @@ -57,6 +57,7 @@ type WastelandService = { evidence: string; }): Promise>; }; +type ContainerUsageService = import("./src/billing/container-usage.billing").ContainerUsageService; declare namespace Cloudflare { interface GlobalProps { mainModule: typeof import("./src/gastown.worker"); @@ -80,6 +81,8 @@ declare namespace Cloudflare { AGENT: DurableObjectNamespace; GIT_TOKEN_SERVICE: GitTokenService; WASTELAND_SERVICE: WastelandService; + CONTAINER_USAGE?: ContainerUsageService; + GASTOWN_BILLING_ENABLED: "false" | "true"; AI: Ai; AGENT_DB_SNAPSHOTS_KV: KVNamespace; } @@ -101,6 +104,8 @@ declare namespace Cloudflare { AGENT: DurableObjectNamespace; GIT_TOKEN_SERVICE: GitTokenService; WASTELAND_SERVICE: WastelandService; + CONTAINER_USAGE?: ContainerUsageService; + GASTOWN_BILLING_ENABLED: "false" | "true"; AI: Ai; AGENT_DB_SNAPSHOTS_KV: KVNamespace; HYPERDRIVE?: Hyperdrive; diff --git a/services/gastown/wrangler.jsonc b/services/gastown/wrangler.jsonc index 1aec708d982..c8559239372 100644 --- a/services/gastown/wrangler.jsonc +++ b/services/gastown/wrangler.jsonc @@ -74,6 +74,7 @@ "KILO_API_URL": "https://api.kilo.ai", "GASTOWN_API_URL": "https://gastown.kiloapps.io", "WASTELAND_API_URL": "https://wasteland.kiloapps.io", + "GASTOWN_BILLING_ENABLED": "false", }, // Hyperdrive binding for Postgres (user lookups, token generation) @@ -134,6 +135,7 @@ "KILO_API_URL": "http://192.168.65.254:3000", "GASTOWN_API_URL": "http://192.168.65.254:8803", "WASTELAND_API_URL": "http://192.168.65.254:8788", + "GASTOWN_BILLING_ENABLED": "false", }, "containers": [ { diff --git a/services/gastown/wrangler.test.jsonc b/services/gastown/wrangler.test.jsonc index fa9ab677865..62c3e8e889f 100644 --- a/services/gastown/wrangler.test.jsonc +++ b/services/gastown/wrangler.test.jsonc @@ -41,5 +41,6 @@ "CF_ACCESS_TEAM": "engineering-e11", "CF_ACCESS_AUD": "f30e3fd893df52fa3ffc50fbdb5ee6a4f111625ae92234233429684e1429d809", "GASTOWN_API_URL": "http://host.docker.internal:9787", + "GASTOWN_BILLING_ENABLED": "false", }, } From bc97429e2aecf513fa32e85adeca2e6e54a9ed30 Mon Sep 17 00:00:00 2001 From: John Fawcett Date: Wed, 22 Jul 2026 11:58:01 -0500 Subject: [PATCH 03/10] feat(gastown): add container cost circuit breaker --- .specs/gastown-usage-based-billing.md | 18 ++ apps/web/src/components/gastown/MayorChat.tsx | 8 +- .../src/components/gastown/TerminalBar.tsx | 214 ++++++++++++++---- apps/web/src/lib/gastown/types/router.d.ts | 140 ++++++++++-- .../src/billing/ContainerBilling.error.ts | 2 +- .../container-usage-state.billing.test.ts | 37 ++- .../billing/container-usage-state.billing.ts | 74 ++++-- .../src/billing/container-usage.billing.ts | 15 +- services/gastown/src/dos/Town.do.ts | 68 ++++-- services/gastown/src/dos/TownContainer.do.ts | 93 +++++++- services/gastown/src/trpc/router.ts | 37 +++ services/gastown/src/trpc/schemas.ts | 14 +- .../test/integration/billing-disabled.test.ts | 24 +- 13 files changed, 634 insertions(+), 110 deletions(-) diff --git a/.specs/gastown-usage-based-billing.md b/.specs/gastown-usage-based-billing.md index 4eb3355cd9d..c0191647a75 100644 --- a/.specs/gastown-usage-based-billing.md +++ b/.specs/gastown-usage-based-billing.md @@ -357,6 +357,19 @@ server admission remains authoritative. | `blocked` | Replace reconnect loops with a paused state explaining the required balance. Primary action: **Add credits**; after funding, **Resume Gas Town**. | | `degraded` | Keep running. Avoid alarming the user for a transient buffered write; show service degradation only if action is required or the delay is prolonged. | +### User circuit breaker + +Every billing-enabled town MUST expose a durable `automatic | paused_by_user` container run policy. +Turning automatic starts off MUST persist `paused_by_user` before stopping the container, stop new +dispatch, gracefully save active work, leave queued work pending, and prevent every cold-start path +from restarting the container. The Town scheduler and Town Container boundary MUST both enforce the +policy. Credit top-ups, alarms, PTY reconnects, and health checks MUST NOT clear a user pause. + +The UI MUST confirm before pausing and show the estimated charge accumulated by the current runtime. +Turning automatic starts back on removes the circuit breaker but MUST NOT itself boot an idle +container; pending eligible work or an explicit user start may boot it afterward. A user pause is +distinct from an insufficient-credit block and uses different recovery copy and controls. + Recommended copy: - Before start: `Container usage is billed at an estimated {rate}/hour while Gas Town is running.` @@ -465,6 +478,11 @@ stops, and counts of `warn` and `stop` verdicts. ## Changelog +### 2026-07-22 -- User circuit breaker + +- Added a durable user-controlled automatic-start policy, forced graceful shutdown, and per-run cost + estimate. + ### 2026-07-21 -- Technical design expansion - Added admission, durable metering state, reporting, low-balance shutdown, API, UI, rollout, and diff --git a/apps/web/src/components/gastown/MayorChat.tsx b/apps/web/src/components/gastown/MayorChat.tsx index c1640f82ed0..65d35436ba8 100644 --- a/apps/web/src/components/gastown/MayorChat.tsx +++ b/apps/web/src/components/gastown/MayorChat.tsx @@ -49,7 +49,12 @@ export function MayorChat({ townId }: MayorChatProps) { // Reset on townId change so ensureMayor fires for each town const ensuredTownRef = useRef(null); useEffect(() => { - if (!billingQuery.isSuccess || billingQuery.data.enabled) return; + if ( + !billingQuery.isSuccess || + billingQuery.data.enabled || + billingQuery.data.runPolicy === 'paused_by_user' + ) + return; if (ensuredTownRef.current === townId) return; ensuredTownRef.current = townId; ensureMayor.mutate({ townId }); @@ -73,6 +78,7 @@ export function MayorChat({ townId }: MayorChatProps) { agentId: mayorAgentId, enabled: billingQuery.isSuccess && + billingQuery.data.runPolicy === 'automatic' && (!billingQuery.data.enabled || billingQuery.data.state === 'running' || billingQuery.data.state === 'warning' || diff --git a/apps/web/src/components/gastown/TerminalBar.tsx b/apps/web/src/components/gastown/TerminalBar.tsx index 12011242b89..53b8359719d 100644 --- a/apps/web/src/components/gastown/TerminalBar.tsx +++ b/apps/web/src/components/gastown/TerminalBar.tsx @@ -7,6 +7,8 @@ import Link from 'next/link'; import { useGastownTRPC, gastownWsUrl, type GastownOutputs } from '@/lib/gastown/trpc'; import { useSidebar } from '@/components/ui/sidebar'; +import { Switch } from '@/components/ui/switch'; +import { useConfirm } from '@/components/ui/confirm'; import { Popover, PopoverContent, PopoverTrigger } from '@/components/ui/popover'; import { DropdownMenu, @@ -42,6 +44,7 @@ import { CircleDollarSign, } from 'lucide-react'; import { motion, AnimatePresence } from 'motion/react'; +import { toast } from 'sonner'; import styles from './TerminalBar.module.css'; type TerminalBarProps = { @@ -58,6 +61,7 @@ type BillingStatus = GastownOutputs['gastown']['getBillingStatus']; * Can be positioned at bottom/top/right/left with drag-to-resize. */ export function TerminalBar({ townId, basePath: basePathOverride }: TerminalBarProps) { + const trpc = useGastownTRPC(); const townBasePath = basePathOverride ?? `/gastown/${townId}`; const { state: sidebarState, isMobile } = useSidebar(); const { @@ -145,6 +149,11 @@ export function TerminalBar({ townId, basePath: basePathOverride }: TerminalBarP onAgentStatus: handleAgentStatus, onUiAction: handleUiAction, }); + const billingQuery = useQuery({ + ...trpc.gastown.getBillingStatus.queryOptions({ townId }), + refetchInterval: 5_000, + }); + const billing = billingQuery.data ?? alarmWs.data?.billing; const sidebarLeft = isMobile ? '0px' : sidebarState === 'expanded' ? '16rem' : '3rem'; const horizontal = isHorizontal(position); @@ -362,7 +371,8 @@ export function TerminalBar({ townId, basePath: basePathOverride }: TerminalBarP setCollapsed={setCollapsed} setPosition={setPosition} closeTab={closeTab} - billing={alarmWs.data?.billing} + billing={billing} + townId={townId} /> @@ -384,6 +395,7 @@ export function TerminalBar({ townId, basePath: basePathOverride }: TerminalBarP size={size} townId={townId} alarmWs={alarmWs} + billing={billing} fullscreen={isFullscreen} /> {!collapsed && (
@@ -458,7 +473,8 @@ export function TerminalBar({ townId, basePath: basePathOverride }: TerminalBarP setCollapsed={setCollapsed} setPosition={setPosition} closeTab={closeTab} - billing={alarmWs.data?.billing} + billing={billing} + townId={townId} /> {!collapsed && ( @@ -506,6 +523,7 @@ function TabBar({ setPosition, closeTab, billing, + townId, }: { allTabs: TabDef[]; effectiveActiveId: string; @@ -518,8 +536,66 @@ function TabBar({ setPosition: (position: TerminalPosition) => void; closeTab: (tabId: string) => void; billing?: BillingStatus; + townId: string; }) { const borderClass = horizontal ? 'border-b border-white/[0.06]' : 'border-r border-white/[0.06]'; + const trpc = useGastownTRPC(); + const queryClient = useQueryClient(); + const confirm = useConfirm(); + const runPolicyMutation = useMutation( + trpc.gastown.setContainerRunPolicy.mutationOptions({ + onSuccess: status => { + queryClient.setQueryData(trpc.gastown.getBillingStatus.queryKey({ townId }), status); + }, + onError: error => toast.error(error.message), + }) + ); + + const setAutomaticStarts = async (enabled: boolean) => { + if (!enabled) { + const accepted = await confirm({ + title: 'Pause Gas Town?', + description: + 'Active work will be saved and the container will shut down. Scheduled work stays queued until automatic starts are enabled again.', + confirmLabel: 'Save and pause', + cancelLabel: 'Keep running', + }); + if (!accepted) return; + } + runPolicyMutation.mutate({ + townId, + policy: enabled ? 'automatic' : 'paused_by_user', + }); + }; + const billingSummary = (() => { + if (!billing) return ''; + if (billing.state === 'stopping') { + return `Saving and pausing${billing.estimatedRunCharge === undefined ? '' : ` · $${billing.estimatedRunCharge.toFixed(2)} this run`}`; + } + if (billing.runPolicy === 'paused_by_user') { + return `Automatic starts paused${billing.estimatedRunCharge === undefined ? '' : ` · $${billing.estimatedRunCharge.toFixed(2)} last run`}`; + } + if (billing.state === 'warning') { + return `Credits low${billing.estimatedRunCharge === undefined ? '' : ` · $${billing.estimatedRunCharge.toFixed(2)} this run`}`; + } + if (billing.estimatedRunCharge !== undefined && billing.state === 'running') { + return `$${billing.estimatedRunCharge.toFixed(2)} this run`; + } + if (billing.state === 'blocked') return 'Usage paused'; + return billing.estimatedHourlyCharge === undefined + ? 'Usage billing active' + : `$${billing.estimatedHourlyCharge.toFixed(2)}/hr active`; + })(); + const billingAnnouncement = + billing?.state === 'warning' + ? 'Credits are running low.' + : billing?.state === 'blocked' + ? 'Gas Town is paused because credits are unavailable.' + : billing?.state === 'stopping' + ? 'Gas Town is saving work and pausing.' + : billing?.runPolicy === 'paused_by_user' + ? 'Automatic container starts are paused.' + : ''; return (
- {horizontal && billing?.enabled && billing.state !== 'idle' && ( + + {billingAnnouncement} + + + {horizontal && billing && (billing.enabled || billing.runPolicy === 'paused_by_user') && (
- {billing.state === 'warning' - ? 'Credits running low' - : billing.state === 'blocked' || billing.state === 'stopping' - ? 'Usage paused' - : billing.estimatedHourlyCharge === undefined - ? 'Usage billing active' - : `$${billing.estimatedHourlyCharge.toFixed(2)}/hr active`} + {billingSummary} {billing.state === 'warning' && ( )} + | + Allow automatic starts + void setAutomaticStarts(value)} + aria-label="Allow automatic container starts" + className="scale-75" + /> +
+ )} + + {!horizontal && billing && (billing.enabled || billing.runPolicy === 'paused_by_user') && ( +
+ void setAutomaticStarts(value)} + aria-label="Allow automatic container starts" + className="scale-75" + />
)} @@ -756,6 +850,7 @@ function TerminalContent({ size, townId, alarmWs, + billing, fullscreen, }: { activeTab: TabDef; @@ -764,6 +859,7 @@ function TerminalContent({ size: number; townId: string; alarmWs: AlarmWsResult; + billing?: BillingStatus; fullscreen?: boolean; }) { if (collapsed) return null; @@ -780,19 +876,11 @@ function TerminalContent({ className={`overflow-hidden ${horizontal ? '' : 'h-full'} ${fullscreen ? 'h-full' : ''}`} > {activeTab.kind === 'mayor' ? ( - + ) : activeTab.kind === 'status' ? ( ) : ( - + )} @@ -1280,10 +1368,18 @@ function MayorTerminalPane({ }, }) ); + const runPolicyMutation = useMutation( + trpc.gastown.setContainerRunPolicy.mutationOptions({ + onSuccess: status => { + queryClient.setQueryData(trpc.gastown.getBillingStatus.queryKey({ townId }), status); + }, + onError: error => toast.error(error.message), + }) + ); const ensuredTownRef = useRef(null); useEffect(() => { - if (!billing || billing.enabled) return; + if (!billing || billing.enabled || billing.runPolicy === 'paused_by_user') return; if (ensuredTownRef.current === townId) return; ensuredTownRef.current = townId; ensureMayor.mutate({ townId }); @@ -1300,10 +1396,13 @@ function MayorTerminalPane({ }); const mayorAgentId = statusQuery.data?.session?.agentId ?? null; - const billingPaused = + const userPaused = billing?.runPolicy === 'paused_by_user'; + const userPauseStopping = userPaused && billing?.state === 'stopping'; + const creditPaused = billing?.enabled && (billing.state === 'blocked' || billing.state === 'stopping'); const terminalEnabled = billing !== undefined && + billing.runPolicy === 'automatic' && (!billing.enabled || billing.state === 'running' || billing.state === 'warning' || @@ -1317,6 +1416,7 @@ function MayorTerminalPane({ const firstTaskSentRef = useRef(false); useEffect(() => { if (firstTaskSentRef.current) return; + if (!billing || billing.runPolicy === 'paused_by_user') return; const storageKey = `${FIRST_TASK_STORAGE_PREFIX}${townId}`; try { const msg = localStorage.getItem(storageKey); @@ -1327,7 +1427,7 @@ function MayorTerminalPane({ } catch { // localStorage unavailable } - }, [townId]); + }, [billing, townId]); const { terminalRef, connectionStatus, status, fitAddonRef } = useXtermPty({ townId, @@ -1351,22 +1451,32 @@ function MayorTerminalPane({
- {billing?.enabled && !terminalEnabled && ( + {billing && !terminalEnabled && (

- {billingPaused ? 'Gas Town is paused' : 'Start the Mayor when you are ready'} + {userPaused + ? userPauseStopping + ? 'Saving work and pausing Gas Town...' + : 'Automatic starts are paused' + : creditPaused + ? 'Gas Town is paused' + : 'Start the Mayor when you are ready'}

- {billingPaused - ? 'Add credits to the billing account, then resume. Your town state is preserved.' - : billing.estimatedHourlyCharge === undefined - ? 'Container usage is charged only while Gas Town is running.' - : `Estimated container charge: $${billing.estimatedHourlyCharge.toFixed(2)}/hour while running.`} + {userPaused + ? userPauseStopping + ? `The container is still shutting down and may continue accumulating usage briefly.${billing.estimatedRunCharge === undefined ? '' : ` Estimated this run: $${billing.estimatedRunCharge.toFixed(2)}.`}` + : `The container is stopped and scheduled work remains queued.${billing.estimatedRunCharge === undefined ? '' : ` Estimated last run: $${billing.estimatedRunCharge.toFixed(2)}.`} Enable automatic starts when you are ready to continue.` + : creditPaused + ? 'Add credits to the billing account, then resume. Your town state is preserved.' + : billing.estimatedHourlyCharge === undefined + ? 'Container usage is charged only while Gas Town is running.' + : `Estimated container charge: $${billing.estimatedHourlyCharge.toFixed(2)}/hour while running.`}

- {billingPaused && ( + {creditPaused && !userPaused && ( )} - + {userPaused ? ( + + ) : ( + + )}
{ensureMayor.error && (

@@ -1412,6 +1533,7 @@ function AgentTerminalPane({ }) { const enabled = billing !== undefined && + billing.runPolicy === 'automatic' && (!billing.enabled || billing.state === 'running' || billing.state === 'warning'); const { terminalRef, connectionStatus, status } = useXtermPty({ townId, diff --git a/apps/web/src/lib/gastown/types/router.d.ts b/apps/web/src/lib/gastown/types/router.d.ts index c2f0652743d..677e5eb8d6b 100644 --- a/apps/web/src/lib/gastown/types/router.d.ts +++ b/apps/web/src/lib/gastown/types/router.d.ts @@ -552,14 +552,27 @@ export declare const gastownRouter: import('@trpc/server').TRPCBuiltRouter< }[]; billing: { enabled: boolean; - state: 'blocked' | 'degraded' | 'idle' | 'running' | 'starting' | 'stopping' | 'warning'; - payer?: { - type: 'org' | 'user'; - id: string; - } | undefined; + state: + | 'blocked' + | 'degraded' + | 'idle' + | 'paused' + | 'running' + | 'starting' + | 'stopping' + | 'warning'; + runPolicy: 'automatic' | 'paused_by_user'; + payer?: + | { + type: 'org' | 'user'; + id: string; + } + | undefined; remaining?: number | undefined; minimumRequired?: number | undefined; estimatedHourlyCharge?: number | undefined; + estimatedRunCharge?: number | undefined; + runUsageSeconds?: number | undefined; intervalStartedAt?: number | undefined; lastReportedAt?: number | undefined; }; @@ -572,14 +585,60 @@ export declare const gastownRouter: import('@trpc/server').TRPCBuiltRouter< }; output: { enabled: boolean; - state: 'blocked' | 'degraded' | 'idle' | 'running' | 'starting' | 'stopping' | 'warning'; - payer?: { - type: 'org' | 'user'; - id: string; - } | undefined; + state: + | 'blocked' + | 'degraded' + | 'idle' + | 'paused' + | 'running' + | 'starting' + | 'stopping' + | 'warning'; + runPolicy: 'automatic' | 'paused_by_user'; + payer?: + | { + type: 'org' | 'user'; + id: string; + } + | undefined; + remaining?: number | undefined; + minimumRequired?: number | undefined; + estimatedHourlyCharge?: number | undefined; + estimatedRunCharge?: number | undefined; + runUsageSeconds?: number | undefined; + intervalStartedAt?: number | undefined; + lastReportedAt?: number | undefined; + }; + meta: object; + }>; + setContainerRunPolicy: import('@trpc/server').TRPCMutationProcedure<{ + input: { + townId: string; + policy: 'automatic' | 'paused_by_user'; + }; + output: { + enabled: boolean; + state: + | 'blocked' + | 'degraded' + | 'idle' + | 'paused' + | 'running' + | 'starting' + | 'stopping' + | 'warning'; + runPolicy: 'automatic' | 'paused_by_user'; + payer?: + | { + type: 'org' | 'user'; + id: string; + } + | undefined; remaining?: number | undefined; minimumRequired?: number | undefined; estimatedHourlyCharge?: number | undefined; + estimatedRunCharge?: number | undefined; + runUsageSeconds?: number | undefined; intervalStartedAt?: number | undefined; lastReportedAt?: number | undefined; }; @@ -2222,17 +2281,23 @@ export declare const wrappedGastownRouter: import('@trpc/server').TRPCBuiltRoute | 'blocked' | 'degraded' | 'idle' + | 'paused' | 'running' | 'starting' | 'stopping' | 'warning'; - payer?: { - type: 'org' | 'user'; - id: string; - } | undefined; + runPolicy: 'automatic' | 'paused_by_user'; + payer?: + | { + type: 'org' | 'user'; + id: string; + } + | undefined; remaining?: number | undefined; minimumRequired?: number | undefined; estimatedHourlyCharge?: number | undefined; + estimatedRunCharge?: number | undefined; + runUsageSeconds?: number | undefined; intervalStartedAt?: number | undefined; lastReportedAt?: number | undefined; }; @@ -2249,17 +2314,56 @@ export declare const wrappedGastownRouter: import('@trpc/server').TRPCBuiltRoute | 'blocked' | 'degraded' | 'idle' + | 'paused' | 'running' | 'starting' | 'stopping' | 'warning'; - payer?: { - type: 'org' | 'user'; - id: string; - } | undefined; + runPolicy: 'automatic' | 'paused_by_user'; + payer?: + | { + type: 'org' | 'user'; + id: string; + } + | undefined; + remaining?: number | undefined; + minimumRequired?: number | undefined; + estimatedHourlyCharge?: number | undefined; + estimatedRunCharge?: number | undefined; + runUsageSeconds?: number | undefined; + intervalStartedAt?: number | undefined; + lastReportedAt?: number | undefined; + }; + meta: object; + }>; + setContainerRunPolicy: import('@trpc/server').TRPCMutationProcedure<{ + input: { + townId: string; + policy: 'automatic' | 'paused_by_user'; + }; + output: { + enabled: boolean; + state: + | 'blocked' + | 'degraded' + | 'idle' + | 'paused' + | 'running' + | 'starting' + | 'stopping' + | 'warning'; + runPolicy: 'automatic' | 'paused_by_user'; + payer?: + | { + type: 'org' | 'user'; + id: string; + } + | undefined; remaining?: number | undefined; minimumRequired?: number | undefined; estimatedHourlyCharge?: number | undefined; + estimatedRunCharge?: number | undefined; + runUsageSeconds?: number | undefined; intervalStartedAt?: number | undefined; lastReportedAt?: number | undefined; }; diff --git a/services/gastown/src/billing/ContainerBilling.error.ts b/services/gastown/src/billing/ContainerBilling.error.ts index 3ce6e004c29..65577807f41 100644 --- a/services/gastown/src/billing/ContainerBilling.error.ts +++ b/services/gastown/src/billing/ContainerBilling.error.ts @@ -1,6 +1,6 @@ export class ContainerBillingError extends Error { constructor( - readonly code: 'INSUFFICIENT_CREDITS' | 'BILLING_UNAVAILABLE', + readonly code: 'INSUFFICIENT_CREDITS' | 'BILLING_UNAVAILABLE' | 'CONTAINER_PAUSED', message: string, readonly details?: { remaining?: number; minimumRequired?: number } ) { diff --git a/services/gastown/src/billing/container-usage-state.billing.test.ts b/services/gastown/src/billing/container-usage-state.billing.test.ts index 88414e6f077..a93dd55aa70 100644 --- a/services/gastown/src/billing/container-usage-state.billing.test.ts +++ b/services/gastown/src/billing/container-usage-state.billing.test.ts @@ -21,7 +21,7 @@ describe('toBillingStatus', () => { blocked: true, latestBudget: { verdict: 'stop', remaining: 0 }, }) - ).toEqual({ enabled: false, state: 'idle' }); + ).toEqual({ enabled: false, state: 'idle', runPolicy: 'automatic' }); }); it('returns a payer-aware blocked state', () => { @@ -35,6 +35,7 @@ describe('toBillingStatus', () => { ).toEqual({ enabled: true, state: 'blocked', + runPolicy: 'automatic', payer: { type: 'org', id: 'org-1' }, remaining: 0.25, }); @@ -57,17 +58,49 @@ describe('toBillingStatus', () => { latestBudget: { verdict, remaining: 4 }, minimumRequired: 1, estimatedHourlyCharge: 1.2, + reportedUsageSeconds: 900, }; - expect(toBillingStatus(true, state)).toMatchObject({ + expect(toBillingStatus(true, state, 'automatic', 2_000)).toMatchObject({ enabled: true, state: expectedState, + runPolicy: 'automatic', payer: context.subject, remaining: 4, minimumRequired: 1, estimatedHourlyCharge: 1.2, intervalStartedAt: 1_000, lastReportedAt: 2_000, + runUsageSeconds: 900, + estimatedRunCharge: 0.3, + }); + }); + + it('reports a user pause separately from a credit block', () => { + expect( + toBillingStatus( + true, + { + phase: 'idle', + context, + blocked: false, + lastRun: { + startedAt: 1_000, + stoppedAt: 2_000, + usageSeconds: 900, + estimatedCharge: 0.3, + }, + }, + 'paused_by_user', + 2_000 + ) + ).toMatchObject({ + enabled: true, + state: 'paused', + runPolicy: 'paused_by_user', + payer: context.subject, + estimatedRunCharge: 0.3, + runUsageSeconds: 900, }); }); }); diff --git a/services/gastown/src/billing/container-usage-state.billing.ts b/services/gastown/src/billing/container-usage-state.billing.ts index b46161dd8d8..0839e1af620 100644 --- a/services/gastown/src/billing/container-usage-state.billing.ts +++ b/services/gastown/src/billing/container-usage-state.billing.ts @@ -1,4 +1,9 @@ -import type { BudgetVerdict, GastownBillingStatus, UsageContext } from './container-usage.billing'; +import type { + BudgetVerdict, + ContainerRunPolicy, + GastownBillingStatus, + UsageContext, +} from './container-usage.billing'; export type PendingHeartbeat = { seq: number; @@ -21,44 +26,85 @@ export type OpenUsageInterval = { latestBudget?: BudgetVerdict; minimumRequired?: number; estimatedHourlyCharge?: number; + reportedUsageSeconds?: number; stopReason?: 'exit' | 'runtime_signal' | 'activity_expired'; stopObservedAt?: number; finalUsageCaptured?: boolean; }; export type StoredUsageState = - | { phase: 'idle'; context?: UsageContext; blocked?: boolean; latestBudget?: BudgetVerdict } + | { + phase: 'idle'; + context?: UsageContext; + blocked?: boolean; + latestBudget?: BudgetVerdict; + lastRun?: { + startedAt: number; + stoppedAt: number; + usageSeconds: number; + estimatedCharge?: number; + }; + } | OpenUsageInterval; -export function toBillingStatus(enabled: boolean, state: StoredUsageState): GastownBillingStatus { - if (!enabled) return { enabled: false, state: 'idle' }; +export function toBillingStatus( + enabled: boolean, + state: StoredUsageState, + runPolicy: ContainerRunPolicy = 'automatic', + now = Date.now() +): GastownBillingStatus { + if (!enabled) return { enabled: false, state: 'idle', runPolicy }; const payer = state.context?.subject; if (state.phase === 'idle') { return { enabled: true, - state: state.blocked ? 'blocked' : 'idle', + state: runPolicy === 'paused_by_user' ? 'paused' : state.blocked ? 'blocked' : 'idle', + runPolicy, ...(payer ? { payer } : {}), ...(state.latestBudget?.remaining === undefined ? {} : { remaining: state.latestBudget.remaining }), + ...(state.lastRun?.estimatedCharge === undefined + ? {} + : { estimatedRunCharge: state.lastRun.estimatedCharge }), + ...(state.lastRun + ? { + runUsageSeconds: state.lastRun.usageSeconds, + intervalStartedAt: state.lastRun.startedAt, + lastReportedAt: state.lastRun.stoppedAt, + } + : {}), }; } const publicState = - state.phase === 'starting' - ? 'starting' - : state.phase === 'stopping' - ? 'stopping' - : state.latestBudget?.verdict === 'stop' + runPolicy === 'paused_by_user' + ? 'stopping' + : state.phase === 'starting' + ? 'starting' + : state.phase === 'stopping' ? 'stopping' - : state.latestBudget?.verdict === 'warn' - ? 'warning' - : 'running'; + : state.latestBudget?.verdict === 'stop' + ? 'stopping' + : state.latestBudget?.verdict === 'warn' + ? 'warning' + : 'running'; + + const currentSliceSeconds = + state.phase === 'running' || (state.phase === 'stopping' && state.stopObservedAt === undefined) + ? Math.max(0, (now - state.lastReportedAt) / 1000) + : 0; + const runUsageSeconds = (state.reportedUsageSeconds ?? 0) + currentSliceSeconds; + const estimatedRunCharge = + state.estimatedHourlyCharge === undefined + ? undefined + : (runUsageSeconds / 3600) * state.estimatedHourlyCharge; return { enabled: true, state: publicState, + runPolicy, payer, ...(state.latestBudget?.remaining === undefined ? {} @@ -67,6 +113,8 @@ export function toBillingStatus(enabled: boolean, state: StoredUsageState): Gast ...(state.estimatedHourlyCharge === undefined ? {} : { estimatedHourlyCharge: state.estimatedHourlyCharge }), + ...(estimatedRunCharge === undefined ? {} : { estimatedRunCharge }), + runUsageSeconds, intervalStartedAt: state.startEpochMs, lastReportedAt: state.lastReportedAt, }; diff --git a/services/gastown/src/billing/container-usage.billing.ts b/services/gastown/src/billing/container-usage.billing.ts index ee8423fc9f4..7cd99e0069c 100644 --- a/services/gastown/src/billing/container-usage.billing.ts +++ b/services/gastown/src/billing/container-usage.billing.ts @@ -80,15 +80,28 @@ export type ContainerUsageService = { export type GastownBillingStatus = { enabled: boolean; - state: 'idle' | 'starting' | 'running' | 'warning' | 'stopping' | 'blocked' | 'degraded'; + state: + | 'idle' + | 'starting' + | 'running' + | 'warning' + | 'stopping' + | 'blocked' + | 'paused' + | 'degraded'; + runPolicy: ContainerRunPolicy; payer?: { type: 'user' | 'org'; id: string }; remaining?: number; minimumRequired?: number; estimatedHourlyCharge?: number; + estimatedRunCharge?: number; + runUsageSeconds?: number; intervalStartedAt?: number; lastReportedAt?: number; }; +export type ContainerRunPolicy = 'automatic' | 'paused_by_user'; + export const GASTOWN_CONTAINER_SKU = 'cloudflare-container-standard-4'; export const USAGE_HEARTBEAT_INTERVAL_MS = 5 * 60_000; diff --git a/services/gastown/src/dos/Town.do.ts b/services/gastown/src/dos/Town.do.ts index 1a881fd801e..957f85384e5 100644 --- a/services/gastown/src/dos/Town.do.ts +++ b/services/gastown/src/dos/Town.do.ts @@ -76,6 +76,7 @@ import { logger, withLogTags } from '../util/log.util'; import { GASTOWN_CONTAINER_SKU, isGastownBillingEnabled, + type ContainerRunPolicy, type GastownBillingStatus, type UsageActor, type UsageContext, @@ -780,6 +781,8 @@ export class TownDO extends DurableObject { this._drainNonce = (await this.ctx.storage.get('town:drainNonce')) ?? null; this._drainStartedAt = (await this.ctx.storage.get('town:drainStartedAt')) ?? null; this._billingBlocked = (await this.ctx.storage.get('billing:blocked')) ?? false; + this._containerRunPolicy = + (await this.ctx.storage.get('container:runPolicy')) ?? 'automatic'; // All tables are now initialized via beads.initBeadTables(): // beads, bead_events, bead_dependencies, agent_metadata, review_metadata, @@ -837,6 +840,7 @@ export class TownDO extends DurableObject { private _drainNonce: string | null = null; private _drainStartedAt: number | null = null; private _billingBlocked = false; + private _containerRunPolicy: ContainerRunPolicy = 'automatic'; /** Instance UUID of the current container, set by the first heartbeat. */ private _containerInstanceId: string | null = null; @@ -952,9 +956,26 @@ export class TownDO extends DurableObject { } async getBillingStatus(): Promise { - if (!isGastownBillingEnabled(this.env)) return { enabled: false, state: 'idle' }; - await this.prepareContainerBilling(); - return getTownContainerStub(this.env, this.townId).getBillingStatus(); + if (isGastownBillingEnabled(this.env)) await this.prepareContainerBilling(); + const status = await getTownContainerStub(this.env, this.townId).getBillingStatus(); + await this.syncContainerRunPolicy(status.runPolicy); + return status; + } + + async setContainerRunPolicy(policy: ContainerRunPolicy): Promise { + const status = await getTownContainerStub(this.env, this.townId).setRunPolicy(policy); + await this.syncContainerRunPolicy(status.runPolicy); + return status; + } + + private async syncContainerRunPolicy(policy: ContainerRunPolicy): Promise { + if (this._containerRunPolicy === policy) return; + this._containerRunPolicy = policy; + await this.ctx.storage.put('container:runPolicy', policy); + } + + private containerStartsBlocked(): boolean { + return this._billingBlocked || this._containerRunPolicy === 'paused_by_user'; } private async prepareContainerBilling( @@ -987,12 +1008,17 @@ export class TownDO extends DurableObject { } private async updateContainerBilling(): Promise { - if (!isGastownBillingEnabled(this.env)) return { enabled: false, state: 'idle' }; + if (!isGastownBillingEnabled(this.env)) { + return { enabled: false, state: 'idle', runPolicy: this._containerRunPolicy }; + } await this.prepareContainerBilling(); const container = getTownContainerStub(this.env, this.townId); const status = await container.recordUsageHeartbeat(); - const blocked = status.state === 'blocked' || status.state === 'stopping'; + await this.syncContainerRunPolicy(status.runPolicy); + const blocked = + status.runPolicy === 'automatic' && + (status.state === 'blocked' || status.state === 'stopping'); if (blocked !== this._billingBlocked) { this._billingBlocked = blocked; @@ -4235,20 +4261,28 @@ export class TownDO extends DurableObject { // Call once per tick — threaded to ensureContainerReady, maybeDispatchTriageAgent, and getAlarmStatus const rigList = rigs.listRigs(this.sql); const hasRigs = rigList.length > 0; - let billingStatus: GastownBillingStatus = { enabled: false, state: 'idle' }; + let billingStatus: GastownBillingStatus = { + enabled: false, + state: 'idle', + runPolicy: this._containerRunPolicy, + }; if (isGastownBillingEnabled(this.env)) { try { billingStatus = await this.updateContainerBilling(); } catch (err) { - billingStatus = { enabled: true, state: 'degraded' }; + billingStatus = { + enabled: true, + state: 'degraded', + runPolicy: this._containerRunPolicy, + }; logger.warn('alarm: container billing update failed', { error: err instanceof Error ? err.message : String(err), }); } } - if (hasRigs && !this._billingBlocked) { + if (hasRigs && !this.containerStartsBlocked()) { try { await this.ensureContainerReady(rigList); } catch (err) { @@ -4301,7 +4335,7 @@ export class TownDO extends DurableObject { ), ]); - if (workingAgentRows.length > 0 && !this._billingBlocked) { + if (workingAgentRows.length > 0 && !this.containerStartsBlocked()) { const statusChecks = workingAgentRows.map(async row => { try { const containerInfo = await dispatch.checkAgentContainerStatus( @@ -4416,7 +4450,7 @@ export class TownDO extends DurableObject { const sideEffects: Array<() => Promise> = []; try { const actions = reconciler.reconcile(this.sql, { - draining: this._draining || this._billingBlocked, + draining: this._draining || this.containerStartsBlocked(), townConfig, }); metrics.actionsEmitted = actions.length; @@ -4556,7 +4590,7 @@ export class TownDO extends DurableObject { error: err instanceof Error ? err.message : String(err), }) ), - ...(this._billingBlocked + ...(this.containerStartsBlocked() ? [] : [ this.maybeDispatchTriageAgent(cachedTriageCount, rigList).catch(err => @@ -4654,7 +4688,7 @@ export class TownDO extends DurableObject { private async stopContainerIfIdle(): Promise { await _stopContainerIfIdle({ hasActiveWork: () => this.hasActiveWork(), - isDraining: () => this._draining || this._billingBlocked, + isDraining: () => this._draining || this.containerStartsBlocked(), getMayor: () => agents.listAgents(this.sql, { role: 'mayor' })[0] ?? null, getTownId: () => this.townId, getLastIdleStopAt: () => this.ctx.storage.get('container:lastIdleStopAt'), @@ -4679,7 +4713,7 @@ export class TownDO extends DurableObject { townId, containerDoId: getTownContainerDoId(this.env, townId), outcome, - isDraining: () => this._draining || this._billingBlocked, + isDraining: () => this._draining || this.containerStartsBlocked(), getConsecutiveHealthFailures: () => this.ctx.storage.get(CONTAINER_HEALTH_STORAGE_KEYS.consecutiveHealthFailures), setConsecutiveHealthFailures: value => @@ -5085,7 +5119,7 @@ export class TownDO extends DurableObject { cachedRigList?: rigs.RigRecord[], cachedActiveWork?: boolean ): Promise { - if (this._billingBlocked) return; + if (this.containerStartsBlocked()) return; const rigList = cachedRigList ?? rigs.listRigs(this.sql); if (rigList.length === 0) return; @@ -5544,7 +5578,11 @@ export class TownDO extends DurableObject { cached?.billingStatus ?? (isGastownBillingEnabled(this.env) ? await getTownContainerStub(this.env, this.townId).getBillingStatus() - : { enabled: false, state: 'idle' as const }); + : { + enabled: false, + state: 'idle' as const, + runPolicy: this._containerRunPolicy, + }); return { alarm: { diff --git a/services/gastown/src/dos/TownContainer.do.ts b/services/gastown/src/dos/TownContainer.do.ts index bb333302491..8faa8e0ddef 100644 --- a/services/gastown/src/dos/TownContainer.do.ts +++ b/services/gastown/src/dos/TownContainer.do.ts @@ -4,6 +4,7 @@ import { getContainerUsageService, isGastownBillingEnabled, USAGE_HEARTBEAT_INTERVAL_MS, + type ContainerRunPolicy, type GastownBillingStatus, type UsageContext, } from '../billing/container-usage.billing'; @@ -16,6 +17,7 @@ import { const TC_LOG = '[TownContainer.do]'; const BILLING_CONTEXT_KEY = 'billing:context'; const BILLING_STATE_KEY = 'billing:state'; +const RUN_POLICY_KEY = 'container:runPolicy'; /** * TownContainer — a Cloudflare Container per town. @@ -96,6 +98,10 @@ export class TownContainerDO extends Container { override async onStart(): Promise { console.log(`${TC_LOG} container started for DO id=${this.ctx.id.toString()}`); + if ((await this.getRunPolicy()) === 'paused_by_user') { + await this.stop(); + return; + } if (!isGastownBillingEnabled(this.env)) return; try { @@ -126,9 +132,11 @@ export class TownContainerDO extends Container { * triggered a cold start. */ async warmUp(): Promise<{ coldStart: boolean; durationMs: number }> { + await this.assertAutomaticStartsEnabled(); if (isGastownBillingEnabled(this.env)) { await this.ensureBillingInterval(); } + await this.assertAutomaticStartsEnabled(); if (this.ctx.container?.running === true) { // Runtime-level fast path only. TownDO's /health probe is the // application-level liveness source and may still recover a wedged @@ -171,7 +179,6 @@ export class TownContainerDO extends Container { ...state, phase: 'stopping', stopReason: 'activity_expired', - stopObservedAt: Date.now(), } satisfies OpenUsageInterval); } } @@ -183,9 +190,11 @@ export class TownContainerDO extends Container { } override async fetch(request: Request): Promise { + await this.assertAutomaticStartsEnabled(); if (isGastownBillingEnabled(this.env)) { await this.ensureBillingInterval(); } + await this.assertAutomaticStartsEnabled(); return super.fetch(request); } @@ -198,11 +207,15 @@ export class TownContainerDO extends Container { } async getBillingStatus(): Promise { - return toBillingStatus(isGastownBillingEnabled(this.env), await this.getUsageState()); + return toBillingStatus( + isGastownBillingEnabled(this.env), + await this.getUsageState(), + await this.getRunPolicy() + ); } async recordUsageHeartbeat(): Promise { - if (!isGastownBillingEnabled(this.env)) return { enabled: false, state: 'idle' }; + if (!isGastownBillingEnabled(this.env)) return this.getBillingStatus(); let state = await this.getUsageState(); if (state.phase === 'starting') { @@ -227,8 +240,17 @@ export class TownContainerDO extends Container { throw error; } } - if (state.phase === 'idle') return toBillingStatus(true, state); + const runPolicy = await this.getRunPolicy(); + if (state.phase === 'idle') return toBillingStatus(true, state, runPolicy); if (state.phase === 'stopping') { + const stoppingRuntimeState = await this.getState(); + if ( + stoppingRuntimeState.status === 'running' || + stoppingRuntimeState.status === 'healthy' || + stoppingRuntimeState.status === 'stopping' + ) { + return toBillingStatus(true, state, runPolicy); + } await this.closeBillingInterval(state.stopReason ?? 'runtime_signal'); return this.getBillingStatus(); } @@ -241,15 +263,15 @@ export class TownContainerDO extends Container { const observedAt = Date.now(); if (observedAt - state.lastReportedAt < USAGE_HEARTBEAT_INTERVAL_MS) { - return toBillingStatus(true, state); + return toBillingStatus(true, state, runPolicy); } try { const updated = await this.captureHeartbeat(state, observedAt); - return toBillingStatus(true, updated); + return toBillingStatus(true, updated, runPolicy); } catch (error) { console.error(`${TC_LOG} billing heartbeat failed`, error); - const status = toBillingStatus(true, state); + const status = toBillingStatus(true, state, runPolicy); return { ...status, state: 'degraded' }; } } @@ -263,7 +285,6 @@ export class TownContainerDO extends Container { ...state, phase: 'stopping', stopReason: 'runtime_signal', - stopObservedAt: Date.now(), } satisfies OpenUsageInterval); } } @@ -278,14 +299,32 @@ export class TownContainerDO extends Container { ...state, phase: 'stopping', stopReason: 'activity_expired', - stopObservedAt: Date.now(), } satisfies OpenUsageInterval); } } await this.stop(); } + async setRunPolicy(policy: ContainerRunPolicy): Promise { + await this.ctx.storage.put(RUN_POLICY_KEY, policy); + if (policy === 'paused_by_user') { + const state = await this.getUsageState(); + const canCloseBillingInterval = + state.phase === 'running' || (state.phase === 'starting' && state.startRecorded); + if (isGastownBillingEnabled(this.env) && canCloseBillingInterval) { + await this.ctx.storage.put(BILLING_STATE_KEY, { + ...state, + phase: 'stopping', + stopReason: 'runtime_signal', + } satisfies OpenUsageInterval); + } + await this.stop(); + } + return this.getBillingStatus(); + } + private async ensureBillingInterval(): Promise { + await this.assertAutomaticStartsEnabled(); if (!isGastownBillingEnabled(this.env)) return { phase: 'idle' }; const existing = await this.getUsageState(); @@ -321,6 +360,7 @@ export class TownContainerDO extends Container { startRecorded: false, seq: 0, lastReportedAt: observedAt, + reportedUsageSeconds: 0, }; await transaction.put(BILLING_STATE_KEY, starting); return starting; @@ -561,12 +601,14 @@ export class TownContainerDO extends Container { const latest = await this.getUsageState(); if (latest.phase === 'idle' || latest.startEpochMs !== interval.startEpochMs) return latest; + if (latest.seq >= pending.seq) return latest; const updated: OpenUsageInterval = { ...latest, phase: latest.phase === 'starting' ? 'running' : latest.phase, seq: pending.seq, lastReportedAt: pending.observedAt, + reportedUsageSeconds: (latest.reportedUsageSeconds ?? 0) + pending.usageSinceLast, latestBudget: ack.budget, }; delete updated.pendingHeartbeat; @@ -581,7 +623,15 @@ export class TownContainerDO extends Container { const selected = await this.ctx.storage.transaction(async transaction => { const latest = await transaction.get(BILLING_STATE_KEY); if (!latest || latest.phase === 'idle') return null; - if (latest.phase === 'stopping') return latest; + if (latest.phase === 'stopping') { + if (latest.stopObservedAt !== undefined) return latest; + const stopping = { + ...latest, + stopObservedAt: observedAtOverride ?? Date.now(), + } satisfies OpenUsageInterval; + await transaction.put(BILLING_STATE_KEY, stopping); + return stopping; + } const stopping: OpenUsageInterval = { ...latest, @@ -624,11 +674,21 @@ export class TownContainerDO extends Container { const latest = await this.getUsageState(); if (latest.phase !== 'idle' && latest.startEpochMs === stopping.startEpochMs) { + const estimatedCharge = + stopping.estimatedHourlyCharge === undefined + ? undefined + : ((stopping.reportedUsageSeconds ?? 0) / 3600) * stopping.estimatedHourlyCharge; await this.ctx.storage.put(BILLING_STATE_KEY, { phase: 'idle', context: stopping.context, blocked: stopping.latestBudget?.verdict === 'stop', latestBudget: stopping.latestBudget, + lastRun: { + startedAt: stopping.startEpochMs, + stoppedAt: stopObservedAt, + usageSeconds: stopping.reportedUsageSeconds ?? 0, + ...(estimatedCharge === undefined ? {} : { estimatedCharge }), + }, } satisfies StoredUsageState); } } catch (error) { @@ -639,6 +699,19 @@ export class TownContainerDO extends Container { private async getUsageState(): Promise { return (await this.ctx.storage.get(BILLING_STATE_KEY)) ?? { phase: 'idle' }; } + + private async getRunPolicy(): Promise { + return (await this.ctx.storage.get(RUN_POLICY_KEY)) ?? 'automatic'; + } + + private async assertAutomaticStartsEnabled(): Promise { + if ((await this.getRunPolicy()) === 'paused_by_user') { + throw new ContainerBillingError( + 'CONTAINER_PAUSED', + 'Automatic starts are paused for this Gas Town' + ); + } + } } export function getTownContainerStub(env: Env, townId: string) { diff --git a/services/gastown/src/trpc/router.ts b/services/gastown/src/trpc/router.ts index 8a87134ca41..9729839c863 100644 --- a/services/gastown/src/trpc/router.ts +++ b/services/gastown/src/trpc/router.ts @@ -97,6 +97,9 @@ function mapContainerBillingError(error: unknown): TRPCError | null { if (code === 'INSUFFICIENT_CREDITS' || message.includes('in credits to start')) { return new TRPCError({ code: 'PRECONDITION_FAILED', message, cause: error }); } + if (code === 'CONTAINER_PAUSED') { + return new TRPCError({ code: 'PRECONDITION_FAILED', message, cause: error }); + } if (code === 'BILLING_UNAVAILABLE' || message.includes('billing')) { return new TRPCError({ code: 'SERVICE_UNAVAILABLE', message, cause: error }); } @@ -1032,6 +1035,40 @@ export const gastownRouter = router({ return getTownDOStub(ctx.env, input.townId).getBillingStatus(); }), + setContainerRunPolicy: gastownProcedure + .input( + z.object({ + townId: z.string().uuid(), + policy: z.enum(['automatic', 'paused_by_user']), + }) + ) + .output(RpcBillingStatusOutput) + .mutation(async ({ ctx, input }) => { + const ownership = await resolveTownOwnership(ctx.env, ctx, input.townId); + if (ownership.type === 'admin') { + throw new TRPCError({ + code: 'FORBIDDEN', + message: 'Admins cannot pause containers for towns they do not own', + }); + } + + const town = getTownDOStub(ctx.env, input.townId); + if (ownership.type === 'org') { + const membership = getOrgMembership(ctx.orgMemberships, ownership.orgId); + const townConfig = await town.getTownConfig(); + const isOrgOwner = membership?.role === 'owner'; + const isTownCreator = ctx.userId === townConfig.created_by_user_id; + if (!isOrgOwner && !isTownCreator) { + throw new TRPCError({ + code: 'FORBIDDEN', + message: 'Only town creators and org owners can change automatic starts', + }); + } + } + + return town.setContainerRunPolicy(input.policy); + }), + ensureMayor: gastownProcedure .input(z.object({ townId: z.string().uuid() })) .output(RpcMayorSendResultOutput) diff --git a/services/gastown/src/trpc/schemas.ts b/services/gastown/src/trpc/schemas.ts index 44009b08260..441effe5433 100644 --- a/services/gastown/src/trpc/schemas.ts +++ b/services/gastown/src/trpc/schemas.ts @@ -104,11 +104,23 @@ export const MayorStatusOutput = z.object({ export const BillingStatusOutput = z.object({ enabled: z.boolean(), - state: z.enum(['idle', 'starting', 'running', 'warning', 'stopping', 'blocked', 'degraded']), + state: z.enum([ + 'idle', + 'starting', + 'running', + 'warning', + 'stopping', + 'blocked', + 'paused', + 'degraded', + ]), + runPolicy: z.enum(['automatic', 'paused_by_user']), payer: z.object({ type: z.enum(['user', 'org']), id: z.string() }).optional(), remaining: z.number().optional(), minimumRequired: z.number().optional(), estimatedHourlyCharge: z.number().optional(), + estimatedRunCharge: z.number().optional(), + runUsageSeconds: z.number().optional(), intervalStartedAt: z.number().optional(), lastReportedAt: z.number().optional(), }); diff --git a/services/gastown/test/integration/billing-disabled.test.ts b/services/gastown/test/integration/billing-disabled.test.ts index de94619959b..4c1169a63bf 100644 --- a/services/gastown/test/integration/billing-disabled.test.ts +++ b/services/gastown/test/integration/billing-disabled.test.ts @@ -7,9 +7,29 @@ describe('Gastown billing feature flag', () => { const town = env.TOWN.get(env.TOWN.idFromName(townId)); await town.setTownId(townId); - await expect(town.getBillingStatus()).resolves.toEqual({ enabled: false, state: 'idle' }); + await expect(town.getBillingStatus()).resolves.toEqual({ + enabled: false, + state: 'idle', + runPolicy: 'automatic', + }); await expect(town.getAlarmStatus()).resolves.toMatchObject({ - billing: { enabled: false, state: 'idle' }, + billing: { enabled: false, state: 'idle', runPolicy: 'automatic' }, + }); + }); + + it('persists a user-controlled automatic-start pause', async () => { + const townId = `billing-pause-${crypto.randomUUID()}`; + const town = env.TOWN.get(env.TOWN.idFromName(townId)); + await town.setTownId(townId); + + await expect(town.setContainerRunPolicy('paused_by_user')).resolves.toMatchObject({ + runPolicy: 'paused_by_user', + }); + await expect(town.getBillingStatus()).resolves.toMatchObject({ + runPolicy: 'paused_by_user', + }); + await expect(town.setContainerRunPolicy('automatic')).resolves.toMatchObject({ + runPolicy: 'automatic', }); }); }); From 228f9b7e6f32e51570098523a9842b8d65409f29 Mon Sep 17 00:00:00 2001 From: John Fawcett Date: Wed, 22 Jul 2026 12:33:54 -0500 Subject: [PATCH 04/10] feat(gastown): integrate container usage meter --- .specs/gastown-usage-based-billing.md | 74 ++--- dev/local/services.ts | 2 +- pnpm-lock.yaml | 3 + services/gastown/package.json | 1 + .../container-usage-state.billing.test.ts | 32 ++- .../billing/container-usage-state.billing.ts | 27 +- .../billing/container-usage.billing.test.ts | 75 +++-- .../src/billing/container-usage.billing.ts | 146 ++-------- .../src/billing/sku-rate.billing.test.ts | 12 + .../gastown/src/billing/sku-rate.billing.ts | 24 ++ services/gastown/src/dos/Town.do.ts | 47 ++- services/gastown/src/dos/TownContainer.do.ts | 269 ++++++------------ services/gastown/worker-configuration.d.ts | 2 +- services/gastown/wrangler.jsonc | 10 + 14 files changed, 325 insertions(+), 399 deletions(-) create mode 100644 services/gastown/src/billing/sku-rate.billing.test.ts create mode 100644 services/gastown/src/billing/sku-rate.billing.ts diff --git a/.specs/gastown-usage-based-billing.md b/.specs/gastown-usage-based-billing.md index c0191647a75..698711ece53 100644 --- a/.specs/gastown-usage-based-billing.md +++ b/.specs/gastown-usage-based-billing.md @@ -43,8 +43,8 @@ usage-based charge, not a subscription or flat-rate entitlement. awake time. 2. The attributable cost MUST be calculated by the metering service from a versioned SKU catalog. Gastown MUST send the catalog SKU and MUST NOT embed Cloudflare prices in application code. -3. The initial production SKU MUST represent Gastown's configured `standard-4` container. A - container-size change MUST use a corresponding new or updated catalog entry before rollout. +3. The initial production SKU is `gastown-standard-2026-07`, representing Gastown's configured + `standard-4` container. A container-size or price change MUST use a new catalog SKU. 4. Charges MUST settle in Kilo credits. One dollar of calculated charge uses one dollar of Kilo credit balance under the existing credit conversion rules. 5. The initial scope includes Cloudflare Container compute and memory costs represented by the @@ -58,8 +58,7 @@ usage-based charge, not a subscription or flat-rate entitlement. 3. Legacy user towns without `owner_id` MAY fall back to `owner_user_id`. A town with no reliable payer MUST NOT start billable work. 4. Interactive work SHOULD identify the authenticated user as the actor. Autonomous scheduling - MUST use a stable Gastown bot identity and MAY include the initiating user as `onBehalfOf` when - known. + MUST use a stable Gastown bot identity with `onBehalfOf` equal to the billing subject. 5. Payer, SKU, and interval start time MUST remain immutable for an open interval. An ownership or SKU change takes effect on the next container runtime. @@ -68,11 +67,10 @@ usage-based charge, not a subscription or flat-rate entitlement. Gastown's `TownContainerDO` is the producer for `service: "gastown"`. It MUST durably retain enough interval state to retry every call after a Durable Object restart. -1. **Start:** Before billable work is admitted, Gastown MUST complete the admission check described - below and await an acknowledged `recordStart`. The context MUST include the Town Container DO ID - as `instanceId`, the town ID as `sessionId`, the configured SKU, payer, actor, and - `metadata.townId`. Failure of both Postgres and the failover buffer MUST block admission and be - retried; `durable: "buffer"` is a successful ack. +1. **Start:** Gastown MUST await a successful `recordStart` before starting the container. The + context MUST include the Town Container DO ID as `instanceId`, the town ID as `sessionId`, the + configured SKU, payer, actor, and `metadata.townId`. The current meter rejects missing, + incompatible, or closed SKUs; transport or persistence failure MUST fail the cold start closed. 2. **Heartbeat:** Approximately every five minutes while `getState()` confirms the container is running, Gastown MUST await `recordHeartbeat` with monotonic `seq`, measured `usageSinceLast`, and the original context for self-healing. Gastown MUST NOT infer billable @@ -99,7 +97,7 @@ Stop reasons map as follows: | Component | Responsibility | |---|---| -| UBB / `@kilocode/container-usage` | SKU catalog, admission decision, reservations, usage ledger, credit debit, balance floor, idempotency, and reconciliation | +| UBB / `@kilocode/container-usage` | Typed client/contracts, SKU validation, usage ledger, idempotency, and reconciliation; wallet admission and budget debits remain follow-up work | | `TownDO` | Resolve payer and actor, gate every dispatch path, expose billing state to the UI, stop scheduling after a budget stop, and request usage heartbeats from the container DO | | `TownContainerDO` | Own the durable interval state, guard cold starts, call `recordStart`/`recordHeartbeat`/`recordStop`, observe actual container runtime, and stop the container when instructed | | Gastown tRPC | Verify town access, pass the authenticated actor to `TownDO`, return stable billing errors/status, and never make an independent billing decision | @@ -109,9 +107,16 @@ Gastown MUST receive the metering WorkerEntrypoint through a Cloudflare service development, and test Wrangler configurations MUST bind the same typed interface; tests MAY use an in-memory fake ledger. -### Required admission contract +The initial integration binds `CONTAINER_USAGE` to the `container-usage-meter` Worker and its +`ContainerUsageMeter` entrypoint. It records real intervals and SKU-rated seconds. The merged +foundation currently returns `continue` for every heartbeat and does not expose wallet admission, +reservations, credit debits, or remaining balance. Therefore `GASTOWN_BILLING_ENABLED` MUST remain +off for customer charging until those ledger capabilities are implemented; the current integration +is suitable for shadow metering and reconciliation. -The proposed three recording calls are not sufficient by themselves to prevent a cold start: +### Required admission contract before customer charging + +The current three recording calls are not sufficient by themselves to prevent a cold start: `recordStart` returns no budget verdict, while `recordHeartbeat` is defined only after `getState()` confirms that the container is running. UBB therefore MUST provide an upstream admission operation before paid rollout. This is a fourth operation, but it is not a usage-recording @@ -223,16 +228,17 @@ The gate behaves as follows: repeated for every agent because the town has one shared container. 2. If the town is in `billing_stopping` or `billing_blocked`, reject new dispatch and do not call a container proxy that would wake the runtime. -3. For a cold start, resolve payer and actor, call `authorizeStart`, persist the authorization and - context in `TownContainerDO`, await `recordStart`, and only then call `startAndWaitForPorts` or a - proxy operation that can boot the container. -4. If authorization is denied, return the stable domain code `INSUFFICIENT_CREDITS` with payer type, - remaining balance, and minimum required balance. Do not mutate a bead or agent to `working`. +3. For a cold start, resolve payer and actor, persist the context in `TownContainerDO`, await the + real package client's `recordStart`, and only then call `startAndWaitForPorts` or a proxy operation + that can boot the container. Once wallet admission exists, `authorizeStart` precedes this step. +4. Current SKU admission failures return `BILLING_UNAVAILABLE` and do not mutate a bead or agent to + `working`. Future wallet denial returns `INSUFFICIENT_CREDITS` with payer type, remaining balance, + and minimum required balance. 5. If admission infrastructure is unavailable, return `BILLING_UNAVAILABLE` and fail closed for the cold start. An already-running interval continues and fails open as described under Failure Handling. 6. If container startup fails after `recordStart`, close the zero- or near-zero-usage interval with - `runtime_signal` and release unused admission reservation. + `runtime_signal`; once reservations exist, release the unused reservation. 7. `TownContainerDO` MUST defend the boundary as well as `TownDO`: a request that could cold-start the runtime without prepared metering context MUST be rejected. This prevents a new proxy route from accidentally bypassing billing. @@ -409,11 +415,11 @@ primary CTA. Amounts and usage counters SHOULD use tabular numerals. ## Failure Handling -1. A metering ack with `durable: "buffer"` MUST be treated as success and monitored as degraded - operation. -2. If a heartbeat throws because both persistence paths are unavailable, Gastown SHOULD fail open - for the running interval, retry with the same idempotency key, and alert. It MUST NOT terminate - customer work solely because the billing service is temporarily unreachable. +1. The current meter acknowledges durable Postgres writes only. Its typed client retries transient + RPC failures with the same generated idempotency key. +2. If a heartbeat remains unavailable after client retries, Gastown SHOULD fail open for the running + interval, retain its pending segment, and alert. It MUST NOT terminate customer work solely + because the billing service is temporarily unreachable. 3. If `recordStop` cannot be acknowledged during shutdown, the pending stop MUST survive and retry later. A missing stop ack MUST NOT cause the next runtime to reuse the previous interval. 4. Metering errors and retries MUST NOT log credit balances, tokens, or other credentials. @@ -422,9 +428,11 @@ primary CTA. Amounts and usage counters SHOULD use tabular numerals. Rollout SHOULD proceed through a single default-off backend flag: -1. **Contract and catalog:** Ship the `standard-4` SKU, `3x` customer rate, and admission operation - behind `GASTOWN_BILLING_ENABLED=false`; provision the production UBB service binding before the - flag can be enabled. +1. **Contract and catalog:** Ship `gastown-standard-2026-07`, its `3x` customer rate, the + `CONTAINER_USAGE` binding to `ContainerUsageMeter`, and the package client behind + `GASTOWN_BILLING_ENABLED=false`. The SKU row is an operational prerequisite and MUST exist in the + production catalog before the flag is enabled; schema migrations intentionally do not seed its + environment-specific rate. 2. **Shadow reporting:** Emit complete intervals and calculated charges without reserving or debiting credits and without enforcing `warn`/`stop`. 3. **Admission and UX:** Enable authoritative cold-start admission and all blocked/warning UI while @@ -442,18 +450,18 @@ admission, debits, warnings, and stops together. A separate default-off disabled; it MUST NOT alter backend billing behavior. Dashboards MUST expose awake seconds, base Cloudflare cost, customer charge, gross multiplier, -buffered writes, unclosed intervals, admission denials, RPC failures, graceful-stop duration, forced -stops, and counts of `warn` and `stop` verdicts. +unclosed intervals, SKU admission denials, RPC failures, graceful-stop duration, forced stops, and +counts of `warn` and `stop` verdicts. ### Required verification 1. Unit tests MUST cover every durable interval transition, identical retry payloads, dedup acks, stop/heartbeat races, ownership mapping, and billing-blocked scheduler behavior. -2. Worker integration tests MUST use a fake UBB binding to cover PG-buffer acks, total failure, - allow/deny admission, all three heartbeat verdicts, lost responses, DO restart between calls, - idle stop, watchdog restart, and final partial-window settlement. -3. Concurrency tests MUST prove two towns cannot consume the same minimum-start reservation for one - payer. +2. Worker integration tests MUST use a fake UBB binding to cover successful and rejected SKU starts, + total failure, all three heartbeat verdicts, lost responses, DO restart between calls, idle stop, + watchdog restart, and final partial-window settlement. +3. Before wallet admission ships, concurrency tests MUST prove two towns cannot consume the same + minimum-start reservation for one payer. 4. End-to-end tests MUST cover a personal town and an organization town through start, warning, graceful pause, top-up, and resume. They MUST also prove that merely opening the town page creates no interval and no Cloudflare container start. diff --git a/dev/local/services.ts b/dev/local/services.ts index c31618c1fd8..470b027308f 100644 --- a/dev/local/services.ts +++ b/dev/local/services.ts @@ -251,7 +251,7 @@ const serviceMeta: Record = { // gastown 'cloudflare-gastown': { group: 'gastown', - dependsOn: ['postgres', 'cloudflare-git-token-service', 'nextjs'], + dependsOn: ['postgres', 'cloudflare-git-token-service', 'container-usage-meter', 'nextjs'], dir: 'services/gastown', }, 'cloudflare-wasteland': { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c8d3cbd7bf6..89ddc5d761b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2164,6 +2164,9 @@ importers: '@hono/trpc-server': specifier: 0.4.2 version: 0.4.2(@trpc/server@11.17.0(typescript@5.9.3))(hono@4.12.25) + '@kilocode/container-usage': + specifier: workspace:* + version: link:../../packages/container-usage '@kilocode/db': specifier: workspace:* version: link:../../packages/db diff --git a/services/gastown/package.json b/services/gastown/package.json index ae7e6b1bf21..9d5abcad00e 100644 --- a/services/gastown/package.json +++ b/services/gastown/package.json @@ -23,6 +23,7 @@ }, "dependencies": { "@cloudflare/containers": "0.1.1", + "@kilocode/container-usage": "workspace:*", "@hono/trpc-server": "0.4.2", "@kilocode/db": "workspace:*", "@kilocode/worker-utils": "workspace:*", diff --git a/services/gastown/src/billing/container-usage-state.billing.test.ts b/services/gastown/src/billing/container-usage-state.billing.test.ts index a93dd55aa70..923f536a458 100644 --- a/services/gastown/src/billing/container-usage-state.billing.test.ts +++ b/services/gastown/src/billing/container-usage-state.billing.test.ts @@ -1,11 +1,16 @@ import { describe, expect, it } from 'vitest'; -import { toBillingStatus, type StoredUsageState } from './container-usage-state.billing'; +import { + createPendingStop, + toBillingStatus, + type OpenUsageInterval, + type StoredUsageState, +} from './container-usage-state.billing'; import type { UsageContext } from './container-usage.billing'; const context: UsageContext = { service: 'gastown', instanceId: 'container-1', - sku: 'cloudflare-container-standard-4', + sku: 'gastown-standard-2026-07', subject: { type: 'org', id: 'org-1' }, actor: { type: 'user', id: 'user-1' }, sessionId: 'town-1', @@ -49,8 +54,6 @@ describe('toBillingStatus', () => { const state: StoredUsageState = { phase: 'running', context, - authorizationId: 'auth-1', - authorizationKey: 'authorize-1', startEpochMs: 1_000, startRecorded: true, seq: 2, @@ -104,3 +107,24 @@ describe('toBillingStatus', () => { }); }); }); + +describe('createPendingStop', () => { + it('captures whole seconds remaining after the last acknowledged segment', () => { + const state: OpenUsageInterval = { + phase: 'stopping', + context, + startEpochMs: 1_000, + startRecorded: true, + seq: 2, + lastReportedAt: 2_000, + reportedUsageSeconds: 1, + }; + + expect(createPendingStop(state, 7_500, 'runtime_signal')).toEqual({ + seq: 3, + usageSinceLast: 5, + measuredAtMs: 7_000, + reason: 'runtime_signal', + }); + }); +}); diff --git a/services/gastown/src/billing/container-usage-state.billing.ts b/services/gastown/src/billing/container-usage-state.billing.ts index 0839e1af620..7ebd05586cd 100644 --- a/services/gastown/src/billing/container-usage-state.billing.ts +++ b/services/gastown/src/billing/container-usage-state.billing.ts @@ -9,27 +9,30 @@ export type PendingHeartbeat = { seq: number; observedAt: number; usageSinceLast: number; - idempotencyKey: string; +}; + +export type PendingStop = { + seq: number; + usageSinceLast: number; + measuredAtMs: number; + reason: 'exit' | 'runtime_signal' | 'activity_expired'; }; export type OpenUsageInterval = { phase: 'starting' | 'running' | 'stopping'; context: UsageContext; - authorizationId: string; - authorizationKey: string; - authorizationExpiresAt?: number; startEpochMs: number; startRecorded: boolean; seq: number; lastReportedAt: number; pendingHeartbeat?: PendingHeartbeat; + pendingStop?: PendingStop; latestBudget?: BudgetVerdict; minimumRequired?: number; estimatedHourlyCharge?: number; reportedUsageSeconds?: number; stopReason?: 'exit' | 'runtime_signal' | 'activity_expired'; stopObservedAt?: number; - finalUsageCaptured?: boolean; }; export type StoredUsageState = @@ -47,6 +50,20 @@ export type StoredUsageState = } | OpenUsageInterval; +export function createPendingStop( + state: OpenUsageInterval, + stopObservedAt: number, + reason: PendingStop['reason'] +): PendingStop { + const usageSinceLast = Math.floor(Math.max(0, stopObservedAt - state.lastReportedAt) / 1000); + return { + seq: state.seq + 1, + usageSinceLast, + measuredAtMs: state.lastReportedAt + usageSinceLast * 1000, + reason, + }; +} + export function toBillingStatus( enabled: boolean, state: StoredUsageState, diff --git a/services/gastown/src/billing/container-usage.billing.test.ts b/services/gastown/src/billing/container-usage.billing.test.ts index 64ad70656e4..94af586fa97 100644 --- a/services/gastown/src/billing/container-usage.billing.test.ts +++ b/services/gastown/src/billing/container-usage.billing.test.ts @@ -1,6 +1,10 @@ -import { describe, expect, it } from 'vitest'; -import { getContainerUsageService, isGastownBillingEnabled } from './container-usage.billing'; -import type { ContainerUsageService } from './container-usage.billing'; +import type { ContainerUsageRpcMethods, RecordStartInput } from '@kilocode/container-usage'; +import { describe, expect, it, vi } from 'vitest'; +import { + GASTOWN_CONTAINER_SKU, + getContainerUsageClient, + isGastownBillingEnabled, +} from './container-usage.billing'; function envFixture(overrides: Partial): Env { return overrides as Env; @@ -13,47 +17,42 @@ describe('Gastown billing configuration', () => { expect(isGastownBillingEnabled(envFixture({}))).toBe(false); }); - it('uses the no-charge development stub when the binding is absent', async () => { - const service = getContainerUsageService(envFixture({ ENVIRONMENT: 'development' })); - const authorization = await service.authorizeStart({ - context: { - service: 'gastown', - instanceId: 'container-1', - sku: 'cloudflare-container-standard-4', - subject: { type: 'user', id: 'user-1' }, - actor: { type: 'user', id: 'user-1' }, - sessionId: 'town-1', - metadata: { townId: 'town-1' }, - }, - idempotencyKey: 'authorize-1', - observedAt: 1_000, - }); - - expect(authorization).toMatchObject({ verdict: 'allow', minimumRequired: 1 }); + it('uses the production Gastown SKU', () => { + expect(GASTOWN_CONTAINER_SKU).toBe('gastown-standard-2026-07'); }); - it('prefers the configured WorkerEntrypoint binding', () => { + it('creates a client backed by the WorkerEntrypoint binding', async () => { + const recordStart = vi.fn(async (_input: RecordStartInput) => ({ + success: true as const, + ack: { intervalId: 'test', durable: 'pg' as const, dedup: false }, + })); const binding = { - authorizeStart: async () => ({ - verdict: 'deny' as const, - remaining: 0, - minimumRequired: 1, - }), - recordStart: async () => ({ intervalId: 'test', durable: 'pg' as const, dedup: false }), - recordHeartbeat: async () => ({ - intervalId: 'test', - durable: 'pg' as const, - dedup: false, - budget: { verdict: 'continue' as const }, - }), - recordStop: async () => ({ intervalId: 'test', durable: 'pg' as const, dedup: false }), - } satisfies ContainerUsageService; + recordStart, + recordHeartbeat: vi.fn(), + recordStop: vi.fn(), + } satisfies ContainerUsageRpcMethods; - expect(getContainerUsageService(envFixture({ CONTAINER_USAGE: binding }))).toBe(binding); + await getContainerUsageClient(envFixture({ CONTAINER_USAGE: binding })).recordStart({ + instanceId: 'container-1', + sku: GASTOWN_CONTAINER_SKU, + subject: { type: 'user', id: 'user-1' }, + actor: { type: 'user', id: 'user-1' }, + sessionId: 'town-1', + metadata: { townId: 'town-1' }, + startEpochMs: 1_000, + }); + + expect(recordStart).toHaveBeenCalledWith( + expect.objectContaining({ + service: 'gastown', + sku: 'gastown-standard-2026-07', + idempotencyKey: 'v1:gastown:container-1:1000:start', + }) + ); }); - it('fails closed in production when the binding is absent', () => { - expect(() => getContainerUsageService(envFixture({ ENVIRONMENT: 'production' }))).toThrow( + it('fails closed when the binding is absent', () => { + expect(() => getContainerUsageClient(envFixture({ ENVIRONMENT: 'development' }))).toThrow( 'CONTAINER_USAGE binding is required' ); }); diff --git a/services/gastown/src/billing/container-usage.billing.ts b/services/gastown/src/billing/container-usage.billing.ts index 7cd99e0069c..10f7001b8b4 100644 --- a/services/gastown/src/billing/container-usage.billing.ts +++ b/services/gastown/src/billing/container-usage.billing.ts @@ -1,83 +1,22 @@ -export type UsageSubject = { type: 'user' | 'org'; id: string }; -export type UsageActor = { type: 'user' | 'bot'; id: string }; - -export type UsageContext = { +import { + createContainerUsageClient, + type BillingActor, + type BillingSubject, + type BudgetVerdict, + type ContainerUsageClient, + type ContainerUsageRpcMethods, + type UsageContext as MeterUsageContext, +} from '@kilocode/container-usage'; + +export type UsageSubject = BillingSubject; +export type UsageActor = BillingActor; +export type { BudgetVerdict }; +export type UsageContext = MeterUsageContext & { service: 'gastown'; - instanceId: string; - sku: string; - subject: UsageSubject; - actor: UsageActor; - onBehalfOf?: UsageSubject; sessionId: string; - region?: string; metadata: Record; }; -export type RecordAck = { - intervalId: string; - durable: 'pg' | 'buffer'; - dedup: boolean; -}; - -export type BudgetVerdict = { - verdict: 'continue' | 'warn' | 'stop'; - remaining?: number; -}; - -export type HeartbeatAck = RecordAck & { - budget: BudgetVerdict; -}; - -export type AuthorizeStartResult = - | { - verdict: 'allow'; - authorizationId: string; - expiresAt: number; - remaining?: number; - minimumRequired: number; - estimatedHourlyCharge?: number; - } - | { - verdict: 'deny'; - remaining: number; - minimumRequired: number; - estimatedHourlyCharge?: number; - }; - -export type ContainerUsageService = { - authorizeStart(input: { - context: UsageContext; - idempotencyKey: string; - observedAt: number; - }): Promise; - recordStart( - input: UsageContext & { - idempotencyKey: string; - startEpochMs: number; - observedAt: number; - } - ): Promise; - recordHeartbeat(input: { - service: 'gastown'; - instanceId: string; - startEpochMs: number; - idempotencyKey: string; - seq: number; - observedAt: number; - usageSinceLast?: number; - context?: UsageContext; - }): Promise; - recordStop(input: { - service: 'gastown'; - instanceId: string; - startEpochMs: number; - idempotencyKey: string; - observedAt: number; - reason: 'exit' | 'runtime_signal' | 'activity_expired'; - exitCode?: number; - }): Promise; -}; - export type GastownBillingStatus = { enabled: boolean; state: @@ -90,7 +29,7 @@ export type GastownBillingStatus = { | 'paused' | 'degraded'; runPolicy: ContainerRunPolicy; - payer?: { type: 'user' | 'org'; id: string }; + payer?: UsageSubject; remaining?: number; minimumRequired?: number; estimatedHourlyCharge?: number; @@ -102,56 +41,23 @@ export type GastownBillingStatus = { export type ContainerRunPolicy = 'automatic' | 'paused_by_user'; -export const GASTOWN_CONTAINER_SKU = 'cloudflare-container-standard-4'; +export const GASTOWN_CONTAINER_SKU = 'gastown-standard-2026-07'; export const USAGE_HEARTBEAT_INTERVAL_MS = 5 * 60_000; export function isGastownBillingEnabled(env: Env): boolean { return env.GASTOWN_BILLING_ENABLED === 'true'; } -/** - * No-charge implementation for local development while the real WorkerEntrypoint - * package is under construction. Production never falls back to this service. - */ -function createDevelopmentContainerUsageStub(): ContainerUsageService { - return { - async authorizeStart({ idempotencyKey, observedAt }) { - return { - verdict: 'allow', - authorizationId: `dev:${idempotencyKey}`, - expiresAt: observedAt + 60_000, - remaining: 100, - minimumRequired: 1, - estimatedHourlyCharge: 1.2, - }; - }, - async recordStart(input) { - return { - intervalId: `${input.instanceId}:${input.startEpochMs}`, - durable: 'buffer', - dedup: false, - }; - }, - async recordHeartbeat(input) { - return { - intervalId: `${input.instanceId}:${input.startEpochMs}`, - durable: 'buffer', - dedup: false, - budget: { verdict: 'continue', remaining: 100 }, - }; - }, - async recordStop(input) { - return { - intervalId: `${input.instanceId}:${input.startEpochMs}`, - durable: 'buffer', - dedup: false, - }; - }, - }; +export function getContainerUsageClient(env: Env): ContainerUsageClient { + if (!env.CONTAINER_USAGE) { + throw new Error('CONTAINER_USAGE binding is required when Gastown billing is enabled'); + } + return createContainerUsageClient(env.CONTAINER_USAGE, { service: 'gastown' }); } -export function getContainerUsageService(env: Env): ContainerUsageService { - if (env.CONTAINER_USAGE) return env.CONTAINER_USAGE; - if (env.ENVIRONMENT === 'development') return createDevelopmentContainerUsageStub(); - throw new Error('CONTAINER_USAGE binding is required when Gastown billing is enabled'); +export type ContainerUsageBinding = ContainerUsageRpcMethods; + +export function clientUsageContext(context: UsageContext): Omit { + const { service: _service, ...clientContext } = context; + return clientContext; } diff --git a/services/gastown/src/billing/sku-rate.billing.test.ts b/services/gastown/src/billing/sku-rate.billing.test.ts new file mode 100644 index 00000000000..e685d4d7f08 --- /dev/null +++ b/services/gastown/src/billing/sku-rate.billing.test.ts @@ -0,0 +1,12 @@ +import { describe, expect, it } from 'vitest'; +import { hourlyChargeFromRate } from './sku-rate.billing'; + +describe('hourlyChargeFromRate', () => { + it('converts SKU cents per second to dollars per hour', () => { + expect(hourlyChargeFromRate('0.033416666667')).toBeCloseTo(1.203, 6); + }); + + it.each(['0', '-1', 'not-a-number'])('rejects an invalid cents-per-second rate: %s', rate => { + expect(hourlyChargeFromRate(rate)).toBeUndefined(); + }); +}); diff --git a/services/gastown/src/billing/sku-rate.billing.ts b/services/gastown/src/billing/sku-rate.billing.ts new file mode 100644 index 00000000000..c16d0e1c9d9 --- /dev/null +++ b/services/gastown/src/billing/sku-rate.billing.ts @@ -0,0 +1,24 @@ +import { cloud_billing_sku } from '@kilocode/db/schema'; +import { getWorkerDb } from '@kilocode/db/client'; +import { eq } from 'drizzle-orm'; + +export async function getSkuHourlyCharge( + connectionString: string, + skuId: string +): Promise { + const db = getWorkerDb(connectionString, { statement_timeout: 5_000 }); + const [sku] = await db + .select({ rateCentsPerSecond: cloud_billing_sku.rate_cents_per_unit }) + .from(cloud_billing_sku) + .where(eq(cloud_billing_sku.id, skuId)) + .limit(1); + if (!sku) return undefined; + + return hourlyChargeFromRate(sku.rateCentsPerSecond); +} + +export function hourlyChargeFromRate(rate: string): number | undefined { + const rateCentsPerSecond = Number(rate); + if (!Number.isFinite(rateCentsPerSecond) || rateCentsPerSecond <= 0) return undefined; + return (rateCentsPerSecond * 3600) / 100; +} diff --git a/services/gastown/src/dos/Town.do.ts b/services/gastown/src/dos/Town.do.ts index 957f85384e5..2a6af46ff78 100644 --- a/services/gastown/src/dos/Town.do.ts +++ b/services/gastown/src/dos/Town.do.ts @@ -992,19 +992,44 @@ export class TownDO extends DurableObject { townConfig.owner_type === 'org' ? ({ type: 'org', id: ownerId } as const) : ({ type: 'user', id: ownerId } as const); + const useBotActor = !actor || (subject.type === 'user' && actor.id !== subject.id); + const billingActor = useBotActor + ? ({ type: 'bot', id: `gastown:${this.townId}` } as const) + : actor; const context: UsageContext = { service: 'gastown', instanceId: getTownContainerDoId(this.env, this.townId), sku: GASTOWN_CONTAINER_SKU, subject, - actor: actor ?? { type: 'bot', id: `gastown:${this.townId}` }, + actor: billingActor, sessionId: this.townId, - metadata: { townId: this.townId }, - ...(townConfig.created_by_user_id - ? { onBehalfOf: { type: 'user' as const, id: townConfig.created_by_user_id } } - : {}), + metadata: { + townId: this.townId, + ...(actor && useBotActor ? { triggeredByUserId: actor.id } : {}), + }, + ...(billingActor.type === 'bot' ? { onBehalfOf: subject } : {}), }; - await getTownContainerStub(this.env, this.townId).setBillingContext(context); + const container = getTownContainerStub(this.env, this.townId); + await container.setBillingContext(context); + const estimatedHourlyCharge = await this.getBillingHourlyEstimate(); + if (estimatedHourlyCharge !== undefined) { + await container.setBillingHourlyEstimate(estimatedHourlyCharge); + } + } + + private async getBillingHourlyEstimate(): Promise { + const storageKey = `billing:hourlyEstimate:${GASTOWN_CONTAINER_SKU}`; + const cached = await this.ctx.storage.get(storageKey); + if (cached !== undefined) return cached; + if (!this.env.HYPERDRIVE) return undefined; + + const { getSkuHourlyCharge } = await import('../billing/sku-rate.billing'); + const estimate = await getSkuHourlyCharge( + this.env.HYPERDRIVE.connectionString, + GASTOWN_CONTAINER_SKU + ); + if (estimate !== undefined) await this.ctx.storage.put(storageKey, estimate); + return estimate; } private async updateContainerBilling(): Promise { @@ -5938,13 +5963,9 @@ export class TownDO extends DurableObject { console.warn(`${TOWN_LOG} destroy: agent cleanup failed`, err); } - // Destroy TownContainerDO (sends SIGKILL to container process, clears state) - try { - const containerStub = getTownContainerStub(this.env, this.townId); - await containerStub.destroy(); - } catch (err) { - console.warn(`${TOWN_LOG} destroy: container cleanup failed`, err); - } + // Usage settlement must be acknowledged before container state is destroyed. + const containerStub = getTownContainerStub(this.env, this.townId); + await containerStub.destroyWithBilling(); await this.ctx.storage.deleteAlarm(); await this.ctx.storage.deleteAll(); diff --git a/services/gastown/src/dos/TownContainer.do.ts b/services/gastown/src/dos/TownContainer.do.ts index 8faa8e0ddef..0e5deef4d93 100644 --- a/services/gastown/src/dos/TownContainer.do.ts +++ b/services/gastown/src/dos/TownContainer.do.ts @@ -1,7 +1,9 @@ import { Container } from '@cloudflare/containers'; +import { ContainerUsageAdmissionError } from '@kilocode/container-usage'; import { ContainerBillingError } from '../billing/ContainerBilling.error'; import { - getContainerUsageService, + clientUsageContext, + getContainerUsageClient, isGastownBillingEnabled, USAGE_HEARTBEAT_INTERVAL_MS, type ContainerRunPolicy, @@ -9,6 +11,7 @@ import { type UsageContext, } from '../billing/container-usage.billing'; import { + createPendingStop, toBillingStatus, type OpenUsageInterval, type StoredUsageState, @@ -17,6 +20,7 @@ import { const TC_LOG = '[TownContainer.do]'; const BILLING_CONTEXT_KEY = 'billing:context'; const BILLING_STATE_KEY = 'billing:state'; +const BILLING_HOURLY_ESTIMATE_KEY = 'billing:estimatedHourlyCharge'; const RUN_POLICY_KEY = 'container:runPolicy'; /** @@ -112,7 +116,6 @@ export class TownContainerDO extends Container { await this.ctx.storage.put(BILLING_STATE_KEY, { ...state, phase: 'running', - lastReportedAt: Date.now(), }); } } catch (error) { @@ -206,6 +209,14 @@ export class TownContainerDO extends Container { } } + async setBillingHourlyEstimate(estimatedHourlyCharge: number): Promise { + await this.ctx.storage.put(BILLING_HOURLY_ESTIMATE_KEY, estimatedHourlyCharge); + const state = await this.getUsageState(); + if (state.phase !== 'idle') { + await this.ctx.storage.put(BILLING_STATE_KEY, { ...state, estimatedHourlyCharge }); + } + } + async getBillingStatus(): Promise { return toBillingStatus( isGastownBillingEnabled(this.env), @@ -323,6 +334,37 @@ export class TownContainerDO extends Container { return this.getBillingStatus(); } + async destroyWithBilling(): Promise { + if (isGastownBillingEnabled(this.env)) { + const state = await this.getUsageState(); + if (state.phase !== 'idle') { + await this.stopForBilling(); + if (!(await this.waitForRuntimeStop())) { + throw new Error('Cannot destroy Town container while graceful shutdown is still running'); + } + for (let attempt = 1; attempt <= 3; attempt += 1) { + await this.closeBillingInterval('runtime_signal'); + if ((await this.getUsageState()).phase === 'idle') break; + if (attempt < 3) await new Promise(resolve => setTimeout(resolve, attempt * 250)); + } + if ((await this.getUsageState()).phase !== 'idle') { + throw new Error('Cannot destroy Town container before usage settlement is acknowledged'); + } + } + } + await this.destroy(); + } + + private async waitForRuntimeStop(timeoutMs = 30_000): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + const state = await this.getState(); + if (state.status === 'stopped' || state.status === 'stopped_with_code') return true; + await new Promise(resolve => setTimeout(resolve, 250)); + } + return false; + } + private async ensureBillingInterval(): Promise { await this.assertAutomaticStartsEnabled(); if (!isGastownBillingEnabled(this.env)) return { phase: 'idle' }; @@ -345,6 +387,7 @@ export class TownContainerDO extends Container { 'Container billing context has not been configured' ); } + const estimatedHourlyCharge = await this.ctx.storage.get(BILLING_HOURLY_ESTIMATE_KEY); const selected = await this.ctx.storage.transaction(async transaction => { const latest = await transaction.get(BILLING_STATE_KEY); @@ -354,13 +397,12 @@ export class TownContainerDO extends Container { const starting: OpenUsageInterval = { phase: 'starting', context: latest?.context ?? context, - authorizationId: '', - authorizationKey: `${context.instanceId}:authorize:${observedAt}`, startEpochMs: observedAt, startRecorded: false, seq: 0, lastReportedAt: observedAt, reportedUsageSeconds: 0, + ...(estimatedHourlyCharge === undefined ? {} : { estimatedHourlyCharge }), }; await transaction.put(BILLING_STATE_KEY, starting); return starting; @@ -377,173 +419,26 @@ export class TownContainerDO extends Container { } private async completeBillingStart(starting: OpenUsageInterval): Promise { - const usage = getContainerUsageService(this.env); - let current = starting; - - if ( - current.authorizationId && - !current.startRecorded && - current.authorizationExpiresAt !== undefined && - current.authorizationExpiresAt <= Date.now() - ) { - const selected = await this.ctx.storage.transaction(async transaction => { - const latest = await transaction.get(BILLING_STATE_KEY); - if (!latest || latest.phase !== 'starting') return latest; - if ( - !latest.authorizationId || - latest.startRecorded || - latest.authorizationExpiresAt === undefined || - latest.authorizationExpiresAt > Date.now() - ) { - return latest; - } - - const observedAt = Date.now(); - const refreshed: OpenUsageInterval = { - ...latest, - context: { - ...latest.context, - metadata: Object.fromEntries( - Object.entries(latest.context.metadata).filter(([key]) => key !== 'authorizationId') - ), - }, - authorizationId: '', - authorizationKey: `${latest.context.instanceId}:authorize:${observedAt}`, - authorizationExpiresAt: undefined, - startEpochMs: observedAt, - lastReportedAt: observedAt, - }; - await transaction.put(BILLING_STATE_KEY, refreshed); - return refreshed; - }); - - if (!selected || selected.phase === 'idle' || selected.phase === 'stopping') { - throw new ContainerBillingError( - 'BILLING_UNAVAILABLE', - 'Container billing state changed while refreshing admission' - ); - } - current = selected; - } - - if (!current.authorizationId) { - let authorization; - try { - authorization = await usage.authorizeStart({ - context: current.context, - idempotencyKey: current.authorizationKey, - observedAt: current.startEpochMs, - }); - } catch (error) { - throw new ContainerBillingError( - 'BILLING_UNAVAILABLE', - error instanceof Error ? error.message : 'Container billing is unavailable' - ); - } - - if (authorization.verdict === 'deny') { - const denied = await this.ctx.storage.transaction(async transaction => { - const latest = await transaction.get(BILLING_STATE_KEY); - if ( - !latest || - latest.phase !== 'starting' || - latest.startEpochMs !== current.startEpochMs || - latest.authorizationKey !== current.authorizationKey - ) { - return false; - } - await transaction.put(BILLING_STATE_KEY, { - phase: 'idle', - context: latest.context, - blocked: true, - latestBudget: { verdict: 'stop', remaining: authorization.remaining }, - } satisfies StoredUsageState); - return true; - }); - if (!denied) { - throw new ContainerBillingError( - 'BILLING_UNAVAILABLE', - 'Container billing state changed during admission' - ); - } - throw new ContainerBillingError( - 'INSUFFICIENT_CREDITS', - `Gas Town needs at least $${authorization.minimumRequired.toFixed(2)} in credits to start`, - { - remaining: authorization.remaining, - minimumRequired: authorization.minimumRequired, - } - ); - } - - const authorized = await this.ctx.storage.transaction(async transaction => { - const latest = await transaction.get(BILLING_STATE_KEY); - if ( - !latest || - latest.phase !== 'starting' || - latest.startEpochMs !== current.startEpochMs || - latest.authorizationKey !== current.authorizationKey - ) { - return null; - } - const updated: OpenUsageInterval = { - ...latest, - context: { - ...latest.context, - metadata: { - ...latest.context.metadata, - authorizationId: authorization.authorizationId, - }, - }, - authorizationId: authorization.authorizationId, - authorizationExpiresAt: authorization.expiresAt, - latestBudget: { - verdict: 'continue', - ...(authorization.remaining === undefined - ? {} - : { remaining: authorization.remaining }), - }, - minimumRequired: authorization.minimumRequired, - ...(authorization.estimatedHourlyCharge === undefined - ? {} - : { estimatedHourlyCharge: authorization.estimatedHourlyCharge }), - }; - await transaction.put(BILLING_STATE_KEY, updated); - return updated; - }); - if (!authorized) { - throw new ContainerBillingError( - 'BILLING_UNAVAILABLE', - 'Container billing state changed during admission' - ); - } - current = authorized; - } - - if (current.startRecorded) return current; + if (starting.startRecorded) return starting; try { - await usage.recordStart({ - ...current.context, - idempotencyKey: `${current.context.instanceId}:start:${current.startEpochMs}`, - startEpochMs: current.startEpochMs, - observedAt: current.startEpochMs, + await getContainerUsageClient(this.env).recordStart({ + ...clientUsageContext(starting.context), + startEpochMs: starting.startEpochMs, }); } catch (error) { + const message = error instanceof Error ? error.message : 'Unable to record container start'; throw new ContainerBillingError( 'BILLING_UNAVAILABLE', - error instanceof Error ? error.message : 'Unable to record container start' + error instanceof ContainerUsageAdmissionError + ? `Container usage was rejected: ${message}` + : message ); } const recorded = await this.ctx.storage.transaction(async transaction => { const latest = await transaction.get(BILLING_STATE_KEY); - if ( - !latest || - latest.phase !== 'starting' || - latest.startEpochMs !== current.startEpochMs || - latest.authorizationId !== current.authorizationId - ) { + if (!latest || latest.phase !== 'starting' || latest.startEpochMs !== starting.startEpochMs) { return null; } const updated: OpenUsageInterval = { ...latest, startRecorded: true }; @@ -572,11 +467,11 @@ export class TownContainerDO extends Container { return { interval: latest, pending: latest.pendingHeartbeat }; } + const usageSinceLast = Math.floor(Math.max(0, observedAt - latest.lastReportedAt) / 1000); const pending = { seq: latest.seq + 1, - observedAt, - usageSinceLast: Math.max(0, (observedAt - latest.lastReportedAt) / 1000), - idempotencyKey: `${latest.context.instanceId}:hb:${latest.seq + 1}`, + observedAt: latest.lastReportedAt + usageSinceLast * 1000, + usageSinceLast, } satisfies NonNullable; const interval = { ...latest, pendingHeartbeat: pending } satisfies OpenUsageInterval; await transaction.put(BILLING_STATE_KEY, interval); @@ -588,15 +483,12 @@ export class TownContainerDO extends Container { } const { interval, pending } = selected; - const ack = await getContainerUsageService(this.env).recordHeartbeat({ - service: 'gastown', + const ack = await getContainerUsageClient(this.env).recordHeartbeat({ instanceId: interval.context.instanceId, startEpochMs: interval.startEpochMs, - idempotencyKey: pending.idempotencyKey, seq: pending.seq, - observedAt: pending.observedAt, usageSinceLast: pending.usageSinceLast, - context: interval.context, + context: clientUsageContext(interval.context), }); const latest = await this.getUsageState(); @@ -649,35 +541,44 @@ export class TownContainerDO extends Container { let stopping = selected; try { - if (!stopping.finalUsageCaptured && stopObservedAt > stopping.lastReportedAt) { + if (stopping.pendingHeartbeat) { const captured = await this.captureHeartbeat(stopping, stopObservedAt); if (captured.phase === 'idle') return; stopping = captured; - stopping = { - ...stopping, - phase: 'stopping', - stopReason, - stopObservedAt, - finalUsageCaptured: true, - }; - await this.ctx.storage.put(BILLING_STATE_KEY, stopping); } - await getContainerUsageService(this.env).recordStop({ - service: 'gastown', + const stopSelection = await this.ctx.storage.transaction(async transaction => { + const latest = await transaction.get(BILLING_STATE_KEY); + if (!latest || latest.phase === 'idle' || latest.startEpochMs !== stopping.startEpochMs) { + return null; + } + if (latest.pendingStop) return { interval: latest, stop: latest.pendingStop }; + + const pendingStop = createPendingStop(latest, stopObservedAt, stopReason); + const interval = { ...latest, pendingStop } satisfies OpenUsageInterval; + await transaction.put(BILLING_STATE_KEY, interval); + return { interval, stop: pendingStop }; + }); + if (!stopSelection) return; + stopping = stopSelection.interval; + + await getContainerUsageClient(this.env).recordStop({ instanceId: stopping.context.instanceId, startEpochMs: stopping.startEpochMs, - idempotencyKey: `${stopping.context.instanceId}:stop:${stopping.startEpochMs}`, - observedAt: stopObservedAt, - reason: stopReason, + seq: stopSelection.stop.seq, + usageSinceLast: stopSelection.stop.usageSinceLast, + reason: stopSelection.stop.reason, + context: clientUsageContext(stopping.context), }); const latest = await this.getUsageState(); if (latest.phase !== 'idle' && latest.startEpochMs === stopping.startEpochMs) { + const totalUsageSeconds = + (stopping.reportedUsageSeconds ?? 0) + stopSelection.stop.usageSinceLast; const estimatedCharge = stopping.estimatedHourlyCharge === undefined ? undefined - : ((stopping.reportedUsageSeconds ?? 0) / 3600) * stopping.estimatedHourlyCharge; + : (totalUsageSeconds / 3600) * stopping.estimatedHourlyCharge; await this.ctx.storage.put(BILLING_STATE_KEY, { phase: 'idle', context: stopping.context, @@ -686,7 +587,7 @@ export class TownContainerDO extends Container { lastRun: { startedAt: stopping.startEpochMs, stoppedAt: stopObservedAt, - usageSeconds: stopping.reportedUsageSeconds ?? 0, + usageSeconds: totalUsageSeconds, ...(estimatedCharge === undefined ? {} : { estimatedCharge }), }, } satisfies StoredUsageState); diff --git a/services/gastown/worker-configuration.d.ts b/services/gastown/worker-configuration.d.ts index d4567090434..b973331ddcd 100644 --- a/services/gastown/worker-configuration.d.ts +++ b/services/gastown/worker-configuration.d.ts @@ -57,7 +57,7 @@ type WastelandService = { evidence: string; }): Promise>; }; -type ContainerUsageService = import("./src/billing/container-usage.billing").ContainerUsageService; +type ContainerUsageService = import("@kilocode/container-usage").ContainerUsageRpcMethods; declare namespace Cloudflare { interface GlobalProps { mainModule: typeof import("./src/gastown.worker"); diff --git a/services/gastown/wrangler.jsonc b/services/gastown/wrangler.jsonc index c8559239372..9ed9af2f07a 100644 --- a/services/gastown/wrangler.jsonc +++ b/services/gastown/wrangler.jsonc @@ -105,6 +105,11 @@ "service": "wasteland", "entrypoint": "WastelandRPCEntrypoint", }, + { + "binding": "CONTAINER_USAGE", + "service": "container-usage-meter", + "entrypoint": "ContainerUsageMeter", + }, ], // PRODUCTION Secrets Store (shared Kilo secrets store) @@ -170,6 +175,11 @@ "service": "wasteland-dev", "entrypoint": "WastelandRPCEntrypoint", }, + { + "binding": "CONTAINER_USAGE", + "service": "container-usage-meter", + "entrypoint": "ContainerUsageMeter", + }, ], "ai": { "binding": "AI" }, From 9d6fd3d33ce88ad25c8e4493aafe79c67d92d412 Mon Sep 17 00:00:00 2001 From: John Fawcett Date: Thu, 23 Jul 2026 14:19:38 -0500 Subject: [PATCH 05/10] chore(workers): upgrade wrangler --- pnpm-lock.yaml | 174 ++++++++++++++++---- pnpm-workspace.yaml | 2 +- services/container-usage-meter/package.json | 1 + 3 files changed, 143 insertions(+), 34 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 89ddc5d761b..8ca84ae3293 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -79,8 +79,8 @@ catalogs: specifier: 1.0.0 version: 1.0.0 wrangler: - specifier: 4.98.0 - version: 4.98.0 + specifier: 4.112.0 + version: 4.112.0 zod: specifier: 4.4.3 version: 4.4.3 @@ -1565,7 +1565,7 @@ importers: version: 5.9.3 wrangler: specifier: 'catalog:' - version: 4.98.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) + version: 4.112.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) services/app-builder: dependencies: @@ -1623,7 +1623,7 @@ importers: version: 4.1.6(@opentelemetry/api@1.9.1)(@types/node@24.12.4)(@vitest/coverage-v8@4.1.6)(@vitest/ui@4.1.6)(esbuild@0.27.4)(jiti@2.7.0)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.4) wrangler: specifier: 'catalog:' - version: 4.98.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) + version: 4.112.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) services/auto-fix-infra: dependencies: @@ -1651,7 +1651,7 @@ importers: version: 4.1.6(@opentelemetry/api@1.9.1)(@types/node@24.12.4)(@vitest/coverage-v8@4.1.6)(@vitest/ui@4.1.6)(esbuild@0.27.4)(jiti@2.7.0)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.4) wrangler: specifier: 'catalog:' - version: 4.98.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) + version: 4.112.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) services/auto-routing: dependencies: @@ -1694,7 +1694,7 @@ importers: version: 4.1.6(@opentelemetry/api@1.9.1)(@types/node@24.12.4)(@vitest/coverage-v8@4.1.6)(@vitest/ui@4.1.6)(esbuild@0.27.4)(jiti@2.7.0)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.4) wrangler: specifier: 'catalog:' - version: 4.98.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) + version: 4.112.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) services/auto-routing-benchmark: dependencies: @@ -1740,7 +1740,7 @@ importers: version: 4.1.6(@opentelemetry/api@1.9.1)(@types/node@24.12.4)(@vitest/coverage-v8@4.1.6)(@vitest/ui@4.1.6)(esbuild@0.27.4)(jiti@2.7.0)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.4) wrangler: specifier: 'catalog:' - version: 4.98.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) + version: 4.112.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) services/auto-triage-infra: dependencies: @@ -1771,7 +1771,7 @@ importers: version: 4.1.6(@opentelemetry/api@1.9.1)(@types/node@24.12.4)(@vitest/coverage-v8@4.1.6)(@vitest/ui@4.1.6)(esbuild@0.27.4)(jiti@2.7.0)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.4) wrangler: specifier: 'catalog:' - version: 4.98.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) + version: 4.112.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) services/cloud-agent-next: dependencies: @@ -1862,7 +1862,7 @@ importers: version: 4.1.6(@opentelemetry/api@1.9.1)(@types/node@24.12.4)(@vitest/coverage-v8@4.1.6)(@vitest/ui@4.1.6)(esbuild@0.27.4)(jiti@2.7.0)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.4) wrangler: specifier: 'catalog:' - version: 4.98.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) + version: 4.112.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) ws: specifier: 8.21.0 version: 8.21.0(bufferutil@4.1.0)(utf-8-validate@6.0.6) @@ -1921,7 +1921,7 @@ importers: version: 4.1.6(@opentelemetry/api@1.9.1)(@types/node@24.12.4)(@vitest/coverage-v8@4.1.6)(@vitest/ui@4.1.6)(esbuild@0.27.4)(jiti@2.7.0)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.4) wrangler: specifier: 'catalog:' - version: 4.98.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) + version: 4.112.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) services/container-usage-meter: dependencies: @@ -1955,7 +1955,7 @@ importers: version: 4.1.6(@opentelemetry/api@1.9.1)(@types/node@24.12.4)(@vitest/coverage-v8@4.1.6)(@vitest/ui@4.1.6)(esbuild@0.27.4)(jiti@2.7.0)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.4) wrangler: specifier: 'catalog:' - version: 4.98.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) + version: 4.112.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) services/db-proxy: dependencies: @@ -1998,7 +1998,7 @@ importers: version: 5.9.3 wrangler: specifier: 'catalog:' - version: 4.98.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) + version: 4.112.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) services/deploy-infra/builder: dependencies: @@ -2062,7 +2062,7 @@ importers: version: 5.9.3 wrangler: specifier: 'catalog:' - version: 4.98.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) + version: 4.112.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) services/deploy-infra/dispatcher: dependencies: @@ -2111,7 +2111,7 @@ importers: version: 4.1.6(@opentelemetry/api@1.9.1)(@types/node@25.5.2)(@vitest/coverage-v8@4.1.6)(@vitest/ui@4.1.6)(esbuild@0.27.4)(jiti@2.7.0)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.4) wrangler: specifier: 'catalog:' - version: 4.98.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) + version: 4.112.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) services/event-service: dependencies: @@ -2154,7 +2154,7 @@ importers: version: 4.1.6(@opentelemetry/api@1.9.1)(@types/node@24.12.4)(@vitest/coverage-v8@4.1.6)(@vitest/ui@4.1.6)(esbuild@0.27.4)(jiti@2.7.0)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.4) wrangler: specifier: 'catalog:' - version: 4.98.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) + version: 4.112.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) services/gastown: dependencies: @@ -2230,7 +2230,7 @@ importers: version: 4.1.6(@opentelemetry/api@1.9.1)(@types/node@24.12.4)(@vitest/coverage-v8@4.1.6)(@vitest/ui@4.1.6)(esbuild@0.27.4)(jiti@2.7.0)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.4) wrangler: specifier: 'catalog:' - version: 4.98.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) + version: 4.112.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) services/gastown/container: dependencies: @@ -2295,7 +2295,7 @@ importers: version: 4.1.6(@opentelemetry/api@1.9.1)(@types/node@24.12.4)(@vitest/coverage-v8@4.1.6)(@vitest/ui@4.1.6)(esbuild@0.27.4)(jiti@2.7.0)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.4) wrangler: specifier: 'catalog:' - version: 4.98.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) + version: 4.112.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) services/gmail-push: dependencies: @@ -2323,7 +2323,7 @@ importers: version: 4.1.6(@opentelemetry/api@1.9.1)(@types/node@25.5.2)(@vitest/coverage-v8@4.1.6)(@vitest/ui@4.1.6)(esbuild@0.27.4)(jiti@2.7.0)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.4) wrangler: specifier: 'catalog:' - version: 4.98.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) + version: 4.112.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) services/images-mcp: dependencies: @@ -2366,7 +2366,7 @@ importers: version: 4.1.6(@opentelemetry/api@1.9.1)(@types/node@24.12.4)(@vitest/coverage-v8@4.1.6)(@vitest/ui@4.1.6)(esbuild@0.27.4)(jiti@2.7.0)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.4) wrangler: specifier: 'catalog:' - version: 4.98.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) + version: 4.112.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) services/kilo-chat: dependencies: @@ -2439,7 +2439,7 @@ importers: version: 4.1.6(@opentelemetry/api@1.9.1)(@types/node@24.12.4)(@vitest/coverage-v8@4.1.6)(@vitest/ui@4.1.6)(esbuild@0.27.4)(jiti@2.7.0)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.4) wrangler: specifier: 'catalog:' - version: 4.98.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) + version: 4.112.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) services/kilo-ops: dependencies: @@ -2464,7 +2464,7 @@ importers: version: 5.9.3 wrangler: specifier: 'catalog:' - version: 4.98.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) + version: 4.112.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) services/kiloclaw: dependencies: @@ -2525,7 +2525,7 @@ importers: version: 4.1.6(@opentelemetry/api@1.9.1)(@types/node@24.12.4)(@vitest/coverage-v8@4.1.6)(@vitest/ui@4.1.6)(esbuild@0.27.4)(jiti@2.7.0)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.4) wrangler: specifier: 'catalog:' - version: 4.98.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) + version: 4.112.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) services/kiloclaw-billing: dependencies: @@ -2568,7 +2568,7 @@ importers: version: 4.1.6(@opentelemetry/api@1.9.1)(@types/node@24.12.4)(@vitest/coverage-v8@4.1.6)(@vitest/ui@4.1.6)(esbuild@0.27.4)(jiti@2.7.0)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.4) wrangler: specifier: 'catalog:' - version: 4.98.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) + version: 4.112.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) services/kiloclaw-inbound-email: dependencies: @@ -2596,7 +2596,7 @@ importers: version: 4.1.6(@opentelemetry/api@1.9.1)(@types/node@25.5.2)(@vitest/coverage-v8@4.1.6)(@vitest/ui@4.1.6)(esbuild@0.27.4)(jiti@2.7.0)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.4) wrangler: specifier: 'catalog:' - version: 4.98.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) + version: 4.112.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) services/kiloclaw/controller: dependencies: @@ -2710,7 +2710,7 @@ importers: version: 4.1.6(@opentelemetry/api@1.9.1)(@types/node@24.12.4)(@vitest/coverage-v8@4.1.6)(@vitest/ui@4.1.6)(esbuild@0.27.4)(jiti@2.7.0)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.4) wrangler: specifier: 'catalog:' - version: 4.98.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) + version: 4.112.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) services/model-eval-ingest: dependencies: @@ -2741,7 +2741,7 @@ importers: version: 4.1.6(@opentelemetry/api@1.9.1)(@types/node@24.12.4)(@vitest/coverage-v8@4.1.6)(@vitest/ui@4.1.6)(esbuild@0.27.4)(jiti@2.7.0)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.4) wrangler: specifier: 'catalog:' - version: 4.98.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) + version: 4.112.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) services/notifications: dependencies: @@ -2793,7 +2793,7 @@ importers: version: 4.1.6(@opentelemetry/api@1.9.1)(@types/node@24.12.4)(@vitest/coverage-v8@4.1.6)(@vitest/ui@4.1.6)(esbuild@0.27.4)(jiti@2.7.0)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.4) wrangler: specifier: 'catalog:' - version: 4.98.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) + version: 4.112.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) services/o11y: dependencies: @@ -2830,7 +2830,7 @@ importers: version: 4.1.6(@opentelemetry/api@1.9.1)(@types/node@24.12.4)(@vitest/coverage-v8@4.1.6)(@vitest/ui@4.1.6)(esbuild@0.27.4)(jiti@2.7.0)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.4) wrangler: specifier: 'catalog:' - version: 4.98.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) + version: 4.112.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) services/security-auto-analysis: dependencies: @@ -2864,7 +2864,7 @@ importers: version: 4.1.6(@opentelemetry/api@1.9.1)(@types/node@24.12.4)(@vitest/coverage-v8@4.1.6)(@vitest/ui@4.1.6)(esbuild@0.27.4)(jiti@2.7.0)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.4) wrangler: specifier: 'catalog:' - version: 4.98.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) + version: 4.112.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) services/security-sync: dependencies: @@ -2895,7 +2895,7 @@ importers: version: 4.1.6(@opentelemetry/api@1.9.1)(@types/node@24.12.4)(@vitest/coverage-v8@4.1.6)(@vitest/ui@4.1.6)(esbuild@0.27.4)(jiti@2.7.0)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.4) wrangler: specifier: 'catalog:' - version: 4.98.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) + version: 4.112.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) services/session-ingest: dependencies: @@ -2950,7 +2950,7 @@ importers: version: 4.1.6(@opentelemetry/api@1.9.1)(@types/node@24.12.4)(@vitest/coverage-v8@4.1.6)(@vitest/ui@4.1.6)(esbuild@0.27.4)(jiti@2.7.0)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.4) wrangler: specifier: 'catalog:' - version: 4.98.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) + version: 4.112.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) services/wasteland: dependencies: @@ -3008,7 +3008,7 @@ importers: version: 4.1.6(@opentelemetry/api@1.9.1)(@types/node@22.19.19)(@vitest/coverage-v8@4.1.6)(@vitest/ui@4.1.6)(esbuild@0.27.4)(jiti@2.7.0)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.4) wrangler: specifier: 'catalog:' - version: 4.98.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) + version: 4.112.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) services/webhook-agent-ingest: dependencies: @@ -3063,7 +3063,7 @@ importers: version: 4.1.6(@opentelemetry/api@1.9.1)(@types/node@24.12.4)(@vitest/coverage-v8@4.1.6)(@vitest/ui@4.1.6)(esbuild@0.27.4)(jiti@2.7.0)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.4) wrangler: specifier: 'catalog:' - version: 4.98.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) + version: 4.112.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6) packages: @@ -4243,30 +4243,60 @@ packages: cpu: [x64] os: [darwin] + '@cloudflare/workerd-darwin-64@1.20260714.1': + resolution: {integrity: sha512-ZWXqAN8G7Cx9hMRQuk+59ziJhR3j1F4iO+Qs8aHdfKZ3Dq5Yi/57xvkJTgCGBnW1YU/L78r8f6HEy51bwbTpNw==} + engines: {node: '>=16'} + cpu: [x64] + os: [darwin] + '@cloudflare/workerd-darwin-arm64@1.20260603.1': resolution: {integrity: sha512-uBPK4LaWJNbbCYwPnUAehlHbbVulhVZPZsdcAhBPfZhHb3QAuAEPAQepO/P67R3V6Cni4YGx1fLbL8A5wwoaNA==} engines: {node: '>=16'} cpu: [arm64] os: [darwin] + '@cloudflare/workerd-darwin-arm64@1.20260714.1': + resolution: {integrity: sha512-tueWxWC3wyCbMG6zRAxsMXX0YLgrRWbiAPYFQ2uJ7dUH8G+5E7UTWaQS9B1HdJ0bpKFW1NWxhs1o2noKVFSUYg==} + engines: {node: '>=16'} + cpu: [arm64] + os: [darwin] + '@cloudflare/workerd-linux-64@1.20260603.1': resolution: {integrity: sha512-ht9l6/8Tk7Rp6kA4S9oFZ4X8u0VjnnFdmU/6B3fnABYKREYTKh2RdOqXqXxcp5eNJseireKnWik/hQOPK1CutQ==} engines: {node: '>=16'} cpu: [x64] os: [linux] + '@cloudflare/workerd-linux-64@1.20260714.1': + resolution: {integrity: sha512-1VChTZRb0l0F7R4e1G5RtLKV4oFi6x+rQgxh2+yu887j3l/3TLgatuv1L8/5zhc9gKEhATTxOh0e52Rtd9dDWQ==} + engines: {node: '>=16'} + cpu: [x64] + os: [linux] + '@cloudflare/workerd-linux-arm64@1.20260603.1': resolution: {integrity: sha512-LJZ6x00rAjSrobV4m0ZW0TpH5ilBbKcWBzlH+y+KOUsIE/CpTuhAzKV43TbSnFLRX5+jrWKiz2v0hO91lPXy6A==} engines: {node: '>=16'} cpu: [arm64] os: [linux] + '@cloudflare/workerd-linux-arm64@1.20260714.1': + resolution: {integrity: sha512-rMm3G+NirG2UdgHIRDdF1asNC6FqgIzZzkRG+VDhhDGcVxAQwvrMT1E38BivEvHr3G04MB4AfhcOczX0+GtRkQ==} + engines: {node: '>=16'} + cpu: [arm64] + os: [linux] + '@cloudflare/workerd-windows-64@1.20260603.1': resolution: {integrity: sha512-DvwqkXMAJRPoDN4PxapAwhlz/6ouD+6R1ttbAEK3cWD/QBvFF5STx7Ds/9Irf+rBly3np3uHWkeX+wZnNFEuzA==} engines: {node: '>=16'} cpu: [x64] os: [win32] + '@cloudflare/workerd-windows-64@1.20260714.1': + resolution: {integrity: sha512-cGqnU3Hg2YZS/k3SAqrMp1DjpdsyFde72tWltdl6ZT9+SFz/Zrk/8gyTU1TcxC4YApXeNVH5TyU5cOGPgUJ0pg==} + engines: {node: '>=16'} + cpu: [x64] + os: [win32] + '@cloudflare/workers-types@4.20260605.1': resolution: {integrity: sha512-9YJaGPQwuQYFHoElVhJP40ZmUO/Z2OiBon58sKpHjgwq5btC/B2BZLC45AQ14k+1I+hjcY2z2t2R6uUxU9AqwQ==} @@ -14574,6 +14604,11 @@ packages: engines: {node: '>=22.0.0'} hasBin: true + miniflare@4.20260714.0: + resolution: {integrity: sha512-MYlTCLdWCPqvrYY2uLwOjXwmglXuiHE3TGGkbOW4BwjUPa1r07E0iuHwrNDIs/sxK21r+o90Jx58AV2KeNdJZw==} + engines: {node: '>=22.0.0'} + hasBin: true + minimalistic-assert@1.0.1: resolution: {integrity: sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A==} @@ -17792,9 +17827,24 @@ packages: engines: {node: '>=16'} hasBin: true + workerd@1.20260714.1: + resolution: {integrity: sha512-oIbQzfdyl9UQUnG6XLegcSq0Mgt/7WKDbFOoqGgOWCS+/fhyGB460uKEgdAQQ9RHCO/ttcNCX/KiMIQzdoeu3Q==} + engines: {node: '>=16'} + hasBin: true + workers-tagged-logger@1.0.0: resolution: {integrity: sha512-tp5PAs48hSpF2GIbH0S186MBXuMe8u9uuHZR0Jmw/I8+NIiQblor5EpYJ5lOaE8u9s84mVme6Iv+ueC1C/beog==} + wrangler@4.112.0: + resolution: {integrity: sha512-5H+XUD0TySCv1LuktFHDIEOkboH2nTfQs+35L+USt3MtntjDTMVIJprLgQcL2WBjulOyjxpd1vyTiSTJVW5MjQ==} + engines: {node: '>=22.0.0'} + hasBin: true + peerDependencies: + '@cloudflare/workers-types': ^5.20260714.1 + peerDependenciesMeta: + '@cloudflare/workers-types': + optional: true + wrangler@4.98.0: resolution: {integrity: sha512-cXfFUuF4rMIvE0hiMnXjEAB27ERryaCgquBJdUoPIjFzYYE1rbRdMUkEdQ18qDPUtsPvhJdqxLntixT9OfSzQw==} engines: {node: '>=22.0.0'} @@ -19760,6 +19810,12 @@ snapshots: optionalDependencies: workerd: 1.20260603.1 + '@cloudflare/unenv-preset@2.16.1(unenv@2.0.0-rc.24)(workerd@1.20260714.1)': + dependencies: + unenv: 2.0.0-rc.24 + optionalDependencies: + workerd: 1.20260714.1 + '@cloudflare/vitest-pool-workers@0.16.13(@cloudflare/workers-types@4.20260605.1)(@vitest/runner@4.1.6)(@vitest/snapshot@4.1.6)(bufferutil@4.1.0)(utf-8-validate@6.0.6)(vitest@4.1.6)': dependencies: '@vitest/runner': 4.1.6 @@ -19778,18 +19834,33 @@ snapshots: '@cloudflare/workerd-darwin-64@1.20260603.1': optional: true + '@cloudflare/workerd-darwin-64@1.20260714.1': + optional: true + '@cloudflare/workerd-darwin-arm64@1.20260603.1': optional: true + '@cloudflare/workerd-darwin-arm64@1.20260714.1': + optional: true + '@cloudflare/workerd-linux-64@1.20260603.1': optional: true + '@cloudflare/workerd-linux-64@1.20260714.1': + optional: true + '@cloudflare/workerd-linux-arm64@1.20260603.1': optional: true + '@cloudflare/workerd-linux-arm64@1.20260714.1': + optional: true + '@cloudflare/workerd-windows-64@1.20260603.1': optional: true + '@cloudflare/workerd-windows-64@1.20260714.1': + optional: true + '@cloudflare/workers-types@4.20260605.1': {} '@codemirror/autocomplete@6.20.1': @@ -32247,6 +32318,18 @@ snapshots: - bufferutil - utf-8-validate + miniflare@4.20260714.0(bufferutil@4.1.0)(utf-8-validate@6.0.6): + dependencies: + '@cspotcode/source-map-support': 0.8.1 + sharp: 0.34.5 + undici: 7.28.0 + workerd: 1.20260714.1 + ws: 8.21.0(bufferutil@4.1.0)(utf-8-validate@6.0.6) + youch: 4.1.0-beta.10 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + minimalistic-assert@1.0.1: {} minimalistic-crypto-utils@1.0.1: {} @@ -36300,12 +36383,37 @@ snapshots: '@cloudflare/workerd-linux-arm64': 1.20260603.1 '@cloudflare/workerd-windows-64': 1.20260603.1 + workerd@1.20260714.1: + optionalDependencies: + '@cloudflare/workerd-darwin-64': 1.20260714.1 + '@cloudflare/workerd-darwin-arm64': 1.20260714.1 + '@cloudflare/workerd-linux-64': 1.20260714.1 + '@cloudflare/workerd-linux-arm64': 1.20260714.1 + '@cloudflare/workerd-windows-64': 1.20260714.1 + workers-tagged-logger@1.0.0: dependencies: zod: 4.4.3 optionalDependencies: hono: 4.12.25 + wrangler@4.112.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6): + dependencies: + '@cloudflare/kv-asset-handler': 0.5.0 + '@cloudflare/unenv-preset': 2.16.1(unenv@2.0.0-rc.24)(workerd@1.20260714.1) + blake3-wasm: 2.1.5 + esbuild: 0.27.4 + miniflare: 4.20260714.0(bufferutil@4.1.0)(utf-8-validate@6.0.6) + path-to-regexp: 8.4.2 + unenv: 2.0.0-rc.24 + workerd: 1.20260714.1 + optionalDependencies: + '@cloudflare/workers-types': 4.20260605.1 + fsevents: 2.3.3 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + wrangler@4.98.0(@cloudflare/workers-types@4.20260605.1)(bufferutil@4.1.0)(utf-8-validate@6.0.6): dependencies: '@cloudflare/kv-asset-handler': 0.5.0 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 96c20040fe4..5e844cb1cfc 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -37,7 +37,7 @@ catalog: ulid: 3.0.2 vitest: 4.1.6 workers-tagged-logger: 1.0.0 - wrangler: 4.98.0 + wrangler: 4.112.0 zod: 4.4.3 minimumReleaseAge: 6842 diff --git a/services/container-usage-meter/package.json b/services/container-usage-meter/package.json index c8db93e5e1c..103c7c9ba9c 100644 --- a/services/container-usage-meter/package.json +++ b/services/container-usage-meter/package.json @@ -9,6 +9,7 @@ "typecheck": "tsgo --noEmit", "lint": "pnpm -w exec oxlint --config .oxlintrc.json services/container-usage-meter/src", "test": "vitest run", + "wrangler": "wrangler", "test:postgres": "vitest run --config vitest.postgres.config.ts" }, "dependencies": { From 6d0ad745312a9ab83416347a9e8ca7102d8a84f0 Mon Sep 17 00:00:00 2001 From: John Fawcett Date: Thu, 23 Jul 2026 14:20:28 -0500 Subject: [PATCH 06/10] fix(gastown): recover billing state and polish terminal --- .specs/gastown-usage-based-billing.md | 6 ++- ENVIRONMENT.md | 4 +- .../src/components/gastown/TerminalBar.tsx | 40 +++++++++++++-- apps/web/src/lib/config.server.ts | 1 + .../container-usage-state.billing.test.ts | 51 +++++++++++++++++++ .../billing/container-usage-state.billing.ts | 25 +++++++++ .../billing/container-usage.billing.test.ts | 13 +++++ .../src/billing/container-usage.billing.ts | 8 +++ services/gastown/src/dos/TownContainer.do.ts | 50 +++++++++++++++--- services/gastown/wrangler.jsonc | 9 +++- 10 files changed, 192 insertions(+), 15 deletions(-) diff --git a/.specs/gastown-usage-based-billing.md b/.specs/gastown-usage-based-billing.md index 698711ece53..03b7c0493a3 100644 --- a/.specs/gastown-usage-based-billing.md +++ b/.specs/gastown-usage-based-billing.md @@ -423,6 +423,9 @@ primary CTA. Amounts and usage counters SHOULD use tabular numerals. 3. If `recordStop` cannot be acknowledged during shutdown, the pending stop MUST survive and retry later. A missing stop ack MUST NOT cause the next runtime to reuse the previous interval. 4. Metering errors and retries MUST NOT log credit balances, tokens, or other credentials. +5. If the meter reports that a locally open interval is missing, Gastown MUST replay the same + idempotent `recordStart` and retry the heartbeat or stop. Pre-WorkerEntrypoint development state + MUST be version-migrated so obsolete authorization and credit-block fields cannot strand a town. ## Rollout And Acceptance @@ -447,7 +450,8 @@ Cloudflare runtime observations, duplicate RPCs do not duplicate charges, and ag reasonably consistent with the Cloudflare invoice. `GASTOWN_BILLING_ENABLED` controls metering, admission, debits, warnings, and stops together. A separate default-off `GASTOWN_BILLING_ANNOUNCEMENT_ENABLED` web flag MAY show advance notice while actual billing remains -disabled; it MUST NOT alter backend billing behavior. +disabled; it MUST NOT alter backend billing behavior. Both flags are always enabled in local +development while retaining default-off production behavior. Dashboards MUST expose awake seconds, base Cloudflare cost, customer charge, gross multiplier, unclosed intervals, SKU admission denials, RPC failures, graceful-stop duration, forced stops, and diff --git a/ENVIRONMENT.md b/ENVIRONMENT.md index 20d8d46ca35..2a93f24ca67 100644 --- a/ENVIRONMENT.md +++ b/ENVIRONMENT.md @@ -205,8 +205,8 @@ Manage shared web env var additions and rotations with `pnpm web:env set {allTabs.map(tab => { @@ -694,7 +703,28 @@ function TabBar({ }`} > - {billingSummary} + {showRunCostTooltip ? ( + + + + + +

Estimated container cost

+

+ This estimate accrues only while the Gas Town container is running. Model and + token usage is billed separately. The final amount may adjust after the latest + meter interval settles. +

+ + + ) : ( + billingSummary + )} {billing.state === 'warning' && ( { }); }); }); + +describe('migrateStoredUsageState', () => { + it('resets hypothetical authorization state that predates the real meter', () => { + const legacy = { + phase: 'stopping', + context, + authorizationId: 'dev:old', + authorizationKey: 'old-key', + startEpochMs: 1_000, + startRecorded: true, + seq: 1, + lastReportedAt: 2_000, + latestBudget: { verdict: 'stop', remaining: 0 }, + } as StoredUsageState; + + expect(migrateStoredUsageState(legacy)).toEqual({ + version: BILLING_STATE_VERSION, + phase: 'idle', + context, + }); + }); + + it('preserves an unversioned real-meter interval', () => { + const current: StoredUsageState = { + phase: 'running', + context, + startEpochMs: 1_000, + startRecorded: true, + seq: 1, + lastReportedAt: 2_000, + }; + + expect(migrateStoredUsageState(current)).toEqual({ + ...current, + version: BILLING_STATE_VERSION, + }); + }); + + it('clears an unversioned stale credit block', () => { + expect( + migrateStoredUsageState({ + phase: 'idle', + context, + blocked: true, + latestBudget: { verdict: 'stop', remaining: 0 }, + }) + ).toEqual({ version: BILLING_STATE_VERSION, phase: 'idle', context }); + }); +}); diff --git a/services/gastown/src/billing/container-usage-state.billing.ts b/services/gastown/src/billing/container-usage-state.billing.ts index 7ebd05586cd..a1a9b1cc253 100644 --- a/services/gastown/src/billing/container-usage-state.billing.ts +++ b/services/gastown/src/billing/container-usage-state.billing.ts @@ -19,6 +19,7 @@ export type PendingStop = { }; export type OpenUsageInterval = { + version?: typeof BILLING_STATE_VERSION; phase: 'starting' | 'running' | 'stopping'; context: UsageContext; startEpochMs: number; @@ -38,6 +39,7 @@ export type OpenUsageInterval = { export type StoredUsageState = | { phase: 'idle'; + version?: typeof BILLING_STATE_VERSION; context?: UsageContext; blocked?: boolean; latestBudget?: BudgetVerdict; @@ -50,6 +52,29 @@ export type StoredUsageState = } | OpenUsageInterval; +export const BILLING_STATE_VERSION = 2 as const; + +export function migrateStoredUsageState( + state: StoredUsageState | undefined, + fallbackContext?: UsageContext +): StoredUsageState { + if (state?.version === BILLING_STATE_VERSION) return state; + const context = state?.context ?? fallbackContext; + const hasLegacyAuthorization = + state !== undefined && + ('authorizationId' in state || + 'authorizationKey' in state || + 'authorizationExpiresAt' in state); + if (state && !hasLegacyAuthorization && !(state.phase === 'idle' && state.blocked)) { + return { ...state, version: BILLING_STATE_VERSION }; + } + return { + version: BILLING_STATE_VERSION, + phase: 'idle', + ...(context ? { context } : {}), + }; +} + export function createPendingStop( state: OpenUsageInterval, stopObservedAt: number, diff --git a/services/gastown/src/billing/container-usage.billing.test.ts b/services/gastown/src/billing/container-usage.billing.test.ts index 94af586fa97..82160dfd421 100644 --- a/services/gastown/src/billing/container-usage.billing.test.ts +++ b/services/gastown/src/billing/container-usage.billing.test.ts @@ -4,6 +4,7 @@ import { GASTOWN_CONTAINER_SKU, getContainerUsageClient, isGastownBillingEnabled, + isUsageIntervalNotFoundError, } from './container-usage.billing'; function envFixture(overrides: Partial): Env { @@ -56,4 +57,16 @@ describe('Gastown billing configuration', () => { 'CONTAINER_USAGE binding is required' ); }); + + it('recognizes a missing remote interval across the RPC error boundary', () => { + const namedError = new Error('missing'); + namedError.name = 'UsageIntervalNotFoundError'; + expect(isUsageIntervalNotFoundError(namedError)).toBe(true); + expect( + isUsageIntervalNotFoundError( + new Error('Container usage interval not found: gastown:instance:start') + ) + ).toBe(true); + expect(isUsageIntervalNotFoundError(new Error('Postgres unavailable'))).toBe(false); + }); }); diff --git a/services/gastown/src/billing/container-usage.billing.ts b/services/gastown/src/billing/container-usage.billing.ts index 10f7001b8b4..ed9923b90a9 100644 --- a/services/gastown/src/billing/container-usage.billing.ts +++ b/services/gastown/src/billing/container-usage.billing.ts @@ -55,6 +55,14 @@ export function getContainerUsageClient(env: Env): ContainerUsageClient { return createContainerUsageClient(env.CONTAINER_USAGE, { service: 'gastown' }); } +export function isUsageIntervalNotFoundError(error: unknown): boolean { + return ( + error instanceof Error && + (error.name === 'UsageIntervalNotFoundError' || + error.message.includes('Container usage interval not found:')) + ); +} + export type ContainerUsageBinding = ContainerUsageRpcMethods; export function clientUsageContext(context: UsageContext): Omit { diff --git a/services/gastown/src/dos/TownContainer.do.ts b/services/gastown/src/dos/TownContainer.do.ts index 0e5deef4d93..e7b697cacdf 100644 --- a/services/gastown/src/dos/TownContainer.do.ts +++ b/services/gastown/src/dos/TownContainer.do.ts @@ -1,10 +1,11 @@ import { Container } from '@cloudflare/containers'; -import { ContainerUsageAdmissionError } from '@kilocode/container-usage'; +import { ContainerUsageAdmissionError, type HeartbeatAck } from '@kilocode/container-usage'; import { ContainerBillingError } from '../billing/ContainerBilling.error'; import { clientUsageContext, getContainerUsageClient, isGastownBillingEnabled, + isUsageIntervalNotFoundError, USAGE_HEARTBEAT_INTERVAL_MS, type ContainerRunPolicy, type GastownBillingStatus, @@ -12,6 +13,8 @@ import { } from '../billing/container-usage.billing'; import { createPendingStop, + BILLING_STATE_VERSION, + migrateStoredUsageState, toBillingStatus, type OpenUsageInterval, type StoredUsageState, @@ -395,6 +398,7 @@ export class TownContainerDO extends Container { const observedAt = Date.now(); const starting: OpenUsageInterval = { + version: BILLING_STATE_VERSION, phase: 'starting', context: latest?.context ?? context, startEpochMs: observedAt, @@ -483,13 +487,22 @@ export class TownContainerDO extends Container { } const { interval, pending } = selected; - const ack = await getContainerUsageClient(this.env).recordHeartbeat({ + const heartbeatInput = { instanceId: interval.context.instanceId, startEpochMs: interval.startEpochMs, seq: pending.seq, usageSinceLast: pending.usageSinceLast, context: clientUsageContext(interval.context), - }); + }; + const usageClient = getContainerUsageClient(this.env); + let ack: HeartbeatAck; + try { + ack = await usageClient.recordHeartbeat(heartbeatInput); + } catch (error) { + if (!isUsageIntervalNotFoundError(error)) throw error; + await this.restoreRemoteInterval(interval); + ack = await usageClient.recordHeartbeat(heartbeatInput); + } const latest = await this.getUsageState(); if (latest.phase === 'idle' || latest.startEpochMs !== interval.startEpochMs) return latest; @@ -562,14 +575,22 @@ export class TownContainerDO extends Container { if (!stopSelection) return; stopping = stopSelection.interval; - await getContainerUsageClient(this.env).recordStop({ + const stopInput = { instanceId: stopping.context.instanceId, startEpochMs: stopping.startEpochMs, seq: stopSelection.stop.seq, usageSinceLast: stopSelection.stop.usageSinceLast, reason: stopSelection.stop.reason, context: clientUsageContext(stopping.context), - }); + }; + const usageClient = getContainerUsageClient(this.env); + try { + await usageClient.recordStop(stopInput); + } catch (error) { + if (!isUsageIntervalNotFoundError(error)) throw error; + await this.restoreRemoteInterval(stopping); + await usageClient.recordStop(stopInput); + } const latest = await this.getUsageState(); if (latest.phase !== 'idle' && latest.startEpochMs === stopping.startEpochMs) { @@ -580,6 +601,7 @@ export class TownContainerDO extends Container { ? undefined : (totalUsageSeconds / 3600) * stopping.estimatedHourlyCharge; await this.ctx.storage.put(BILLING_STATE_KEY, { + version: BILLING_STATE_VERSION, phase: 'idle', context: stopping.context, blocked: stopping.latestBudget?.verdict === 'stop', @@ -598,7 +620,23 @@ export class TownContainerDO extends Container { } private async getUsageState(): Promise { - return (await this.ctx.storage.get(BILLING_STATE_KEY)) ?? { phase: 'idle' }; + const stored = await this.ctx.storage.get(BILLING_STATE_KEY); + const fallbackContext = await this.ctx.storage.get(BILLING_CONTEXT_KEY); + const migrated = migrateStoredUsageState(stored, fallbackContext); + if (!stored || stored.version !== BILLING_STATE_VERSION) { + await this.ctx.storage.put(BILLING_STATE_KEY, migrated); + if (stored) { + console.warn(`${TC_LOG} reset pre-integration billing state for ${this.ctx.id.toString()}`); + } + } + return migrated; + } + + private async restoreRemoteInterval(state: OpenUsageInterval): Promise { + await getContainerUsageClient(this.env).recordStart({ + ...clientUsageContext(state.context), + startEpochMs: state.startEpochMs, + }); } private async getRunPolicy(): Promise { diff --git a/services/gastown/wrangler.jsonc b/services/gastown/wrangler.jsonc index 9ed9af2f07a..501aedb1d21 100644 --- a/services/gastown/wrangler.jsonc +++ b/services/gastown/wrangler.jsonc @@ -140,7 +140,7 @@ "KILO_API_URL": "http://192.168.65.254:3000", "GASTOWN_API_URL": "http://192.168.65.254:8803", "WASTELAND_API_URL": "http://192.168.65.254:8788", - "GASTOWN_BILLING_ENABLED": "false", + "GASTOWN_BILLING_ENABLED": "true", }, "containers": [ { @@ -161,6 +161,13 @@ { "name": "AGENT", "class_name": "AgentDO" }, ], }, + "hyperdrive": [ + { + "binding": "HYPERDRIVE", + "id": "624ec80650dd414199349f4e217ddb10", + "localConnectionString": "postgres://postgres:postgres@localhost:5432/postgres", + }, + ], "services": [ { "binding": "GIT_TOKEN_SERVICE", From fe1acfd3e15f35a91fe6fe283f7556ddc923afb7 Mon Sep 17 00:00:00 2001 From: John Fawcett Date: Mon, 27 Jul 2026 07:59:55 -0500 Subject: [PATCH 07/10] test(dev): assert container usage meter starts with gastown --- dev/local/services.test.ts | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/dev/local/services.test.ts b/dev/local/services.test.ts index b102636bba9..a672c2e0961 100644 --- a/dev/local/services.test.ts +++ b/dev/local/services.test.ts @@ -9,6 +9,7 @@ import { portOffset, resolveGroups, resolveSessionNextAuthUrl, + resolveTargets, } from './services'; test('uses an automatic port offset for secondary worktrees by default', () => { @@ -100,6 +101,37 @@ test('keeps auto routing package dev script compatible with local launcher flags assert.equal(launcherFlags.filter(part => part === '--ip').length, 1); }); +test('starts the container usage meter whenever Gastown starts', () => { + // Gastown's TownContainerDO binds container-usage-meter via a service binding, + // which only connects when the meter is registered in the same local Wrangler + // dev registry. Starting Gastown must therefore always launch the meter. + const gastownTargets = resolveTargets(['gastown']); + assert.ok( + gastownTargets.includes('container-usage-meter'), + `expected container-usage-meter in gastown start targets, got: ${gastownTargets.join(', ')}` + ); + + const meter = getService('container-usage-meter'); + assert.equal(meter.type, 'worker'); + assert.equal(meter.dir, 'services/container-usage-meter'); + assert.equal(meter.port, 8813 + portOffset); +}); + +test('binds the container usage meter under its unsuffixed Wrangler name', () => { + // Gastown binds CONTAINER_USAGE to service "container-usage-meter" with no + // "-dev" suffix. The meter's dev script must not pass --env (which would + // register it as a different name) and must accept the launcher's flags. + const meter = getService('container-usage-meter'); + const packageJson = JSON.parse(fs.readFileSync(`${meter.dir}/package.json`, 'utf-8')) as { + scripts?: { dev?: string }; + }; + const scriptFlags = packageJson.scripts?.dev?.split(/\s+/) ?? []; + + assert.equal(scriptFlags.filter(part => part === '--env').length, 0); + assert.equal(scriptFlags.filter(part => part === '-e').length, 0); + assert.equal(meter.command.filter(part => part === '--ip').length, 1); +}); + test('starts Storybook with Storybook v10 port flags', () => { const service = getService('storybook'); From dcef296f82b9c44c14e7a0edd2c56286cd4542f2 Mon Sep 17 00:00:00 2001 From: John Fawcett Date: Mon, 27 Jul 2026 08:00:04 -0500 Subject: [PATCH 08/10] feat(gastown): report usage independent of billing enforcement --- .specs/gastown-usage-based-billing.md | 29 +++- ENVIRONMENT.md | 2 +- apps/web/src/components/gastown/MayorChat.tsx | 4 +- .../src/components/gastown/TerminalBar.tsx | 31 ++-- apps/web/src/lib/gastown/types/router.d.ts | 6 + .../container-usage-state.billing.test.ts | 56 +++++++- .../billing/container-usage-state.billing.ts | 28 +++- .../billing/container-usage.billing.test.ts | 33 ++++- .../src/billing/container-usage.billing.ts | 25 +++- services/gastown/src/dos/Town.do.ts | 41 ++++-- services/gastown/src/dos/TownContainer.do.ts | 135 ++++++++++++++---- services/gastown/src/trpc/schemas.ts | 1 + .../test/integration/billing-disabled.test.ts | 7 +- 13 files changed, 327 insertions(+), 71 deletions(-) diff --git a/.specs/gastown-usage-based-billing.md b/.specs/gastown-usage-based-billing.md index 03b7c0493a3..d1c821796dc 100644 --- a/.specs/gastown-usage-based-billing.md +++ b/.specs/gastown-usage-based-billing.md @@ -447,11 +447,24 @@ Rollout SHOULD proceed through a single default-off backend flag: Before charging customers, shadow data MUST demonstrate that starts and stops reconcile with Cloudflare runtime observations, duplicate RPCs do not duplicate charges, and aggregate base cost is -reasonably consistent with the Cloudflare invoice. `GASTOWN_BILLING_ENABLED` controls metering, -admission, debits, warnings, and stops together. A separate default-off -`GASTOWN_BILLING_ANNOUNCEMENT_ENABLED` web flag MAY show advance notice while actual billing remains -disabled; it MUST NOT alter backend billing behavior. Both flags are always enabled in local -development while retaining default-off production behavior. +reasonably consistent with the Cloudflare invoice. + +Metering and enforcement are separately controlled: + +- **Metering (reporting)** runs whenever the `CONTAINER_USAGE` binding is present, regardless of any + flag. Usage intervals are recorded for every town so realized usage can be monitored before + charging. This is the mechanism that produces the shadow data above. +- **Enforcement** — admission blocks, low-balance stops, and treating a billing block as draining — + is gated by `GASTOWN_BILLING_ENABLED` (`isGastownBillingEnforced`). With it off, the meter is still + called on every start, heartbeat, and stop, but a `blocked`/`stop` outcome MUST NOT stop the + container or block starts. +- **Announcement** — the default-off `GASTOWN_BILLING_ANNOUNCEMENT_ENABLED` web flag MAY show advance + notice; it MUST NOT alter backend billing behavior. + +The status object distinguishes these: `enabled` means metering is active (drives the run estimate +and the automatic-start control), while `enforcing` gates cold-start/terminal blocking in the UI. +Enforcement and announcement flags are always enabled in local development while retaining +default-off production behavior; metering runs in development because the meter binding is present. Dashboards MUST expose awake seconds, base Cloudflare cost, customer charge, gross multiplier, unclosed intervals, SKU admission denials, RPC failures, graceful-stop duration, forced stops, and @@ -490,6 +503,12 @@ counts of `warn` and `stop` verdicts. ## Changelog +### 2026-07-24 -- Separate reporting from enforcement + +- Metering now always reports to the meter when the `CONTAINER_USAGE` binding is present; + `GASTOWN_BILLING_ENABLED` gates enforcement only. Added an `enforcing` field to the billing status + so the UI blocks cold starts only under enforcement while still showing the run estimate. + ### 2026-07-22 -- User circuit breaker - Added a durable user-controlled automatic-start policy, forced graceful shutdown, and per-run cost diff --git a/ENVIRONMENT.md b/ENVIRONMENT.md index 2a93f24ca67..c1bcc39c4a8 100644 --- a/ENVIRONMENT.md +++ b/ENVIRONMENT.md @@ -205,7 +205,7 @@ Manage shared web env var additions and rotations with `pnpm web:env set { if ( !billingQuery.isSuccess || - billingQuery.data.enabled || + billingQuery.data.enforcing || billingQuery.data.runPolicy === 'paused_by_user' ) return; @@ -79,7 +79,7 @@ export function MayorChat({ townId }: MayorChatProps) { enabled: billingQuery.isSuccess && billingQuery.data.runPolicy === 'automatic' && - (!billingQuery.data.enabled || + (!billingQuery.data.enforcing || billingQuery.data.state === 'running' || billingQuery.data.state === 'warning' || billingQuery.data.state === 'starting'), diff --git a/apps/web/src/components/gastown/TerminalBar.tsx b/apps/web/src/components/gastown/TerminalBar.tsx index 83d43b840ee..3638e9b2b5b 100644 --- a/apps/web/src/components/gastown/TerminalBar.tsx +++ b/apps/web/src/components/gastown/TerminalBar.tsx @@ -1409,7 +1409,7 @@ function MayorTerminalPane({ const ensuredTownRef = useRef(null); useEffect(() => { - if (!billing || billing.enabled || billing.runPolicy === 'paused_by_user') return; + if (!billing || billing.enforcing || billing.runPolicy === 'paused_by_user') return; if (ensuredTownRef.current === townId) return; ensuredTownRef.current = townId; ensureMayor.mutate({ townId }); @@ -1428,12 +1428,13 @@ function MayorTerminalPane({ const mayorAgentId = statusQuery.data?.session?.agentId ?? null; const userPaused = billing?.runPolicy === 'paused_by_user'; const userPauseStopping = userPaused && billing?.state === 'stopping'; - const creditPaused = - billing?.enabled && (billing.state === 'blocked' || billing.state === 'stopping'); + const creditPaused = billing?.enforcing && billing.state === 'blocked'; + const automaticStopInProgress = + billing?.enforcing && billing.runPolicy === 'automatic' && billing.state === 'stopping'; const terminalEnabled = billing !== undefined && billing.runPolicy === 'automatic' && - (!billing.enabled || + (!billing.enforcing || billing.state === 'running' || billing.state === 'warning' || billing.state === 'starting'); @@ -1492,7 +1493,9 @@ function MayorTerminalPane({ : 'Automatic starts are paused' : creditPaused ? 'Gas Town is paused' - : 'Start the Mayor when you are ready'} + : automaticStopInProgress + ? 'Saving work and stopping Gas Town...' + : 'Start the Mayor when you are ready'}

{userPaused @@ -1501,9 +1504,11 @@ function MayorTerminalPane({ : `The container is stopped and scheduled work remains queued.${billing.estimatedRunCharge === undefined ? '' : ` Estimated last run: $${billing.estimatedRunCharge.toFixed(2)}.`} Enable automatic starts when you are ready to continue.` : creditPaused ? 'Add credits to the billing account, then resume. Your town state is preserved.' - : billing.estimatedHourlyCharge === undefined - ? 'Container usage is charged only while Gas Town is running.' - : `Estimated container charge: $${billing.estimatedHourlyCharge.toFixed(2)}/hour while running.`} + : automaticStopInProgress + ? 'The container is shutting down after becoming idle. You can start it again when shutdown completes.' + : billing.estimatedHourlyCharge === undefined + ? 'Container usage is charged only while Gas Town is running.' + : `Estimated container charge: $${billing.estimatedHourlyCharge.toFixed(2)}/hour while running.`}

{creditPaused && !userPaused && ( @@ -1532,9 +1537,11 @@ function MayorTerminalPane({ > {ensureMayor.isPending ? 'Starting Gas Town...' - : creditPaused - ? 'Resume Gas Town' - : 'Start Mayor'} + : automaticStopInProgress + ? 'Stopping Gas Town...' + : creditPaused + ? 'Resume Gas Town' + : 'Start Mayor'} )}
@@ -1564,7 +1571,7 @@ function AgentTerminalPane({ const enabled = billing !== undefined && billing.runPolicy === 'automatic' && - (!billing.enabled || billing.state === 'running' || billing.state === 'warning'); + (!billing.enforcing || billing.state === 'running' || billing.state === 'warning'); const { terminalRef, connectionStatus, status } = useXtermPty({ townId, agentId, diff --git a/apps/web/src/lib/gastown/types/router.d.ts b/apps/web/src/lib/gastown/types/router.d.ts index 677e5eb8d6b..efa1234c9fa 100644 --- a/apps/web/src/lib/gastown/types/router.d.ts +++ b/apps/web/src/lib/gastown/types/router.d.ts @@ -552,6 +552,7 @@ export declare const gastownRouter: import('@trpc/server').TRPCBuiltRouter< }[]; billing: { enabled: boolean; + enforcing: boolean; state: | 'blocked' | 'degraded' @@ -585,6 +586,7 @@ export declare const gastownRouter: import('@trpc/server').TRPCBuiltRouter< }; output: { enabled: boolean; + enforcing: boolean; state: | 'blocked' | 'degraded' @@ -618,6 +620,7 @@ export declare const gastownRouter: import('@trpc/server').TRPCBuiltRouter< }; output: { enabled: boolean; + enforcing: boolean; state: | 'blocked' | 'degraded' @@ -2277,6 +2280,7 @@ export declare const wrappedGastownRouter: import('@trpc/server').TRPCBuiltRoute }[]; billing: { enabled: boolean; + enforcing: boolean; state: | 'blocked' | 'degraded' @@ -2310,6 +2314,7 @@ export declare const wrappedGastownRouter: import('@trpc/server').TRPCBuiltRoute }; output: { enabled: boolean; + enforcing: boolean; state: | 'blocked' | 'degraded' @@ -2343,6 +2348,7 @@ export declare const wrappedGastownRouter: import('@trpc/server').TRPCBuiltRoute }; output: { enabled: boolean; + enforcing: boolean; state: | 'blocked' | 'degraded' diff --git a/services/gastown/src/billing/container-usage-state.billing.test.ts b/services/gastown/src/billing/container-usage-state.billing.test.ts index fd125370983..59e598df45d 100644 --- a/services/gastown/src/billing/container-usage-state.billing.test.ts +++ b/services/gastown/src/billing/container-usage-state.billing.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from 'vitest'; import { createPendingStop, + isRuntimeStoppedStatus, BILLING_STATE_VERSION, migrateStoredUsageState, toBillingStatus, @@ -28,7 +29,19 @@ describe('toBillingStatus', () => { blocked: true, latestBudget: { verdict: 'stop', remaining: 0 }, }) - ).toEqual({ enabled: false, state: 'idle', runPolicy: 'automatic' }); + ).toEqual({ enabled: false, enforcing: false, state: 'idle', runPolicy: 'automatic' }); + }); + + it('reports metering without enforcement by default', () => { + const status = toBillingStatus(true, { phase: 'idle', context }); + expect(status.enabled).toBe(true); + expect(status.enforcing).toBe(false); + }); + + it('reflects the enforcing flag when passed', () => { + const status = toBillingStatus(true, { phase: 'idle', context }, 'automatic', Date.now(), true); + expect(status.enabled).toBe(true); + expect(status.enforcing).toBe(true); }); it('returns a payer-aware blocked state', () => { @@ -41,6 +54,7 @@ describe('toBillingStatus', () => { }) ).toEqual({ enabled: true, + enforcing: false, state: 'blocked', runPolicy: 'automatic', payer: { type: 'org', id: 'org-1' }, @@ -110,6 +124,46 @@ describe('toBillingStatus', () => { }); }); +describe('locally force-closed interval', () => { + it('returns to a non-blocked idle state after unsettled shutdown', () => { + const status = toBillingStatus( + true, + { + version: BILLING_STATE_VERSION, + phase: 'idle', + context, + lastRun: { + startedAt: 1_000, + stoppedAt: 5_000, + usageSeconds: 4, + estimatedCharge: 0.02, + unsettled: true, + }, + }, + 'automatic' + ); + + expect(status.state).toBe('idle'); + expect(status.estimatedRunCharge).toBeCloseTo(0.02, 6); + }); +}); + +describe('isRuntimeStoppedStatus', () => { + it('treats definitively stopped runtimes as stopped', () => { + expect(isRuntimeStoppedStatus('stopped')).toBe(true); + expect(isRuntimeStoppedStatus('stopped_with_code')).toBe(true); + }); + + it('treats live and transient boot/shutdown states as not stopped', () => { + // A freshly created town briefly reports these before it is healthy; + // treating any of them as stopped would immediately close the interval and + // shut the new container down. + for (const status of ['running', 'healthy', 'stopping', 'starting', 'scheduling', '']) { + expect(isRuntimeStoppedStatus(status)).toBe(false); + } + }); +}); + describe('createPendingStop', () => { it('captures whole seconds remaining after the last acknowledged segment', () => { const state: OpenUsageInterval = { diff --git a/services/gastown/src/billing/container-usage-state.billing.ts b/services/gastown/src/billing/container-usage-state.billing.ts index a1a9b1cc253..ce9d3596c98 100644 --- a/services/gastown/src/billing/container-usage-state.billing.ts +++ b/services/gastown/src/billing/container-usage-state.billing.ts @@ -34,6 +34,7 @@ export type OpenUsageInterval = { reportedUsageSeconds?: number; stopReason?: 'exit' | 'runtime_signal' | 'activity_expired'; stopObservedAt?: number; + settlementAttempts?: number; }; export type StoredUsageState = @@ -48,12 +49,21 @@ export type StoredUsageState = stoppedAt: number; usageSeconds: number; estimatedCharge?: number; + unsettled?: boolean; }; } | OpenUsageInterval; export const BILLING_STATE_VERSION = 2 as const; +/** + * How many alarm-driven attempts to settle a stopping interval with the meter + * before Gastown gives up locally. This prevents an unreachable meter from + * stranding a town in the draining/stopping state indefinitely; the interval is + * flagged unsettled for later reconciliation. + */ +export const MAX_SETTLEMENT_ATTEMPTS = 5; + export function migrateStoredUsageState( state: StoredUsageState | undefined, fallbackContext?: UsageContext @@ -75,6 +85,17 @@ export function migrateStoredUsageState( }; } +/** + * Whether a Cloudflare container runtime status means the runtime has + * definitively stopped. Transient boot-time and shutdown states (running, + * healthy, stopping, or any unknown value) are treated as still-live so a + * just-started billing interval is not force-closed by a single heartbeat + * observation; the authoritative close happens in the container's onStop. + */ +export function isRuntimeStoppedStatus(status: string): boolean { + return status === 'stopped' || status === 'stopped_with_code'; +} + export function createPendingStop( state: OpenUsageInterval, stopObservedAt: number, @@ -93,14 +114,16 @@ export function toBillingStatus( enabled: boolean, state: StoredUsageState, runPolicy: ContainerRunPolicy = 'automatic', - now = Date.now() + now = Date.now(), + enforcing = false ): GastownBillingStatus { - if (!enabled) return { enabled: false, state: 'idle', runPolicy }; + if (!enabled) return { enabled: false, enforcing, state: 'idle', runPolicy }; const payer = state.context?.subject; if (state.phase === 'idle') { return { enabled: true, + enforcing, state: runPolicy === 'paused_by_user' ? 'paused' : state.blocked ? 'blocked' : 'idle', runPolicy, ...(payer ? { payer } : {}), @@ -145,6 +168,7 @@ export function toBillingStatus( return { enabled: true, + enforcing, state: publicState, runPolicy, payer, diff --git a/services/gastown/src/billing/container-usage.billing.test.ts b/services/gastown/src/billing/container-usage.billing.test.ts index 82160dfd421..6a3943abef8 100644 --- a/services/gastown/src/billing/container-usage.billing.test.ts +++ b/services/gastown/src/billing/container-usage.billing.test.ts @@ -3,19 +3,42 @@ import { describe, expect, it, vi } from 'vitest'; import { GASTOWN_CONTAINER_SKU, getContainerUsageClient, - isGastownBillingEnabled, + isContainerUsageMeteringEnabled, + isGastownBillingEnforced, isUsageIntervalNotFoundError, } from './container-usage.billing'; +const usageBinding = { + recordStart: vi.fn(), + recordHeartbeat: vi.fn(), + recordStop: vi.fn(), +} satisfies ContainerUsageRpcMethods; + function envFixture(overrides: Partial): Env { return overrides as Env; } describe('Gastown billing configuration', () => { - it('enables billing only for the exact true value', () => { - expect(isGastownBillingEnabled(envFixture({ GASTOWN_BILLING_ENABLED: 'true' }))).toBe(true); - expect(isGastownBillingEnabled(envFixture({ GASTOWN_BILLING_ENABLED: 'false' }))).toBe(false); - expect(isGastownBillingEnabled(envFixture({}))).toBe(false); + it('enforces billing only for the exact true value', () => { + expect(isGastownBillingEnforced(envFixture({ GASTOWN_BILLING_ENABLED: 'true' }))).toBe(true); + expect(isGastownBillingEnforced(envFixture({ GASTOWN_BILLING_ENABLED: 'false' }))).toBe(false); + expect(isGastownBillingEnforced(envFixture({}))).toBe(false); + }); + + it('meters usage whenever the binding exists, regardless of the enforcement flag', () => { + // Reporting must run even with enforcement off so we can monitor real usage + // before turning enforcement on. + expect( + isContainerUsageMeteringEnabled( + envFixture({ CONTAINER_USAGE: usageBinding, GASTOWN_BILLING_ENABLED: 'false' }) + ) + ).toBe(true); + expect( + isContainerUsageMeteringEnabled( + envFixture({ CONTAINER_USAGE: usageBinding, GASTOWN_BILLING_ENABLED: 'true' }) + ) + ).toBe(true); + expect(isContainerUsageMeteringEnabled(envFixture({}))).toBe(false); }); it('uses the production Gastown SKU', () => { diff --git a/services/gastown/src/billing/container-usage.billing.ts b/services/gastown/src/billing/container-usage.billing.ts index ed9923b90a9..92ff8090434 100644 --- a/services/gastown/src/billing/container-usage.billing.ts +++ b/services/gastown/src/billing/container-usage.billing.ts @@ -18,7 +18,10 @@ export type UsageContext = MeterUsageContext & { }; export type GastownBillingStatus = { + /** Metering is active: usage is reported and the run estimate is populated. */ enabled: boolean; + /** Enforcement is active: low balance can block starts and stop the container. */ + enforcing: boolean; state: | 'idle' | 'starting' @@ -44,13 +47,31 @@ export type ContainerRunPolicy = 'automatic' | 'paused_by_user'; export const GASTOWN_CONTAINER_SKU = 'gastown-standard-2026-07'; export const USAGE_HEARTBEAT_INTERVAL_MS = 5 * 60_000; -export function isGastownBillingEnabled(env: Env): boolean { +/** + * Whether Gastown should enforce billing outcomes: blocking cold starts on + * insufficient credits, honoring `stop` budget verdicts, and treating a + * billing block as draining. Enforcement is gated by GASTOWN_BILLING_ENABLED. + * + * This is independent of metering: usage is always reported to the meter (see + * isContainerUsageMeteringEnabled) so we can monitor real usage before turning + * enforcement on. + */ +export function isGastownBillingEnforced(env: Env): boolean { return env.GASTOWN_BILLING_ENABLED === 'true'; } +/** + * Whether container usage should be metered and reported to the meter service. + * Reporting runs regardless of the enforcement flag whenever the CONTAINER_USAGE + * binding is available, so shadow usage data is collected before enforcement. + */ +export function isContainerUsageMeteringEnabled(env: Env): boolean { + return env.CONTAINER_USAGE !== undefined; +} + export function getContainerUsageClient(env: Env): ContainerUsageClient { if (!env.CONTAINER_USAGE) { - throw new Error('CONTAINER_USAGE binding is required when Gastown billing is enabled'); + throw new Error('CONTAINER_USAGE binding is required to report container usage'); } return createContainerUsageClient(env.CONTAINER_USAGE, { service: 'gastown' }); } diff --git a/services/gastown/src/dos/Town.do.ts b/services/gastown/src/dos/Town.do.ts index 2a6af46ff78..7359df62e99 100644 --- a/services/gastown/src/dos/Town.do.ts +++ b/services/gastown/src/dos/Town.do.ts @@ -75,7 +75,8 @@ import { writeEvent, type GastownEventData } from '../util/analytics.util'; import { logger, withLogTags } from '../util/log.util'; import { GASTOWN_CONTAINER_SKU, - isGastownBillingEnabled, + isContainerUsageMeteringEnabled, + isGastownBillingEnforced, type ContainerRunPolicy, type GastownBillingStatus, type UsageActor, @@ -285,7 +286,7 @@ export class TownDO extends DurableObject { emitEvent: data => this.emitEvent(data), prepareContainerBilling: async () => { await this.prepareContainerBilling(); - if (isGastownBillingEnabled(this.env)) { + if (isContainerUsageMeteringEnabled(this.env)) { await getTownContainerStub(this.env, this.townId).warmUp(); this._billingBlocked = false; await this.ctx.storage.put('billing:blocked', false); @@ -956,7 +957,7 @@ export class TownDO extends DurableObject { } async getBillingStatus(): Promise { - if (isGastownBillingEnabled(this.env)) await this.prepareContainerBilling(); + if (isContainerUsageMeteringEnabled(this.env)) await this.prepareContainerBilling(); const status = await getTownContainerStub(this.env, this.townId).getBillingStatus(); await this.syncContainerRunPolicy(status.runPolicy); return status; @@ -982,7 +983,7 @@ export class TownDO extends DurableObject { actor?: UsageActor, providedConfig?: TownConfig ): Promise { - if (!isGastownBillingEnabled(this.env)) return; + if (!isContainerUsageMeteringEnabled(this.env)) return; const townConfig = providedConfig ?? (await this.getTownConfig()); const ownerId = townConfig.owner_id ?? townConfig.organization_id ?? townConfig.owner_user_id; @@ -1033,14 +1034,31 @@ export class TownDO extends DurableObject { } private async updateContainerBilling(): Promise { - if (!isGastownBillingEnabled(this.env)) { - return { enabled: false, state: 'idle', runPolicy: this._containerRunPolicy }; + if (!isContainerUsageMeteringEnabled(this.env)) { + return { + enabled: false, + enforcing: false, + state: 'idle', + runPolicy: this._containerRunPolicy, + }; } + // Usage is always metered and reported. The blocked/graceful-stop reactions + // below are enforcement and only apply when GASTOWN_BILLING_ENABLED is on; + // with enforcement off we collect shadow usage without affecting the town. await this.prepareContainerBilling(); const container = getTownContainerStub(this.env, this.townId); const status = await container.recordUsageHeartbeat(); await this.syncContainerRunPolicy(status.runPolicy); + + if (!isGastownBillingEnforced(this.env)) { + if (this._billingBlocked) { + this._billingBlocked = false; + await this.ctx.storage.put('billing:blocked', false); + } + return status; + } + const blocked = status.runPolicy === 'automatic' && (status.state === 'blocked' || status.state === 'stopping'); @@ -2893,7 +2911,7 @@ export class TownDO extends DurableObject { }> { const townId = this.townId; await this.prepareContainerBilling(actorUserId ? { type: 'user', id: actorUserId } : undefined); - if (isGastownBillingEnabled(this.env)) { + if (isContainerUsageMeteringEnabled(this.env)) { await getTownContainerStub(this.env, townId).warmUp(); this._billingBlocked = false; await this.ctx.storage.put('billing:blocked', false); @@ -3107,7 +3125,7 @@ export class TownDO extends DurableObject { }> { const townId = this.townId; await this.prepareContainerBilling(actorUserId ? { type: 'user', id: actorUserId } : undefined); - if (isGastownBillingEnabled(this.env)) { + if (isContainerUsageMeteringEnabled(this.env)) { await getTownContainerStub(this.env, townId).warmUp(); this._billingBlocked = false; await this.ctx.storage.put('billing:blocked', false); @@ -4288,16 +4306,18 @@ export class TownDO extends DurableObject { const hasRigs = rigList.length > 0; let billingStatus: GastownBillingStatus = { enabled: false, + enforcing: false, state: 'idle', runPolicy: this._containerRunPolicy, }; - if (isGastownBillingEnabled(this.env)) { + if (isContainerUsageMeteringEnabled(this.env)) { try { billingStatus = await this.updateContainerBilling(); } catch (err) { billingStatus = { enabled: true, + enforcing: isGastownBillingEnforced(this.env), state: 'degraded', runPolicy: this._containerRunPolicy, }; @@ -5601,10 +5621,11 @@ export class TownDO extends DurableObject { const billing = cached?.billingStatus ?? - (isGastownBillingEnabled(this.env) + (isContainerUsageMeteringEnabled(this.env) ? await getTownContainerStub(this.env, this.townId).getBillingStatus() : { enabled: false, + enforcing: false, state: 'idle' as const, runPolicy: this._containerRunPolicy, }); diff --git a/services/gastown/src/dos/TownContainer.do.ts b/services/gastown/src/dos/TownContainer.do.ts index e7b697cacdf..0a88e8d5d4b 100644 --- a/services/gastown/src/dos/TownContainer.do.ts +++ b/services/gastown/src/dos/TownContainer.do.ts @@ -4,7 +4,8 @@ import { ContainerBillingError } from '../billing/ContainerBilling.error'; import { clientUsageContext, getContainerUsageClient, - isGastownBillingEnabled, + isContainerUsageMeteringEnabled, + isGastownBillingEnforced, isUsageIntervalNotFoundError, USAGE_HEARTBEAT_INTERVAL_MS, type ContainerRunPolicy, @@ -13,7 +14,9 @@ import { } from '../billing/container-usage.billing'; import { createPendingStop, + isRuntimeStoppedStatus, BILLING_STATE_VERSION, + MAX_SETTLEMENT_ATTEMPTS, migrateStoredUsageState, toBillingStatus, type OpenUsageInterval, @@ -109,7 +112,7 @@ export class TownContainerDO extends Container { await this.stop(); return; } - if (!isGastownBillingEnabled(this.env)) return; + if (!isContainerUsageMeteringEnabled(this.env)) return; try { const state = await this.ensureBillingInterval(); @@ -139,7 +142,7 @@ export class TownContainerDO extends Container { */ async warmUp(): Promise<{ coldStart: boolean; durationMs: number }> { await this.assertAutomaticStartsEnabled(); - if (isGastownBillingEnabled(this.env)) { + if (isContainerUsageMeteringEnabled(this.env)) { await this.ensureBillingInterval(); } await this.assertAutomaticStartsEnabled(); @@ -163,7 +166,7 @@ export class TownContainerDO extends Container { console.log( `${TC_LOG} container stopped: exitCode=${exitCode} reason=${reason} id=${this.ctx.id.toString()}` ); - if (!isGastownBillingEnabled(this.env)) return; + if (!isContainerUsageMeteringEnabled(this.env)) return; const current = await this.getUsageState(); const stopReason = @@ -178,7 +181,7 @@ export class TownContainerDO extends Container { } override async onActivityExpired(): Promise { - if (isGastownBillingEnabled(this.env)) { + if (isContainerUsageMeteringEnabled(this.env)) { const state = await this.getUsageState(); if (state.phase !== 'idle' && state.phase !== 'stopping') { await this.ctx.storage.put(BILLING_STATE_KEY, { @@ -197,7 +200,7 @@ export class TownContainerDO extends Container { override async fetch(request: Request): Promise { await this.assertAutomaticStartsEnabled(); - if (isGastownBillingEnabled(this.env)) { + if (isContainerUsageMeteringEnabled(this.env)) { await this.ensureBillingInterval(); } await this.assertAutomaticStartsEnabled(); @@ -222,14 +225,16 @@ export class TownContainerDO extends Container { async getBillingStatus(): Promise { return toBillingStatus( - isGastownBillingEnabled(this.env), + isContainerUsageMeteringEnabled(this.env), await this.getUsageState(), - await this.getRunPolicy() + await this.getRunPolicy(), + Date.now(), + isGastownBillingEnforced(this.env) ); } async recordUsageHeartbeat(): Promise { - if (!isGastownBillingEnabled(this.env)) return this.getBillingStatus(); + if (!isContainerUsageMeteringEnabled(this.env)) return this.getBillingStatus(); let state = await this.getUsageState(); if (state.phase === 'starting') { @@ -248,50 +253,54 @@ export class TownContainerDO extends Container { } } catch (error) { if (error instanceof ContainerBillingError && error.code === 'INSUFFICIENT_CREDITS') { - await this.stopForBilling(); + // Only stop the container when enforcement is on. With enforcement + // off we keep running and continue metering; the block is advisory. + if (isGastownBillingEnforced(this.env)) { + await this.stopForBilling(); + } return this.getBillingStatus(); } throw error; } } const runPolicy = await this.getRunPolicy(); - if (state.phase === 'idle') return toBillingStatus(true, state, runPolicy); + const enforcing = isGastownBillingEnforced(this.env); + if (state.phase === 'idle') + return toBillingStatus(true, state, runPolicy, Date.now(), enforcing); if (state.phase === 'stopping') { - const stoppingRuntimeState = await this.getState(); - if ( - stoppingRuntimeState.status === 'running' || - stoppingRuntimeState.status === 'healthy' || - stoppingRuntimeState.status === 'stopping' - ) { - return toBillingStatus(true, state, runPolicy); + if (!(await this.isRuntimeStopped())) { + return toBillingStatus(true, state, runPolicy, Date.now(), enforcing); } await this.closeBillingInterval(state.stopReason ?? 'runtime_signal'); return this.getBillingStatus(); } - const runtimeState = await this.getState(); - if (runtimeState.status !== 'running' && runtimeState.status !== 'healthy') { + // Only close a running interval once the runtime has definitively stopped. + // A freshly started container briefly reports transient non-running states + // (and onStop is the authoritative close), so reacting to a single + // non-running reading here would shut down a just-created town. + if (await this.isRuntimeStopped()) { await this.closeBillingInterval('runtime_signal', state.lastReportedAt); return this.getBillingStatus(); } const observedAt = Date.now(); if (observedAt - state.lastReportedAt < USAGE_HEARTBEAT_INTERVAL_MS) { - return toBillingStatus(true, state, runPolicy); + return toBillingStatus(true, state, runPolicy, observedAt, enforcing); } try { const updated = await this.captureHeartbeat(state, observedAt); - return toBillingStatus(true, updated, runPolicy); + return toBillingStatus(true, updated, runPolicy, Date.now(), enforcing); } catch (error) { console.error(`${TC_LOG} billing heartbeat failed`, error); - const status = toBillingStatus(true, state, runPolicy); + const status = toBillingStatus(true, state, runPolicy, Date.now(), enforcing); return { ...status, state: 'degraded' }; } } async stopForBilling(): Promise { - if (!isGastownBillingEnabled(this.env)) return; + if (!isContainerUsageMeteringEnabled(this.env)) return; const state = await this.getUsageState(); if (state.phase !== 'idle') { if (state.phase !== 'stopping') { @@ -306,7 +315,7 @@ export class TownContainerDO extends Container { } async stopForInactivity(): Promise { - if (isGastownBillingEnabled(this.env)) { + if (isContainerUsageMeteringEnabled(this.env)) { const state = await this.getUsageState(); if (state.phase !== 'idle' && state.phase !== 'stopping') { await this.ctx.storage.put(BILLING_STATE_KEY, { @@ -325,7 +334,7 @@ export class TownContainerDO extends Container { const state = await this.getUsageState(); const canCloseBillingInterval = state.phase === 'running' || (state.phase === 'starting' && state.startRecorded); - if (isGastownBillingEnabled(this.env) && canCloseBillingInterval) { + if (isContainerUsageMeteringEnabled(this.env) && canCloseBillingInterval) { await this.ctx.storage.put(BILLING_STATE_KEY, { ...state, phase: 'stopping', @@ -338,7 +347,7 @@ export class TownContainerDO extends Container { } async destroyWithBilling(): Promise { - if (isGastownBillingEnabled(this.env)) { + if (isContainerUsageMeteringEnabled(this.env)) { const state = await this.getUsageState(); if (state.phase !== 'idle') { await this.stopForBilling(); @@ -370,7 +379,7 @@ export class TownContainerDO extends Container { private async ensureBillingInterval(): Promise { await this.assertAutomaticStartsEnabled(); - if (!isGastownBillingEnabled(this.env)) return { phase: 'idle' }; + if (!isContainerUsageMeteringEnabled(this.env)) return { phase: 'idle' }; const existing = await this.getUsageState(); if (existing.phase === 'running') return existing; @@ -615,8 +624,68 @@ export class TownContainerDO extends Container { } satisfies StoredUsageState); } } catch (error) { + await this.handleSettlementFailure(stopping, stopObservedAt, error); + } + } + + /** + * Records a failed settlement attempt. While the meter is unreachable the + * interval stays open and retries on the next alarm. After + * MAX_SETTLEMENT_ATTEMPTS, Gastown force-closes the interval locally so the + * town cannot be stranded in the stopping/draining state, and flags it + * unsettled for later reconciliation. + */ + private async handleSettlementFailure( + stopping: OpenUsageInterval, + stopObservedAt: number, + error: unknown + ): Promise { + const attempts = await this.ctx.storage.transaction(async transaction => { + const latest = await transaction.get(BILLING_STATE_KEY); + if (!latest || latest.phase === 'idle' || latest.startEpochMs !== stopping.startEpochMs) { + return undefined; + } + const nextAttempts = (latest.settlementAttempts ?? 0) + 1; + await transaction.put(BILLING_STATE_KEY, { + ...latest, + settlementAttempts: nextAttempts, + } satisfies OpenUsageInterval); + return nextAttempts; + }); + + if (attempts === undefined || attempts < MAX_SETTLEMENT_ATTEMPTS) { console.error(`${TC_LOG} failed to close billing interval; will retry`, error); + return; } + + console.error( + `${TC_LOG} giving up on meter settlement after ${attempts} attempts; closing interval locally and flagging for reconciliation`, + error + ); + await this.ctx.storage.transaction(async transaction => { + const latest = await transaction.get(BILLING_STATE_KEY); + if (!latest || latest.phase === 'idle' || latest.startEpochMs !== stopping.startEpochMs) { + return; + } + const usageSeconds = + (latest.reportedUsageSeconds ?? 0) + (latest.pendingStop?.usageSinceLast ?? 0); + const estimatedCharge = + latest.estimatedHourlyCharge === undefined + ? undefined + : (usageSeconds / 3600) * latest.estimatedHourlyCharge; + await transaction.put(BILLING_STATE_KEY, { + version: BILLING_STATE_VERSION, + phase: 'idle', + context: latest.context, + lastRun: { + startedAt: latest.startEpochMs, + stoppedAt: latest.stopObservedAt ?? stopObservedAt, + usageSeconds, + unsettled: true, + ...(estimatedCharge === undefined ? {} : { estimatedCharge }), + }, + } satisfies StoredUsageState); + }); } private async getUsageState(): Promise { @@ -639,6 +708,16 @@ export class TownContainerDO extends Container { }); } + /** + * Whether the container runtime has definitively stopped. Delegates to the + * pure isRuntimeStoppedStatus predicate so the tolerance policy is unit + * tested independently of the Durable Object. + */ + private async isRuntimeStopped(): Promise { + const runtimeState = await this.getState(); + return isRuntimeStoppedStatus(runtimeState.status); + } + private async getRunPolicy(): Promise { return (await this.ctx.storage.get(RUN_POLICY_KEY)) ?? 'automatic'; } diff --git a/services/gastown/src/trpc/schemas.ts b/services/gastown/src/trpc/schemas.ts index 441effe5433..614f0531a5a 100644 --- a/services/gastown/src/trpc/schemas.ts +++ b/services/gastown/src/trpc/schemas.ts @@ -104,6 +104,7 @@ export const MayorStatusOutput = z.object({ export const BillingStatusOutput = z.object({ enabled: z.boolean(), + enforcing: z.boolean(), state: z.enum([ 'idle', 'starting', diff --git a/services/gastown/test/integration/billing-disabled.test.ts b/services/gastown/test/integration/billing-disabled.test.ts index 4c1169a63bf..ec985c6aa46 100644 --- a/services/gastown/test/integration/billing-disabled.test.ts +++ b/services/gastown/test/integration/billing-disabled.test.ts @@ -1,19 +1,20 @@ import { env } from 'cloudflare:test'; import { describe, expect, it } from 'vitest'; -describe('Gastown billing feature flag', () => { - it('keeps billing disabled in both dedicated and alarm status', async () => { +describe('Gastown billing feature flags', () => { + it('reports neither metering nor enforcement without the meter binding', async () => { const townId = `billing-disabled-${crypto.randomUUID()}`; const town = env.TOWN.get(env.TOWN.idFromName(townId)); await town.setTownId(townId); await expect(town.getBillingStatus()).resolves.toEqual({ enabled: false, + enforcing: false, state: 'idle', runPolicy: 'automatic', }); await expect(town.getAlarmStatus()).resolves.toMatchObject({ - billing: { enabled: false, state: 'idle', runPolicy: 'automatic' }, + billing: { enabled: false, enforcing: false, state: 'idle', runPolicy: 'automatic' }, }); }); From f2892c4fe488d3b272a18b6ff85b1e929f1d802d Mon Sep 17 00:00:00 2001 From: John Fawcett Date: Mon, 27 Jul 2026 09:06:44 -0500 Subject: [PATCH 09/10] fix(gastown): harden billing error handling and degraded-state UX --- apps/web/src/components/gastown/MayorChat.tsx | 5 +- .../src/components/gastown/TerminalBar.tsx | 14 ++++- services/gastown/src/dos/Town.do.ts | 52 ++++++++++++++--- services/gastown/src/dos/TownContainer.do.ts | 57 +++++++++++-------- services/gastown/src/trpc/router.ts | 31 +++++----- 5 files changed, 108 insertions(+), 51 deletions(-) diff --git a/apps/web/src/components/gastown/MayorChat.tsx b/apps/web/src/components/gastown/MayorChat.tsx index 2b319b533c4..01beeefefcb 100644 --- a/apps/web/src/components/gastown/MayorChat.tsx +++ b/apps/web/src/components/gastown/MayorChat.tsx @@ -82,7 +82,10 @@ export function MayorChat({ townId }: MayorChatProps) { (!billingQuery.data.enforcing || billingQuery.data.state === 'running' || billingQuery.data.state === 'warning' || - billingQuery.data.state === 'starting'), + billingQuery.data.state === 'starting' || + // `degraded` is a transient metering hiccup; the container keeps + // running per spec, so don't disconnect the terminal. + billingQuery.data.state === 'degraded'), retries: 10, retryDelay: 3_000, }); diff --git a/apps/web/src/components/gastown/TerminalBar.tsx b/apps/web/src/components/gastown/TerminalBar.tsx index 3638e9b2b5b..9911737f7cd 100644 --- a/apps/web/src/components/gastown/TerminalBar.tsx +++ b/apps/web/src/components/gastown/TerminalBar.tsx @@ -1437,7 +1437,10 @@ function MayorTerminalPane({ (!billing.enforcing || billing.state === 'running' || billing.state === 'warning' || - billing.state === 'starting'); + billing.state === 'starting' || + // `degraded` is a transient metering hiccup; per spec the container keeps + // running, so we must not tear down the terminal. + billing.state === 'degraded'); const creditsHref = billing?.payer?.type === 'org' ? `/organizations/${billing.payer.id}` : '/credits'; @@ -1568,10 +1571,17 @@ function AgentTerminalPane({ agentId: string; billing?: BillingStatus; }) { + // Keep the agent terminal's connect/retry gating consistent with the Mayor + // panes: allow `starting` (still coming up) and `degraded` (transient + // metering hiccup, container keeps running per spec). const enabled = billing !== undefined && billing.runPolicy === 'automatic' && - (!billing.enforcing || billing.state === 'running' || billing.state === 'warning'); + (!billing.enforcing || + billing.state === 'running' || + billing.state === 'warning' || + billing.state === 'starting' || + billing.state === 'degraded'); const { terminalRef, connectionStatus, status } = useXtermPty({ townId, agentId, diff --git a/services/gastown/src/dos/Town.do.ts b/services/gastown/src/dos/Town.do.ts index 7359df62e99..3e4dadbf001 100644 --- a/services/gastown/src/dos/Town.do.ts +++ b/services/gastown/src/dos/Town.do.ts @@ -957,10 +957,25 @@ export class TownDO extends DurableObject { } async getBillingStatus(): Promise { - if (isContainerUsageMeteringEnabled(this.env)) await this.prepareContainerBilling(); - const status = await getTownContainerStub(this.env, this.townId).getBillingStatus(); - await this.syncContainerRunPolicy(status.runPolicy); - return status; + try { + if (isContainerUsageMeteringEnabled(this.env)) await this.prepareContainerBilling(); + const status = await getTownContainerStub(this.env, this.townId).getBillingStatus(); + await this.syncContainerRunPolicy(status.runPolicy); + return status; + } catch (error) { + // A read-only status query must never fail the dashboard. Fall back to a + // degraded status if billing setup or the container DO is unreachable. + logger.warn('billing: getBillingStatus failed, returning degraded', { + townId: this.townId, + error: error instanceof Error ? error.message : String(error), + }); + return { + enabled: isContainerUsageMeteringEnabled(this.env), + enforcing: isGastownBillingEnforced(this.env), + state: 'degraded', + runPolicy: this._containerRunPolicy, + }; + } } async setContainerRunPolicy(policy: ContainerRunPolicy): Promise { @@ -987,7 +1002,15 @@ export class TownDO extends DurableObject { const townConfig = providedConfig ?? (await this.getTownConfig()); const ownerId = townConfig.owner_id ?? townConfig.organization_id ?? townConfig.owner_user_id; - if (!ownerId) throw new Error('Town has no billing owner'); + if (!ownerId) { + // A town without a resolvable billing owner (e.g. a freshly created town + // before its owner is set) simply isn't metered yet. Skip billing setup + // rather than throwing so unrelated config/token updates aren't broken. + logger.warn('billing: skipping container billing setup, town has no billing owner', { + townId: this.townId, + }); + return; + } const subject = townConfig.owner_type === 'org' @@ -5622,7 +5645,10 @@ export class TownDO extends DurableObject { const billing = cached?.billingStatus ?? (isContainerUsageMeteringEnabled(this.env) - ? await getTownContainerStub(this.env, this.townId).getBillingStatus() + ? // getBillingStatus is guarded and falls back to a degraded status, so + // the Status tab never fails just because the container DO is + // momentarily unreachable. + await this.getBillingStatus() : { enabled: false, enforcing: false, @@ -5984,9 +6010,19 @@ export class TownDO extends DurableObject { console.warn(`${TOWN_LOG} destroy: agent cleanup failed`, err); } - // Usage settlement must be acknowledged before container state is destroyed. + // Attempt to settle usage before destroying container state, but never let + // a billing-settlement failure block storage teardown — otherwise the town + // would be left un-destroyable until the container/billing path recovers. + // The container DO force-closes stale intervals for reconciliation anyway. const containerStub = getTownContainerStub(this.env, this.townId); - await containerStub.destroyWithBilling(); + try { + await containerStub.destroyWithBilling(); + } catch (err) { + console.warn(`${TOWN_LOG} destroy: container billing settlement failed`, err); + await containerStub.destroy().catch(destroyErr => { + console.warn(`${TOWN_LOG} destroy: container cleanup failed`, destroyErr); + }); + } await this.ctx.storage.deleteAlarm(); await this.ctx.storage.deleteAll(); diff --git a/services/gastown/src/dos/TownContainer.do.ts b/services/gastown/src/dos/TownContainer.do.ts index 0a88e8d5d4b..4f921021f6b 100644 --- a/services/gastown/src/dos/TownContainer.do.ts +++ b/services/gastown/src/dos/TownContainer.do.ts @@ -237,31 +237,27 @@ export class TownContainerDO extends Container { if (!isContainerUsageMeteringEnabled(this.env)) return this.getBillingStatus(); let state = await this.getUsageState(); - if (state.phase === 'starting') { - state = await this.completeBillingStart(state); - } - const initialRuntimeState = state.phase === 'idle' ? await this.getState() : null; - if ( - state.phase === 'idle' && - (initialRuntimeState?.status === 'running' || initialRuntimeState?.status === 'healthy') - ) { - try { - state = await this.ensureBillingInterval(); - if (state.phase === 'starting' && state.startRecorded) { - state = { ...state, phase: 'running' }; - await this.ctx.storage.put(BILLING_STATE_KEY, state); - } - } catch (error) { - if (error instanceof ContainerBillingError && error.code === 'INSUFFICIENT_CREDITS') { - // Only stop the container when enforcement is on. With enforcement - // off we keep running and continue metering; the block is advisory. - if (isGastownBillingEnforced(this.env)) { - await this.stopForBilling(); - } - return this.getBillingStatus(); + try { + if (state.phase === 'starting') { + state = await this.promoteStartedInterval(await this.completeBillingStart(state)); + } + const initialRuntimeState = state.phase === 'idle' ? await this.getState() : null; + if ( + state.phase === 'idle' && + (initialRuntimeState?.status === 'running' || initialRuntimeState?.status === 'healthy') + ) { + state = await this.promoteStartedInterval(await this.ensureBillingInterval()); + } + } catch (error) { + if (error instanceof ContainerBillingError && error.code === 'INSUFFICIENT_CREDITS') { + // Only stop the container when enforcement is on. With enforcement + // off we keep running and continue metering; the block is advisory. + if (isGastownBillingEnforced(this.env)) { + await this.stopForBilling(); } - throw error; + return this.getBillingStatus(); } + throw error; } const runPolicy = await this.getRunPolicy(); const enforcing = isGastownBillingEnforced(this.env); @@ -299,6 +295,21 @@ export class TownContainerDO extends Container { } } + /** + * Once the start RPC has been recorded, promote the interval from `starting` + * to `running` and persist it, mirroring what onStart does. This prevents + * callers from observing a stale `starting` status for up to a heartbeat + * interval after the interval has actually started. + */ + private async promoteStartedInterval(state: StoredUsageState): Promise { + if (state.phase === 'starting' && state.startRecorded) { + const running = { ...state, phase: 'running' as const }; + await this.ctx.storage.put(BILLING_STATE_KEY, running); + return running; + } + return state; + } + async stopForBilling(): Promise { if (!isContainerUsageMeteringEnabled(this.env)) return; const state = await this.getUsageState(); diff --git a/services/gastown/src/trpc/router.ts b/services/gastown/src/trpc/router.ts index 9729839c863..2936ae72f03 100644 --- a/services/gastown/src/trpc/router.ts +++ b/services/gastown/src/trpc/router.ts @@ -86,24 +86,21 @@ function userFromCtx(ctx: TRPCContext): { id: string; api_token_pepper: string | } function mapContainerBillingError(error: unknown): TRPCError | null { - let code: unknown; - if (error instanceof ContainerBillingError) { - code = error.code; - } else if (typeof error === 'object' && error !== null && 'code' in error) { - code = error.code; + // Only classify errors we actually own. Matching on message substrings is + // unreliable (the fallback message itself contains "billing"), so a + // non-billing failure could be misreported as a billing error and mask the + // real cause. Classify strictly on the ContainerBillingError code. + if (!(error instanceof ContainerBillingError)) return null; + + switch (error.code) { + case 'INSUFFICIENT_CREDITS': + case 'CONTAINER_PAUSED': + return new TRPCError({ code: 'PRECONDITION_FAILED', message: error.message, cause: error }); + case 'BILLING_UNAVAILABLE': + return new TRPCError({ code: 'SERVICE_UNAVAILABLE', message: error.message, cause: error }); + default: + return null; } - - const message = error instanceof Error ? error.message : 'Container billing is unavailable'; - if (code === 'INSUFFICIENT_CREDITS' || message.includes('in credits to start')) { - return new TRPCError({ code: 'PRECONDITION_FAILED', message, cause: error }); - } - if (code === 'CONTAINER_PAUSED') { - return new TRPCError({ code: 'PRECONDITION_FAILED', message, cause: error }); - } - if (code === 'BILLING_UNAVAILABLE' || message.includes('billing')) { - return new TRPCError({ code: 'SERVICE_UNAVAILABLE', message, cause: error }); - } - return null; } /** Look up a user's membership for a specific org from the JWT claims. */ From 43dff9e1d20e6a56996bdea9eb6bfa14dd455f5a Mon Sep 17 00:00:00 2001 From: John Fawcett Date: Mon, 27 Jul 2026 15:44:34 -0500 Subject: [PATCH 10/10] refactor(gastown): remove debug script, simplify billing state derivation --- services/container-usage-meter/package.json | 1 - .../billing/container-usage-state.billing.ts | 38 ++++++++++++------- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/services/container-usage-meter/package.json b/services/container-usage-meter/package.json index 103c7c9ba9c..c8db93e5e1c 100644 --- a/services/container-usage-meter/package.json +++ b/services/container-usage-meter/package.json @@ -9,7 +9,6 @@ "typecheck": "tsgo --noEmit", "lint": "pnpm -w exec oxlint --config .oxlintrc.json services/container-usage-meter/src", "test": "vitest run", - "wrangler": "wrangler", "test:postgres": "vitest run --config vitest.postgres.config.ts" }, "dependencies": { diff --git a/services/gastown/src/billing/container-usage-state.billing.ts b/services/gastown/src/billing/container-usage-state.billing.ts index ce9d3596c98..8cad8792e6c 100644 --- a/services/gastown/src/billing/container-usage-state.billing.ts +++ b/services/gastown/src/billing/container-usage-state.billing.ts @@ -110,6 +110,29 @@ export function createPendingStop( }; } +/** Derive the public billing state for an idle (no open interval) town. */ +function idleBillingState( + state: Extract, + runPolicy: ContainerRunPolicy +): GastownBillingStatus['state'] { + if (runPolicy === 'paused_by_user') return 'paused'; + if (state.blocked) return 'blocked'; + return 'idle'; +} + +/** Derive the public billing state for a town with an open interval. */ +function openIntervalBillingState( + state: OpenUsageInterval, + runPolicy: ContainerRunPolicy +): GastownBillingStatus['state'] { + if (runPolicy === 'paused_by_user') return 'stopping'; + if (state.phase === 'starting') return 'starting'; + if (state.phase === 'stopping') return 'stopping'; + if (state.latestBudget?.verdict === 'stop') return 'stopping'; + if (state.latestBudget?.verdict === 'warn') return 'warning'; + return 'running'; +} + export function toBillingStatus( enabled: boolean, state: StoredUsageState, @@ -124,7 +147,7 @@ export function toBillingStatus( return { enabled: true, enforcing, - state: runPolicy === 'paused_by_user' ? 'paused' : state.blocked ? 'blocked' : 'idle', + state: idleBillingState(state, runPolicy), runPolicy, ...(payer ? { payer } : {}), ...(state.latestBudget?.remaining === undefined @@ -143,18 +166,7 @@ export function toBillingStatus( }; } - const publicState = - runPolicy === 'paused_by_user' - ? 'stopping' - : state.phase === 'starting' - ? 'starting' - : state.phase === 'stopping' - ? 'stopping' - : state.latestBudget?.verdict === 'stop' - ? 'stopping' - : state.latestBudget?.verdict === 'warn' - ? 'warning' - : 'running'; + const publicState = openIntervalBillingState(state, runPolicy); const currentSliceSeconds = state.phase === 'running' || (state.phase === 'stopping' && state.stopObservedAt === undefined)