From 73299e2a7bed02b4a06f3365f945844e2e8100ab Mon Sep 17 00:00:00 2001 From: jinye Date: Tue, 4 Aug 2026 10:44:41 +0800 Subject: [PATCH 1/6] feat(serve): add the child-heap admission primitives, unwired MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Groundwork for #8182 step 2. Nothing calls any of this yet, so no child is sized differently and no spawn is refused. `ProcessRegistry.committedProcessCount` counts attached children plus reservations that have not attached. That is the figure admission has to key on: `reserve()` inserts its token synchronously before `spawn()`, so two racing spawns each see the other, while neither appears in `activeProcessCount` until its child attaches. A child leaves the count on exit rather than when `terminate()` starts, so a channel swap counts twice while the old process winds down — deliberate, since its memory is still resident. `getAcpMemoryArgs(explicitMb?)` takes an optional share that bypasses both the module cache and the raise-only guard. Both bypasses are load-bearing. The cache, because the share depends on how many children are live now rather than on the host. The guard, because a budget-derived share is normally *below* the daemon's own heap limit, so routing it through `targetMB > currentLimitMB` would drop the flag, silently restore the overcommit, and leave every test green — the trap #8182 calls out. The regression test asserts the flag survives 614 MB against a multi-GB runner, and mutation-checking it by reinstating the guard fails two tests. `createChildHeapPolicy` holds the mode, the budget, and the would-be refusal counter, and answers `decide(concurrentChildren)`. The refusal is derived from the unclamped quotient, not from `recommendedChildShareMb`, because that function clamps *up* to the 512 MB floor: past the point where the pool stops covering the count its answer saturates and can no longer distinguish "barely does not fit" from "wildly does not fit". `ChildHeapPoolExhaustedError` with both transport mappings — REST 503 with Retry-After, ACP `child_heap_pool_exhausted` — added together, since the two mappings are hand-written and drift silently otherwise. Refusing at spawn rather than at registration is the correction #8182 demands: registration allocates nothing, so this surfaces as "no new session in this workspace right now", which is true and retryable. Refs #8182. Co-Authored-By: Claude Opus 5 --- packages/acp-bridge/src/bridgeErrors.ts | 34 ++++++ .../acp-bridge/src/child-heap-policy.test.ts | 102 ++++++++++++++++++ packages/acp-bridge/src/child-heap-policy.ts | 102 ++++++++++++++++++ .../acp-bridge/src/process-registry.test.ts | 40 +++++++ packages/acp-bridge/src/process-registry.ts | 16 +++ packages/acp-bridge/src/spawnChannel.test.ts | 33 ++++++ packages/acp-bridge/src/spawnChannel.ts | 22 +++- packages/cli/src/serve/acp-http/dispatch.ts | 13 +++ packages/cli/src/serve/acp-session-bridge.ts | 1 + .../cli/src/serve/server/error-response.ts | 21 ++++ 10 files changed, 383 insertions(+), 1 deletion(-) create mode 100644 packages/acp-bridge/src/child-heap-policy.test.ts create mode 100644 packages/acp-bridge/src/child-heap-policy.ts diff --git a/packages/acp-bridge/src/bridgeErrors.ts b/packages/acp-bridge/src/bridgeErrors.ts index 6ac2a5bf247..b2a6bb25e18 100644 --- a/packages/acp-bridge/src/bridgeErrors.ts +++ b/packages/acp-bridge/src/bridgeErrors.ts @@ -186,6 +186,40 @@ export class TotalSessionLimitExceededError extends Error { } } +/** + * Thrown at spawn time when the daemon's child pool cannot cover another ACP + * child at the minimum heap. Refusing here rather than at registration is + * deliberate: registration allocates nothing (a dormant workspace has no + * child), so this surfaces as "no new session in this workspace right now", + * which is both true and retryable — the condition clears as soon as another + * child exits. + * + * Only `--child-heap-mode enforce` throws it. Under `observe` the same + * condition is counted and reported instead, so a deployment can find out + * whether enforcement would have refused anything before it does. + */ +export class ChildHeapPoolExhaustedError extends Error { + readonly childPoolMb: number; + readonly concurrentChildren: number; + readonly minChildHeapMb: number; + constructor( + childPoolMb: number, + concurrentChildren: number, + minChildHeapMb: number, + ) { + super( + `Daemon child heap pool (${childPoolMb} MB) cannot cover ` + + `${concurrentChildren} concurrent children at the ${minChildHeapMb} MB ` + + `minimum. Wait for a child to exit, reduce concurrent workspaces, or ` + + `raise --memory-budget-mb.`, + ); + this.name = 'ChildHeapPoolExhaustedError'; + this.childPoolMb = childPoolMb; + this.concurrentChildren = concurrentChildren; + this.minChildHeapMb = minChildHeapMb; + } +} + /** * Thrown by `sendPrompt` when a session already has too many accepted * prompts waiting or running. The REST route maps this to 503 with diff --git a/packages/acp-bridge/src/child-heap-policy.test.ts b/packages/acp-bridge/src/child-heap-policy.test.ts new file mode 100644 index 00000000000..b3cf71c18b6 --- /dev/null +++ b/packages/acp-bridge/src/child-heap-policy.test.ts @@ -0,0 +1,102 @@ +/** + * @license + * Copyright 2026 Qwen Team + * SPDX-License-Identifier: Apache-2.0 + */ + +import { describe, expect, it } from 'vitest'; +import { createChildHeapPolicy } from './child-heap-policy.js'; +import { + MIN_CHILD_HEAP_MB, + resolveDaemonMemoryBudget, +} from './daemon-memory-budget.js'; + +// 8 GB of available memory, so the derived pool is a realistic size and the +// refusal boundary lands at a child count a real daemon could reach. +const budget = resolveDaemonMemoryBudget({ availableMemoryMb: 8_192 }); +const poolMb = budget.childPoolMb; +// The last count the pool can still cover at the floor, and the first it cannot. +const lastFitting = Math.floor(poolMb / MIN_CHILD_HEAP_MB); + +describe('createChildHeapPolicy', () => { + it('refuses exactly when the pool can no longer cover another child', () => { + const policy = createChildHeapPolicy({ budget, mode: 'enforce' }); + + expect(policy.decide(lastFitting).refuse).toBe(false); + expect(policy.decide(lastFitting + 1).refuse).toBe(true); + + // The boundary is the unclamped quotient, not the returned share. Past + // the boundary the share saturates at the floor and stops carrying any + // information: "barely does not fit" and "wildly does not fit" both read + // as 512, so a refusal derived from the share could never tell them apart. + expect(policy.decide(lastFitting).ceilingMb).toBeGreaterThan( + MIN_CHILD_HEAP_MB, + ); + expect(policy.decide(lastFitting + 1).ceilingMb).toBe(MIN_CHILD_HEAP_MB); + expect(policy.decide(lastFitting * 4).ceilingMb).toBe(MIN_CHILD_HEAP_MB); + }); + + it('shrinks the share as children arrive, and never below the floor', () => { + const policy = createChildHeapPolicy({ budget, mode: 'enforce' }); + + const one = policy.decide(1).ceilingMb!; + const two = policy.decide(2).ceilingMb!; + const four = policy.decide(4).ceilingMb!; + expect(one).toBeGreaterThan(two); + expect(two).toBeGreaterThan(four); + expect(four).toBeGreaterThanOrEqual(MIN_CHILD_HEAP_MB); + // Concurrency, not registration: this is a share of the pool at the count + // passed in, so it is capped by the legacy ceiling rather than the pool. + expect(one).toBeLessThanOrEqual(budget.legacyChildCeilingMb); + }); + + it('computes in observe mode but reports nothing as enforced', () => { + const observe = createChildHeapPolicy({ budget, mode: 'observe' }); + + // The point of `observe`: the numbers exist, so a caller can be wired up + // and tested, while `enforced` stays false because nothing is applied. + const decision = observe.decide(lastFitting + 1); + expect(decision.ceilingMb).toBe(MIN_CHILD_HEAP_MB); + expect(decision.refuse).toBe(true); + expect(observe.snapshot()).toMatchObject({ + mode: 'observe', + enforced: false, + refusals: 1, + }); + }); + + it('counts would-be refusals so calibration does not need a broken deployment', () => { + const policy = createChildHeapPolicy({ budget, mode: 'observe' }); + expect(policy.snapshot().refusals).toBe(0); + + policy.decide(1); + policy.decide(lastFitting); + expect(policy.snapshot().refusals).toBe(0); + + policy.decide(lastFitting + 1); + policy.decide(lastFitting + 9); + expect(policy.snapshot().refusals).toBe(2); + }); + + it('computes nothing at all when off', () => { + const off = createChildHeapPolicy({ budget, mode: 'off' }); + + // Not "a share of zero" — no share, so the caller keeps the historical + // host-derived ceiling. And an off policy must never accrue refusals, + // or the calibration counter would report on a daemon that never applied + // the policy in the first place. + expect(off.decide(lastFitting + 1)).toEqual({ + ceilingMb: undefined, + refuse: false, + }); + expect(off.snapshot()).toMatchObject({ enforced: false, refusals: 0 }); + }); + + it('treats a zero or negative count as one child', () => { + // Defensive: the caller reads a live count that should always include the + // spawn being admitted, but a 0 would otherwise divide the pool by zero. + const policy = createChildHeapPolicy({ budget, mode: 'enforce' }); + expect(policy.decide(0)).toEqual(policy.decide(1)); + expect(Number.isFinite(policy.decide(0).ceilingMb!)).toBe(true); + }); +}); diff --git a/packages/acp-bridge/src/child-heap-policy.ts b/packages/acp-bridge/src/child-heap-policy.ts new file mode 100644 index 00000000000..d89fe930af9 --- /dev/null +++ b/packages/acp-bridge/src/child-heap-policy.ts @@ -0,0 +1,102 @@ +/** + * @license + * Copyright 2026 Qwen Team + * SPDX-License-Identifier: Apache-2.0 + */ + +import { + MIN_CHILD_HEAP_MB, + recommendedChildShareMb, + type DaemonMemoryBudget, +} from './daemon-memory-budget.js'; + +/** + * What the daemon does with the child-heap share it computes. + * + * `off` — do not compute it. Children get the historical host-derived ceiling. + * + * `observe` — compute the share and the admission decision, apply **neither**, + * and count the refusals that would have happened. Deliberately the default: + * the thresholds this policy divides by have never been checked against a real + * multi-workspace deployment, and a non-zero refusal count is the evidence + * that turning `enforce` on would have broken someone. + * + * `enforce` — pass the share to the child and refuse the spawn when the pool + * cannot cover another one. + */ +export type ChildHeapMode = 'off' | 'observe' | 'enforce'; + +export interface ChildHeapDecision { + /** + * The share this child should receive, or `undefined` when the mode does not + * produce one. `undefined` means "spawn as before", never "zero". + */ + ceilingMb: number | undefined; + /** Whether the pool cannot cover this child. Only acted on under `enforce`. */ + refuse: boolean; +} + +export interface ChildHeapPolicySnapshot { + mode: ChildHeapMode; + /** True only under `enforce` — i.e. only when a spawn argument really derives from this. */ + enforced: boolean; + childPoolMb: number; + minChildHeapMb: number; + /** + * Spawns this policy refused, or would have refused under `enforce`. The + * calibration signal: non-zero under `observe` means enforcement would have + * failed a real spawn, including the channel-swap case where a replacement + * is counted alongside the process it replaces. + */ + refusals: number; +} + +export interface ChildHeapPolicy { + /** + * @param concurrentChildren Children already committed *including this one* + * — `ProcessRegistry.committedProcessCount` taken after `reserve()`. + */ + decide(concurrentChildren: number): ChildHeapDecision; + snapshot(): ChildHeapPolicySnapshot; +} + +export function createChildHeapPolicy(options: { + budget: DaemonMemoryBudget; + mode: ChildHeapMode; +}): ChildHeapPolicy { + const { budget, mode } = options; + let refusals = 0; + + return { + decide(concurrentChildren) { + if (mode === 'off') return { ceilingMb: undefined, refuse: false }; + const children = Math.max(concurrentChildren, 1); + + // `recommendedChildShareMb` clamps UP to MIN_CHILD_HEAP_MB, so past the + // point where the pool stops covering the count its answer saturates at + // 512 and can no longer say "will not fit" — a 600 MB pool split four + // ways still returns 512. Derive the refusal from the unclamped + // quotient, or the clamp silently authorises the very overcommit this + // policy exists to bound. + const refuse = + Math.floor(budget.childPoolMb / children) < MIN_CHILD_HEAP_MB; + if (refuse) refusals += 1; + + return { + // Reported in both modes; only `enforce` lets the caller apply it. + ceilingMb: recommendedChildShareMb(budget, children), + refuse, + }; + }, + + snapshot() { + return { + mode, + enforced: mode === 'enforce', + childPoolMb: budget.childPoolMb, + minChildHeapMb: MIN_CHILD_HEAP_MB, + refusals, + }; + }, + }; +} diff --git a/packages/acp-bridge/src/process-registry.test.ts b/packages/acp-bridge/src/process-registry.test.ts index d9f80a668c8..cc8ed4da423 100644 --- a/packages/acp-bridge/src/process-registry.test.ts +++ b/packages/acp-bridge/src/process-registry.test.ts @@ -23,6 +23,46 @@ afterEach(() => { }); describe('ProcessRegistry', () => { + it('counts unattached reservations, which is what admission must key on', () => { + const registry = new ProcessRegistry(); + expect(registry.committedProcessCount).toBe(0); + + // Two spawns racing: both reserve before either attaches. This is the + // invariant an admission check rests on — `activeProcessCount` shows + // neither of them yet, so keying off it would let both through. + const first = registry.reserve(); + const second = registry.reserve(); + expect(registry.activeProcessCount).toBe(0); + expect(registry.committedProcessCount).toBe(2); + + first.attach(fakeChild(1)); + expect(registry.committedProcessCount).toBe(2); + + // A cancelled reservation releases its slot; leaking it would inflate the + // count for every later spawn. + second.cancel(); + expect(registry.committedProcessCount).toBe(1); + second.cancel(); + expect(registry.committedProcessCount).toBe(1); + }); + + it('releases a committed slot on exit, not when terminate starts', async () => { + const registry = new ProcessRegistry(); + const child = fakeChild(4321); + const tracked = registry.reserve().attach(child); + expect(registry.committedProcessCount).toBe(1); + + // Winding down still occupies the pool: the process is alive and its + // memory is still resident, so a swap legitimately counts twice. + const terminating = tracked.terminate(); + await Promise.resolve(); + expect(registry.committedProcessCount).toBe(1); + + child.emit('exit', 0, null); + await terminating; + expect(registry.committedProcessCount).toBe(0); + }); + it('classifies an error without a pid as no process', async () => { const registry = new ProcessRegistry(); const child = fakeChild(undefined); diff --git a/packages/acp-bridge/src/process-registry.ts b/packages/acp-bridge/src/process-registry.ts index df6d0c71a86..4191a3f1ac1 100644 --- a/packages/acp-bridge/src/process-registry.ts +++ b/packages/acp-bridge/src/process-registry.ts @@ -79,6 +79,22 @@ export class ProcessRegistry { get activeProcessCount(): number { return this.children.size; } + + /** + * Children this registry has committed to: attached ones plus reservations + * that have not attached yet. Larger than {@link activeProcessCount}, and + * the right figure for admission — `reserve()` inserts its token + * synchronously before `spawn()`, so two racing spawns each see the other + * here, while neither is visible in `activeProcessCount` until its child is + * attached. + * + * A child leaves this count when it *exits*, not when `terminate()` starts, + * so a channel swap is counted twice while the old process is still winding + * down. That is deliberate: its memory is still resident. + */ + get committedProcessCount(): number { + return this.children.size + this.reservations.size; + } } class TrackedChild implements TrackedChildProcess { diff --git a/packages/acp-bridge/src/spawnChannel.test.ts b/packages/acp-bridge/src/spawnChannel.test.ts index 098d5986038..0291ea6f60d 100644 --- a/packages/acp-bridge/src/spawnChannel.test.ts +++ b/packages/acp-bridge/src/spawnChannel.test.ts @@ -35,6 +35,7 @@ import { EventEmitter } from 'node:events'; import type { ChildProcess } from 'node:child_process'; import { PassThrough } from 'node:stream'; +import { getHeapStatistics } from 'node:v8'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; const mockSpawn = vi.hoisted(() => vi.fn()); @@ -527,4 +528,36 @@ describe('getAcpMemoryArgs', () => { expect(sizeMB).toBeLessThanOrEqual(16_384); } }); + + it('emits an explicit share even far below this process own heap limit', () => { + // THE regression guard for #8182. The no-argument path emits the flag only + // when it would RAISE the child above the spawning process's own limit. A + // budget-derived share is normally well below it — 614 MB against a + // multi-GB test runner — so routing it through that guard would drop the + // flag, silently restore the 25x overcommit, and break nothing else. If + // this assertion ever goes soft, the fix is gone. + const currentLimitMb = Math.floor( + getHeapStatistics().heap_size_limit / (1024 * 1024), + ); + expect(614).toBeLessThan(currentLimitMb); + expect(getAcpMemoryArgs(614)).toEqual([ + '--max-old-space-size=614', + '--expose-gc', + ]); + }); + + it('keeps the explicit path out of the module cache, in both directions', () => { + // The share depends on how many children are live right now, so caching it + // would pin the first spawn's answer for the process lifetime. Asserting + // both directions is what makes a cache-reset hook unnecessary. + const first = getAcpMemoryArgs(1_024); + const second = getAcpMemoryArgs(2_048); + expect(first).toEqual(['--max-old-space-size=1024', '--expose-gc']); + expect(second).toEqual(['--max-old-space-size=2048', '--expose-gc']); + + // And it neither poisons nor is poisoned by the cached default. + const derived = getAcpMemoryArgs(); + expect(derived).not.toContain('--max-old-space-size=2048'); + expect(getAcpMemoryArgs()).toBe(derived); + }); }); diff --git a/packages/acp-bridge/src/spawnChannel.ts b/packages/acp-bridge/src/spawnChannel.ts index 7940725bf58..6e4a8332ca4 100644 --- a/packages/acp-bridge/src/spawnChannel.ts +++ b/packages/acp-bridge/src/spawnChannel.ts @@ -15,7 +15,27 @@ import { MissingCliEntryError } from './status.js'; import { ProcessRegistry } from './process-registry.js'; let cachedMemoryArgs: string[] | undefined; -export function getAcpMemoryArgs(): string[] { +/** + * V8 flags for a spawned ACP child. + * + * With no argument this is the historical behaviour: half of cgroup/host + * memory, capped at 16 GB, emitted only when it would *raise* the child above + * the spawning process's own heap limit, and cached for the process lifetime. + * Single-child callers — the interactive CLI, the IDE companion, direct-embed + * bridges — want exactly that and are unchanged. + * + * `explicitMb` is the daemon's budget-derived share, and deliberately bypasses + * **both** the cache and the raise-only guard. The cache, because the share + * depends on how many children are live right now rather than on the host. The + * guard, because a budget-derived share is normally *below* the daemon's own + * heap limit — so passing it through `targetMB > currentLimitMB` would drop + * the flag, silently restore the overcommit, and leave every test green. That + * failure mode is the whole reason this parameter exists; see #8182. + */ +export function getAcpMemoryArgs(explicitMb?: number): string[] { + if (explicitMb !== undefined) { + return [`--max-old-space-size=${explicitMb}`, '--expose-gc']; + } if (cachedMemoryArgs) return cachedMemoryArgs; const constrainedMemory = (process as { constrainedMemory?: () => number }) .constrainedMemory; diff --git a/packages/cli/src/serve/acp-http/dispatch.ts b/packages/cli/src/serve/acp-http/dispatch.ts index 457005d1071..c25b9e29587 100644 --- a/packages/cli/src/serve/acp-http/dispatch.ts +++ b/packages/cli/src/serve/acp-http/dispatch.ts @@ -747,6 +747,19 @@ export function toRpcError(err: unknown): { retryable: true, }, }; + case 'ChildHeapPoolExhaustedError': + return { + code: RPC.INTERNAL_ERROR, + message: errMsg(err), + data: { + errorKind: 'child_heap_pool_exhausted', + childPoolMb: (err as { childPoolMb?: unknown }).childPoolMb, + concurrentChildren: (err as { concurrentChildren?: unknown }) + .concurrentChildren, + httpStatus: 503, + retryable: true, + }, + }; case 'TotalSessionLimitExceededError': return { code: RPC.INTERNAL_ERROR, diff --git a/packages/cli/src/serve/acp-session-bridge.ts b/packages/cli/src/serve/acp-session-bridge.ts index dbec1b27e91..6f820a8bc3f 100644 --- a/packages/cli/src/serve/acp-session-bridge.ts +++ b/packages/cli/src/serve/acp-session-bridge.ts @@ -125,6 +125,7 @@ export { WorkspaceDrainingError, InvalidRewindTargetError, TotalSessionLimitExceededError, + ChildHeapPoolExhaustedError, NOT_CURRENTLY_GENERATING_CANCEL_MESSAGE, // Multi-client permission coordination errors. CancelSentinelCollisionError, diff --git a/packages/cli/src/serve/server/error-response.ts b/packages/cli/src/serve/server/error-response.ts index 70b5cedfd69..e98a44d7f07 100644 --- a/packages/cli/src/serve/server/error-response.ts +++ b/packages/cli/src/serve/server/error-response.ts @@ -49,6 +49,7 @@ import { WorkspaceMismatchError, WorkspaceDrainingError, TotalSessionLimitExceededError, + ChildHeapPoolExhaustedError, } from '../acp-session-bridge.js'; import type { DaemonLogger } from '../daemon-logger.js'; import { @@ -496,6 +497,26 @@ export function sendBridgeError( }); return; } + if (err instanceof ChildHeapPoolExhaustedError) { + daemonLog?.warn('child heap pool exhausted', { + ...(ctx?.route ? { route: ctx.route } : {}), + ...(ctx?.sessionId ? { sessionId: ctx.sessionId } : {}), + childPoolMb: err.childPoolMb, + concurrentChildren: err.concurrentChildren, + minChildHeapMb: err.minChildHeapMb, + }); + // Retryable in the same sense as the session limit above: the condition + // clears when any child exits, which needs no operator action. + res.set('Retry-After', '5'); + res.status(503).json({ + error: err.message, + code: 'child_heap_pool_exhausted', + childPoolMb: err.childPoolMb, + concurrentChildren: err.concurrentChildren, + minChildHeapMb: err.minChildHeapMb, + }); + return; + } if (err instanceof TotalSessionLimitExceededError) { const totalSessionError = err as TotalSessionLimitExceededError & { operation?: string; From ecf35d31eda975ad16190785b9784fc2b830f179 Mon Sep 17 00:00:00 2001 From: jinye Date: Tue, 4 Aug 2026 11:09:25 +0800 Subject: [PATCH 2/6] feat(serve): size each ACP child by concurrently live children MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wires the primitives from the previous commit into the spawn path, behind `--child-heap-mode off | observe | enforce`, default `observe`. Under `enforce` a child's `--max-old-space-size` is a share of the child pool divided by the children concurrently committed at the moment it spawns — read from the shared ProcessRegistry after `reserve()`, so two racing spawns each see the other. When the pool cannot cover another child at the 512 MB floor the spawn is refused with ChildHeapPoolExhaustedError, which is what turns a per-child ceiling into an aggregate bound: concurrent children can never exceed pool/512. Keyed on concurrency, never on registrations. A dormant workspace has no child, so it costs nothing — the specific correction #8182 records against the withdrawn proposal, which would have shrunk a lone live child to 614 MB because of 24 idle registrations. Default `observe` computes the share and the admission decision and applies neither, counting the refusals that would have happened. The divisor has never been checked against a real multi-workspace deployment, and a non-zero count is how an operator learns enforcement would have broken them without being broken. It also catches the case worth worrying about: a channel swap counts the dying child alongside its replacement, so on a saturated pool enforcement could refuse a restart and leave that workspace with no child at all. Excluding terminating children would authorise real overcommit to dodge a hypothetical refusal, so the count reports it instead. Ceilings already granted are not revisited — V8 cannot lower them — so granted ceilings transiently exceed the pool. Acceptable: the flag is a ceiling, not a reservation, and a workspace with no live sessions has no child and picks up the current share on its next spawn. `limits.memory.enforced` stops being a required literal `false`. #8245 made it one so a client could never mistake that namespace for enforcement that had not shipped; it has now, so the field is a boolean derived from the mode — and stays `false` under `observe`, which applies nothing. Refs #8182. Co-Authored-By: Claude Opus 5 --- packages/acp-bridge/package.json | 4 + packages/acp-bridge/src/spawnChannel.test.ts | 112 ++++++++++++++++++ packages/acp-bridge/src/spawnChannel.ts | 44 ++++++- packages/cli/src/commands/serve.test.ts | 28 +++++ packages/cli/src/commands/serve.ts | 17 +++ packages/cli/src/serve/daemon-status.ts | 43 +++++-- packages/cli/src/serve/fast-path.test.ts | 18 +++ packages/cli/src/serve/fast-path.ts | 18 +++ .../cli/src/serve/routes/daemon-status.ts | 3 + packages/cli/src/serve/run-qwen-serve.ts | 24 ++++ packages/cli/src/serve/server.ts | 3 + packages/cli/src/serve/types.ts | 20 ++++ packages/sdk-typescript/src/daemon/types.ts | 21 +++- 13 files changed, 346 insertions(+), 9 deletions(-) diff --git a/packages/acp-bridge/package.json b/packages/acp-bridge/package.json index e43d2ff0f23..ec3336f647d 100644 --- a/packages/acp-bridge/package.json +++ b/packages/acp-bridge/package.json @@ -99,6 +99,10 @@ "types": "./dist/daemon-memory-budget.d.ts", "import": "./dist/daemon-memory-budget.js" }, + "./childHeapPolicy": { + "types": "./dist/child-heap-policy.d.ts", + "import": "./dist/child-heap-policy.js" + }, "./channelControlTimeouts": { "types": "./dist/channel-control-timeouts.d.ts", "import": "./dist/channel-control-timeouts.js" diff --git a/packages/acp-bridge/src/spawnChannel.test.ts b/packages/acp-bridge/src/spawnChannel.test.ts index 0291ea6f60d..5115d822373 100644 --- a/packages/acp-bridge/src/spawnChannel.test.ts +++ b/packages/acp-bridge/src/spawnChannel.test.ts @@ -36,6 +36,13 @@ import { EventEmitter } from 'node:events'; import type { ChildProcess } from 'node:child_process'; import { PassThrough } from 'node:stream'; import { getHeapStatistics } from 'node:v8'; +import { ProcessRegistry } from './process-registry.js'; +import { createChildHeapPolicy } from './child-heap-policy.js'; +import { + MIN_CHILD_HEAP_MB, + resolveDaemonMemoryBudget, +} from './daemon-memory-budget.js'; +import { ChildHeapPoolExhaustedError } from './bridgeErrors.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; const mockSpawn = vi.hoisted(() => vi.fn()); @@ -230,6 +237,111 @@ describe('createSpawnChannelFactory env policy', () => { }); }); +describe('createSpawnChannelFactory child-heap admission', () => { + const originalArgv1 = process.argv[1]; + // Big enough that several children fit — otherwise the very first spawn + // sits on the refusal boundary and there is no shrink to observe — but + // small enough that the boundary is still reachable in a few spawns. + const budget = resolveDaemonMemoryBudget({ availableMemoryMb: 8_192 }); + + beforeEach(() => { + mockSpawn.mockReset(); + mockSpawn.mockReturnValue(createFakeChildProcess()); + process.argv[1] = '/tmp/qwen.js'; + process.env['QWEN_CLI_ENTRY'] = '/tmp/qwen.js'; + }); + afterEach(() => { + process.argv[1] = originalArgv1; + delete process.env['QWEN_CLI_ENTRY']; + }); + + const heapArg = () => { + const argv = mockSpawn.mock.calls[0]?.[1] as string[] | undefined; + return argv?.find((a) => a.startsWith('--max-old-space-size=')); + }; + + it('applies the share under enforce and shrinks it as children accumulate', async () => { + const processRegistry = new ProcessRegistry(); + const factory = createSpawnChannelFactory({ + processRegistry, + childHeapPolicy: createChildHeapPolicy({ budget, mode: 'enforce' }), + }); + + await factory('/tmp/a'); + const first = Number(heapArg()!.split('=')[1]); + // First child alone gets the whole pool, capped by the legacy ceiling. + expect(first).toBe( + Math.min(budget.childPoolMb, budget.legacyChildCeilingMb), + ); + + mockSpawn.mockClear(); + await factory('/tmp/b'); + const second = Number(heapArg()!.split('=')[1]); + // Two live children now, so the second is sized for two — the whole point + // of keying on concurrency rather than on the host. + expect(second).toBeLessThan(first); + }); + + it('computes but applies nothing under observe', async () => { + const policy = createChildHeapPolicy({ budget, mode: 'observe' }); + const observeRegistry = new ProcessRegistry(); + await createSpawnChannelFactory({ + processRegistry: observeRegistry, + childHeapPolicy: policy, + })('/tmp/a'); + const observed = mockSpawn.mock.calls[0]?.[1] as string[]; + + mockSpawn.mockClear(); + const bareRegistry = new ProcessRegistry(); + await createSpawnChannelFactory({ processRegistry: bareRegistry })( + '/tmp/a', + ); + const bare = mockSpawn.mock.calls[0]?.[1] as string[]; + + // Byte-identical argv: passing --max-old-space-size changes child GC and + // OOM behaviour, which a reporting mode must not do. + expect(observed).toEqual(bare); + }); + + it('refuses under enforce once the pool cannot cover another child, and keeps the slot', async () => { + const processRegistry = new ProcessRegistry(); + const policy = createChildHeapPolicy({ budget, mode: 'enforce' }); + const factory = createSpawnChannelFactory({ + processRegistry, + childHeapPolicy: policy, + }); + + const fits = Math.floor(budget.childPoolMb / MIN_CHILD_HEAP_MB); + for (let i = 0; i < fits; i++) await factory(`/tmp/w${i}`); + expect(processRegistry.committedProcessCount).toBe(fits); + + await expect(factory('/tmp/over')).rejects.toBeInstanceOf( + ChildHeapPoolExhaustedError, + ); + // The refused spawn released its reservation. Leaking it would inflate + // every later count until the daemon refused everything. + expect(processRegistry.committedProcessCount).toBe(fits); + expect(policy.snapshot().refusals).toBe(1); + }); + + it('never refuses under observe, but counts what enforce would have', async () => { + const processRegistry = new ProcessRegistry(); + const policy = createChildHeapPolicy({ budget, mode: 'observe' }); + const factory = createSpawnChannelFactory({ + processRegistry, + childHeapPolicy: policy, + }); + + const fits = Math.floor(budget.childPoolMb / MIN_CHILD_HEAP_MB); + for (let i = 0; i < fits + 2; i++) await factory(`/tmp/w${i}`); + + // Everything spawned, and the counter is the calibration signal that + // enforcing here would have failed two real spawns. + expect(processRegistry.committedProcessCount).toBe(fits + 2); + expect(policy.snapshot()).toMatchObject({ enforced: false, refusals: 2 }); + }); +}); + describe('createStderrForwarder', () => { it('calls onDiagnosticLine for each complete line', () => { const captured: Array<{ line: string; level?: string }> = []; diff --git a/packages/acp-bridge/src/spawnChannel.ts b/packages/acp-bridge/src/spawnChannel.ts index 6e4a8332ca4..9e040745f57 100644 --- a/packages/acp-bridge/src/spawnChannel.ts +++ b/packages/acp-bridge/src/spawnChannel.ts @@ -13,6 +13,8 @@ import { redactLogCredentials } from './logRedaction.js'; import { ndJsonStream, type NdJsonStreamHooks } from './ndJsonStream.js'; import { MissingCliEntryError } from './status.js'; import { ProcessRegistry } from './process-registry.js'; +import type { ChildHeapPolicy } from './child-heap-policy.js'; +import { ChildHeapPoolExhaustedError } from './bridgeErrors.js'; let cachedMemoryArgs: string[] | undefined; /** @@ -127,6 +129,17 @@ export interface SpawnChannelFactoryOptions { pipeHooks?: NdJsonStreamHooks; sourceEnv?: Readonly; processRegistry?: ProcessRegistry; + /** + * Daemon child-heap policy. Only meaningful together with a **shared** + * `processRegistry`: the factory otherwise builds its own, every spawn sees + * a concurrent count of 1, and each child is handed the whole pool — the + * current overcommit, now with a policy object attesting to it. All three + * daemon factories pass the same registry. + * + * Omitted by every single-child caller (interactive CLI, IDE companion, + * direct-embed), which keeps the host-derived ceiling. + */ + childHeapPolicy?: ChildHeapPolicy; } /** @@ -155,11 +168,40 @@ export function createSpawnChannelFactory( ); childEnv['QWEN_CODE_NO_RELAUNCH'] = 'true'; - const memoryArgs = getAcpMemoryArgs(); const execArgs = process.execArgv.filter( (a) => !/^--inspect(-brk)?($|=)/.test(a), ); + // Reserve BEFORE deciding: the reservation is what makes this spawn + // visible to any other spawn racing it, so the count below includes this + // child and two concurrent spawns cannot both be told they are alone. const reservation = processRegistry.reserve(); + let memoryArgs: string[]; + try { + const policy = options.childHeapPolicy; + // Read the count once, while this reservation is still held, so the + // figure decided on and the figure reported are the same number. + const concurrentChildren = processRegistry.committedProcessCount; + const decision = policy?.decide(concurrentChildren); + const enforced = policy?.snapshot().enforced ?? false; + if (decision?.refuse && enforced) { + const { childPoolMb, minChildHeapMb } = policy!.snapshot(); + throw new ChildHeapPoolExhaustedError( + childPoolMb, + concurrentChildren, + minChildHeapMb, + ); + } + // `observe` computed a share above and must not apply it: passing the + // flag changes the child's GC and OOM behaviour, which is not something + // a reporting mode may do. Only `enforce` reaches the child. + memoryArgs = getAcpMemoryArgs(enforced ? decision?.ceilingMb : undefined); + } catch (error) { + // Covers both the refusal above and anything the policy throws. Leaking + // the reservation would inflate the count for every later spawn until + // the daemon refused everything. + reservation.cancel(); + throw error; + } let child; try { child = spawn( diff --git a/packages/cli/src/commands/serve.test.ts b/packages/cli/src/commands/serve.test.ts index 452e620a29a..b4a1cee177d 100644 --- a/packages/cli/src/commands/serve.test.ts +++ b/packages/cli/src/commands/serve.test.ts @@ -342,6 +342,34 @@ describe('serve rate limit env parsing', () => { ); }); + it('passes --child-heap-mode to runQwenServe', async () => { + mockRunQwenServe.mockResolvedValueOnce({ + url: 'http://127.0.0.1:4170/', + webShellMounted: false, + }); + + await startServeHandlerWithArgs('--no-web --child-heap-mode enforce'); + + expect(mockRunQwenServe).toHaveBeenCalledWith( + expect.objectContaining({ childHeapMode: 'enforce' }), + ); + }); + + it('defaults the child heap mode to observe, never enforce', async () => { + mockRunQwenServe.mockResolvedValueOnce({ + url: 'http://127.0.0.1:4170/', + webShellMounted: false, + }); + + await startServeHandlerWithArgs('--no-web'); + + // The default is the safety property of this whole feature: enforcement + // must never switch itself on for a daemon that did not ask. + expect(mockRunQwenServe).toHaveBeenCalledWith( + expect.objectContaining({ childHeapMode: 'observe' }), + ); + }); + it('defaults the memory pressure mode to observe', async () => { mockRunQwenServe.mockResolvedValueOnce({ url: 'http://127.0.0.1:4170/', diff --git a/packages/cli/src/commands/serve.ts b/packages/cli/src/commands/serve.ts index b14cb0b0db7..80e7b615e66 100644 --- a/packages/cli/src/commands/serve.ts +++ b/packages/cli/src/commands/serve.ts @@ -129,6 +129,7 @@ interface ServeArgs { 'mcp-client-budget'?: number; 'memory-budget-mb'?: number; 'memory-pressure-mode'?: 'off' | 'observe'; + 'child-heap-mode'?: 'off' | 'observe' | 'enforce'; 'mcp-budget-mode'?: 'enforce' | 'warn' | 'off'; 'allow-origin'?: string[]; 'allow-private-auth-base-url': boolean; @@ -346,6 +347,21 @@ export const serveCommand: CommandModule = { 'if you alert on the top-level status. Nothing remediates in ' + 'either mode.', }) + .option('child-heap-mode', { + choices: ['off', 'observe', 'enforce'] as const, + default: 'observe' as const, + description: + 'What the daemon does with the per-child heap share it derives ' + + 'from the memory budget. `observe` (default) computes the share ' + + 'and the spawn-admission decision, applies neither, and reports ' + + 'how many spawns would have been refused — use that count to ' + + 'decide whether `enforce` is safe for your deployment. `enforce` ' + + 'passes the share to each `qwen --acp` child and refuses a spawn ' + + 'when the child pool cannot cover another at the minimum heap; ' + + 'that refusal surfaces as a failure to open a new session in the ' + + 'affected workspace, and clears when any child exits. `off` ' + + 'computes nothing and leaves children on the host-derived ceiling.', + }) .option('mcp-client-budget', { type: 'number', description: @@ -659,6 +675,7 @@ export const serveCommand: CommandModule = { mcpBudgetMode: resolvedMcpMode, ...(memoryBudgetMb !== undefined ? { memoryBudgetMb } : {}), memoryPressureMode: argv['memory-pressure-mode'], + childHeapMode: argv['child-heap-mode'], ...(argv['allow-origin'] && argv['allow-origin'].length > 0 ? { allowOrigins: argv['allow-origin'] } : {}), diff --git a/packages/cli/src/serve/daemon-status.ts b/packages/cli/src/serve/daemon-status.ts index 272d1dc7c2a..5d40b3b947c 100644 --- a/packages/cli/src/serve/daemon-status.ts +++ b/packages/cli/src/serve/daemon-status.ts @@ -23,6 +23,7 @@ import { recommendedChildShareMb, type DaemonMemoryBudget, } from '@qwen-code/acp-bridge/daemonMemoryBudget'; +import type { ChildHeapPolicySnapshot } from '@qwen-code/acp-bridge/childHeapPolicy'; import { computeDaemonMemoryPressure, type DaemonMemoryPressure, @@ -126,6 +127,8 @@ export interface BuildDaemonStatusOptions { getPerfSnapshot?: () => DaemonPerfSnapshot; getMetricsSeries?: () => DaemonMetricsBucket[]; getTotalSessionAdmissionSnapshot?: () => TotalSessionAdmissionSnapshot; + /** Returns undefined when no policy was built — direct-embed, or no budget. */ + getChildHeapPolicySnapshot?: () => ChildHeapPolicySnapshot | undefined; } interface DaemonStatusSection { @@ -190,12 +193,29 @@ interface DaemonStatusLimits { export interface DaemonStatusMemoryLimits { /** - * False, and required. Every figure in this section is resolved input or a - * model of a policy that does not exist yet; nothing here is applied to a - * process. The flag exists so a client can never mistake the `limits` - * namespace for enforcement that has not shipped. + * Whether a spawn argument actually derives from these numbers — i.e. only + * under `--child-heap-mode enforce`. + * + * This was a required literal `false` while the whole section was a model of + * a policy that had not shipped. It is a boolean now because the policy has, + * and it stays narrow on purpose: `observe` computes every figure below and + * applies none of them, so it still reports `false`. A client must be able + * to read this as "children are being sized by this", never as "the feature + * exists". */ - enforced: false; + enforced: boolean; + /** How the derived per-child share is used. `null` when no policy was built. */ + childHeap: { + mode: 'off' | 'observe' | 'enforce'; + /** + * Spawns refused, or — under `observe` — that would have been refused. + * The calibration signal for whether `enforce` is safe here: non-zero + * means enforcement would have failed a real spawn. Includes the + * channel-swap case, where a replacement child is counted alongside the + * process it is replacing. + */ + refusals: number; + } | null; /** What was asked for: the flag value, or half of available memory. */ configuredBudgetMb: number; /** `configured` capped at resolved cgroup/host memory. */ @@ -226,10 +246,16 @@ export interface DaemonStatusMemoryLimits { export function toDaemonStatusMemoryLimits( budget: DaemonMemoryBudget | undefined, + childHeap?: ChildHeapPolicySnapshot, ): DaemonStatusMemoryLimits | null { if (!budget) return null; return { - enforced: false, + // Derived, never hardcoded: the whole point of the field is that a client + // can trust it to track what the daemon actually does. + enforced: childHeap?.enforced ?? false, + childHeap: childHeap + ? { mode: childHeap.mode, refusals: childHeap.refusals } + : null, configuredBudgetMb: budget.configuredBudgetMb, effectiveBudgetMb: budget.effectiveBudgetMb, budgetSource: budget.budgetSource, @@ -801,7 +827,10 @@ export async function buildDaemonStatusResponse( channelIdleTimeoutMs: bridgeSnapshot.limits.channelIdleTimeoutMs, sessionIdleTimeoutMs: bridgeSnapshot.limits.sessionIdleTimeoutMs, acpConnectionCap: acpSnapshot?.connectionCap ?? null, - memory: toDaemonStatusMemoryLimits(memoryBudget), + memory: toDaemonStatusMemoryLimits( + memoryBudget, + input.getChildHeapPolicySnapshot?.(), + ), }, ...(workspaceRuntimes && workspaceRuntimes.length > 1 ? { diff --git a/packages/cli/src/serve/fast-path.test.ts b/packages/cli/src/serve/fast-path.test.ts index d0c8d7bdc18..03ed7413032 100644 --- a/packages/cli/src/serve/fast-path.test.ts +++ b/packages/cli/src/serve/fast-path.test.ts @@ -687,6 +687,7 @@ describe('serve fast path argument parsing', () => { ['http-bridge', ['--no-http-bridge']], ['memory-budget-mb', ['--memory-budget-mb', '8192']], ['memory-pressure-mode', ['--memory-pressure-mode', 'observe']], + ['child-heap-mode', ['--child-heap-mode', 'observe']], ['mcp-client-budget', ['--mcp-client-budget', '10']], ['mcp-budget-mode', ['--mcp-budget-mode', 'warn']], ['allow-origin', ['--allow-origin', 'http://localhost:3000']], @@ -767,6 +768,23 @@ describe('serve fast path argument parsing', () => { ).toEqual({ kind: 'fallback' }); }); + it('parses --child-heap-mode and falls back on an unknown value', () => { + for (const argv of [ + ['serve', '--child-heap-mode', 'enforce'], + ['serve', '--child-heap-mode=enforce'], + ]) { + expect(parseServeFastPathArgs(argv)).toMatchObject({ + kind: 'serve', + options: { childHeapMode: 'enforce' }, + }); + } + // Unlike memory-pressure-mode, `enforce` is a real value here — so the + // rejected sample has to be something else entirely. + expect( + parseServeFastPathArgs(['serve', '--child-heap-mode', 'warn']), + ).toEqual({ kind: 'fallback' }); + }); + it('parses --memory-budget-mb on the fast path in both spellings', () => { for (const argv of [ ['serve', '--memory-budget-mb', '8192'], diff --git a/packages/cli/src/serve/fast-path.ts b/packages/cli/src/serve/fast-path.ts index 501cfaaff17..dd106cc5d8f 100644 --- a/packages/cli/src/serve/fast-path.ts +++ b/packages/cli/src/serve/fast-path.ts @@ -421,6 +421,24 @@ export function parseServeFastPathArgs( continue; } + if (flag === 'child-heap-mode') { + const read = readOptionValue(argv, i, inlineValue); + if (!read) return { kind: 'fallback' }; + i = read.nextIndex; + // Same reasoning as memory-pressure-mode: yargs `choices` already owns + // the error message for a bad value, and letting an unknown string past + // here would put a value in `ServeOptions` its own type forbids. + if ( + read.value !== 'off' && + read.value !== 'observe' && + read.value !== 'enforce' + ) { + return { kind: 'fallback' }; + } + options.childHeapMode = read.value; + continue; + } + if (flag === 'mcp-budget-mode') { const read = readOptionValue(argv, i, inlineValue); if (!read) return { kind: 'fallback' }; diff --git a/packages/cli/src/serve/routes/daemon-status.ts b/packages/cli/src/serve/routes/daemon-status.ts index 2c4bc19943b..252e89a498a 100644 --- a/packages/cli/src/serve/routes/daemon-status.ts +++ b/packages/cli/src/serve/routes/daemon-status.ts @@ -28,6 +28,7 @@ import type { DaemonWorkspaceService } from '../workspace-service/index.js'; import { getServeProtocolVersions } from '../capabilities.js'; import type { TotalSessionAdmissionSnapshot } from '../total-session-admission.js'; import type { WorkspaceRegistry } from '../workspace-registry.js'; +import type { ChildHeapPolicySnapshot } from '@qwen-code/acp-bridge/childHeapPolicy'; interface RegisterDaemonStatusRoutesDeps { opts: ServeOptions; @@ -52,6 +53,7 @@ interface RegisterDaemonStatusRoutesDeps { getPerfSnapshot?: () => DaemonPerfSnapshot; getMetricsSeries?: () => DaemonMetricsBucket[]; getTotalSessionAdmissionSnapshot?: () => TotalSessionAdmissionSnapshot; + getChildHeapPolicySnapshot?: () => ChildHeapPolicySnapshot | undefined; } export function registerDaemonStatusRoutes( @@ -92,6 +94,7 @@ export function registerDaemonStatusRoutes( getMetricsSeries: deps.getMetricsSeries, getTotalSessionAdmissionSnapshot: deps.getTotalSessionAdmissionSnapshot, + getChildHeapPolicySnapshot: deps.getChildHeapPolicySnapshot, }), ); } catch (err) { diff --git a/packages/cli/src/serve/run-qwen-serve.ts b/packages/cli/src/serve/run-qwen-serve.ts index e37fa7b774c..dda9c18c5a8 100644 --- a/packages/cli/src/serve/run-qwen-serve.ts +++ b/packages/cli/src/serve/run-qwen-serve.ts @@ -45,6 +45,10 @@ import { formatMemoryBudgetStderr, resolveDaemonMemoryBudget, } from '@qwen-code/acp-bridge/daemonMemoryBudget'; +import { + createChildHeapPolicy, + type ChildHeapPolicy, +} from '@qwen-code/acp-bridge/childHeapPolicy'; import { canonicalizeWorkspace, translateAndCheckAbsoluteWorkspacePath, @@ -1602,6 +1606,9 @@ function createBootstrapServeApp(input: { channelIdleTimeoutMs: channelIdleTimeoutMs(opts.channelIdleTimeoutMs), sessionIdleTimeoutMs: sessionIdleTimeoutMs(opts.sessionIdleTimeoutMs), acpConnectionCap: null, + // No child-heap policy during bootstrap: it is built with the + // runtime, so `enforced` is correctly false and `childHeap` null in + // this window even when the flag says `enforce`. memory: toDaemonStatusMemoryLimits(opts.daemonMemoryBudget), }, capabilities: { @@ -2776,6 +2783,9 @@ async function runQwenServeImpl( killAllSync(): void; } | undefined; + // Held for daemon status: `observe` mode's whole product is the would-be + // refusal count, which is useless unless it can be read back out. + let managedChildHeapPolicy: ChildHeapPolicy | undefined; const internalRuntimeBridgesForCleanup: AcpSessionBridge[] = []; let daemonEventLoopMonitor: | ReturnType @@ -3484,6 +3494,16 @@ async function runQwenServeImpl( workspaceTrustOperationGate.runExclusive('runtime-topology', operation); const processRegistry = new runtime.ProcessRegistry(); managedProcessRegistry = processRegistry; + // One policy for the whole daemon, beside the one registry it reads. Both + // must be shared: a per-factory registry would report a concurrent count + // of 1 on every spawn and hand each child the entire pool. + const childHeapPolicy: ChildHeapPolicy | undefined = opts.daemonMemoryBudget + ? createChildHeapPolicy({ + budget: opts.daemonMemoryBudget, + mode: opts.childHeapMode ?? 'observe', + }) + : undefined; + managedChildHeapPolicy = childHeapPolicy; const fsFactory = runtime.resolveBridgeFsFactory({ // Secondary roots share a write-capable factory only after their own // folder trust check passes; untrusted secondary roots stay outside. @@ -3507,6 +3527,7 @@ async function runQwenServeImpl( }); const channelFactory = runtime.createSpawnChannelFactory({ processRegistry, + childHeapPolicy, sourceEnv: runtimeEffectiveEnv, onDiagnosticLine: diagnosticSink, pipeHooks: { @@ -4120,6 +4141,7 @@ async function runQwenServeImpl( }); const secondaryChannelFactory = runtime.createSpawnChannelFactory({ processRegistry, + childHeapPolicy, sourceEnv: secondaryEnv.effectiveEnv, onDiagnosticLine: diagnosticSink, pipeHooks: { @@ -4637,6 +4659,7 @@ async function runQwenServeImpl( : wsFsFactory; const wsChannelFactory = runtime.createSpawnChannelFactory({ processRegistry, + childHeapPolicy, sourceEnv: wsEnv.effectiveEnv, onDiagnosticLine: diagnosticSink, pipeHooks: { @@ -5373,6 +5396,7 @@ async function runQwenServeImpl( }), getMetricsSeries: () => metricsRing.snapshot(), getTotalSessionAdmissionSnapshot: totalSessionAdmission.snapshot, + getChildHeapPolicySnapshot: () => managedChildHeapPolicy?.snapshot(), recordDaemonRequest: (durationMs, statusCode) => metricsRing.recordRequest(durationMs, statusCode), workspace: workspaceService, diff --git a/packages/cli/src/serve/server.ts b/packages/cli/src/serve/server.ts index f002223c768..fbf7e9fd061 100644 --- a/packages/cli/src/serve/server.ts +++ b/packages/cli/src/serve/server.ts @@ -258,6 +258,7 @@ import { } from '../commands/channel/config-utils.js'; import { loadChannelsConfig } from '../commands/channel/runtime.js'; import { writeStderrLine } from '../utils/stdioHelpers.js'; +import type { ChildHeapPolicySnapshot } from '@qwen-code/acp-bridge/childHeapPolicy'; export { createDefaultFsAuditEmit, @@ -484,6 +485,7 @@ export interface ServeAppDeps { /** Rolling metrics series for the Daemon Status charts (oldest→newest). */ getMetricsSeries?: () => DaemonMetricsBucket[]; getTotalSessionAdmissionSnapshot?: () => TotalSessionAdmissionSnapshot; + getChildHeapPolicySnapshot?: () => ChildHeapPolicySnapshot | undefined; /** * Sink fed one (durationMs, statusCode) per matched daemon HTTP request, so * the metrics ring can bucket request rate and latency for the charts. @@ -1334,6 +1336,7 @@ export function createServeApp( getMetricsSeries: deps.getMetricsSeries, getTotalSessionAdmissionSnapshot: deps.getTotalSessionAdmissionSnapshot ?? totalSessionAdmission?.snapshot, + getChildHeapPolicySnapshot: deps.getChildHeapPolicySnapshot, }); registerCapabilitiesRoutes(app, { diff --git a/packages/cli/src/serve/types.ts b/packages/cli/src/serve/types.ts index 2ab7772a94c..aca1ce44f2e 100644 --- a/packages/cli/src/serve/types.ts +++ b/packages/cli/src/serve/types.ts @@ -245,6 +245,26 @@ export interface ServeOptions { * enforcement. */ memoryPressureMode?: 'off' | 'observe'; + /** + * What the daemon does with the per-child heap share it derives from + * `memoryBudgetMb`. + * + * `observe` (default) computes the share and the spawn-admission decision + * and applies **neither**, counting the refusals that would have happened. + * That counter is the point: the divisor has never been checked against a + * real multi-workspace deployment, and a non-zero count says enforcement + * would have refused a real spawn — including the channel-swap case, where + * a replacement child is counted alongside the one it replaces. + * + * `enforce` passes the share to the child and refuses the spawn when the + * pool cannot cover another. Unlike `memoryPressureMode`, this mode is + * included from the start because it is reachable and testable; the default + * is what keeps it safe. + * + * `off` computes nothing and leaves children on the historical host-derived + * ceiling. + */ + childHeapMode?: 'off' | 'observe' | 'enforce'; /** * Resolved at boot by `runQwenServe`. Not an operator input, and not * consumed by any spawn path — it is reported under `limits.memory` on diff --git a/packages/sdk-typescript/src/daemon/types.ts b/packages/sdk-typescript/src/daemon/types.ts index ff1534a4dca..cbd827cc469 100644 --- a/packages/sdk-typescript/src/daemon/types.ts +++ b/packages/sdk-typescript/src/daemon/types.ts @@ -628,7 +628,26 @@ export interface DaemonStatusReport { */ memory?: { /** Always false: nothing in this section is applied to a process. */ - enforced: false; + /** + * Whether children are actually being sized by these numbers — true only + * under `--child-heap-mode enforce`. Was a required literal `false` while + * the section described a policy that had not shipped; now a boolean, and + * still `false` under `observe`, which computes everything and applies + * nothing. Read it as "this is in effect", never as "this exists". + */ + enforced: boolean; + /** + * How the derived per-child heap share is used. `null` when the daemon + * built no policy, and absent entirely on daemons predating the field. + */ + childHeap?: { + mode: 'off' | 'observe' | 'enforce'; + /** + * Spawns refused, or under `observe` that would have been refused — + * the signal for whether enabling `enforce` is safe on this host. + */ + refusals: number; + } | null; configuredBudgetMb: number; effectiveBudgetMb: number; budgetSource: 'flag' | 'derived'; From cfd7e1032fb588b74f1b69ec2216bd643333f270 Mon Sep 17 00:00:00 2001 From: jinye Date: Tue, 4 Aug 2026 11:34:08 +0800 Subject: [PATCH 3/6] docs(serve): correct the claims child-heap enforcement makes false MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two sentences in the protocol doc described the memory section as unconditionally observational: "a required `enforced: false`", and "no child spawn argument derives from these values, and no request is refused on their basis". Both are false under `--child-heap-mode enforce`, so both are rewritten rather than left to rot — `enforced` is now documented as the boolean that answers exactly this, and the refusal is documented with its wire shape on both transports. Also documents `childHeap.refusals` as the calibration signal, since a would-be-refusal count is useless if operators do not know to read it before switching to `enforce`; the flag row in the three operator docs; and the design doc's Part 1, which listed applying a share as a compatibility risk without recording how that was resolved. The end-to-end test asserts the policy reaches a real booted daemon's status with `enforced: false` under the default mode — the wire type in that test is a hand-written mirror, so its `enforced: false` literal had to widen too, which is the check that caught the type not being widened everywhere. Refs #8182. Co-Authored-By: Claude Opus 5 --- ...-07-31-daemon-capacity-model-and-memory-bounds.md | 4 ++++ docs/developers/daemon/17-configuration.md | 1 + docs/developers/daemon/20-quickstart-operations.md | 1 + docs/developers/qwen-serve-protocol.md | 2 +- docs/users/qwen-serve.md | 1 + packages/acp-bridge/src/spawnChannel.ts | 12 +++++++----- packages/cli/src/serve/run-qwen-serve.test.ts | 12 +++++++++++- 7 files changed, 26 insertions(+), 7 deletions(-) diff --git a/docs/design/2026-07-31-daemon-capacity-model-and-memory-bounds.md b/docs/design/2026-07-31-daemon-capacity-model-and-memory-bounds.md index 099c2026589..78988b7d97d 100644 --- a/docs/design/2026-07-31-daemon-capacity-model-and-memory-bounds.md +++ b/docs/design/2026-07-31-daemon-capacity-model-and-memory-bounds.md @@ -113,6 +113,10 @@ The real control is admission at spawn time keyed on concurrently live children, - **It must never raise a ceiling.** Clamping to `legacyChildCeilingMb` is what makes the policy safe to apply unconditionally; without it the minimum-budget constant and an over-large explicit flag both inflate the share. - **The spawn path has a trap.** `getAcpMemoryArgs()` emits `--max-old-space-size` only when its computed target exceeds the _spawning daemon's own_ `heap_size_limit` (`spawnChannel.ts:27-34`). A budget-derived share is normally below that, so a naive change is silently dropped and the overcommit returns. The regression test must assert the flag survives a value below the test process's own limit. +**Shipped as `--child-heap-mode`.** The share is keyed on children _concurrently committed_ at spawn time — read from the shared `ProcessRegistry` after `reserve()`, so racing spawns see each other — never on registrations, which allocate nothing. The trap above is handled by an explicit-value parameter that bypasses both the module cache and the raise-only guard, with a regression test asserting a 614 MB share survives against a multi-GB runner. + +The compatibility point is why the default is `observe`: it computes the share and the admission decision, applies neither, and counts the refusals that would have happened. Enforcement is a deliberate operator action taken once that count shows it is safe — notably for channel swaps, where the dying child is counted alongside its replacement and a saturated pool could otherwise refuse a restart. `enforce` also supplies the aggregate bound a per-child ceiling alone cannot: refusing once the pool cannot cover another child at the floor caps concurrent children at `childPoolMb / 512`. + ### Part 2 — Observe, with a denominator, before enforcing This part splits by what each piece measures, because the denominators are independent and the cheap one is worth landing first. diff --git a/docs/developers/daemon/17-configuration.md b/docs/developers/daemon/17-configuration.md index b7d4bf58f96..376df2a2b09 100644 --- a/docs/developers/daemon/17-configuration.md +++ b/docs/developers/daemon/17-configuration.md @@ -23,6 +23,7 @@ This page collects every setting that affects the `qwen serve` daemon and its ad | `--compacted-replay-max-bytes ` | positive integer | `4194304` | Byte cap for the bounded in-memory replay snapshot returned by `POST /session/:id/load`; hard cap is `268435456`. | | `--memory-budget-mb ` | integer in `[1024, 1048576]` | 50% of cgroup-constrained or host memory, capped at the flag maximum (1048576 MB) | Total memory budget for the daemon process tree, capped at resolved available memory. Observed and reported under `limits.memory` in daemon status; it does not size any child process. Boot rejects out-of-range values. | | `--memory-pressure-mode ` | `off` \| `observe` | `observe` | Whether the daemon derives a memory-pressure level from its own RSS and V8 heap. Both modes report `runtime.memory.pressure`; only `observe` raises `daemon_memory_pressure`. Root process only; no remediation. | +| `--child-heap-mode ` | `off \| observe \| enforce` | `observe` | How the per-child heap share derived from the budget is used. `enforce` sizes each child by the live child count and refuses a spawn the pool cannot cover; `observe` computes both but applies neither. | | `--http-bridge` | boolean | `true` | Stage 1 bridge mode. `--no-http-bridge` still falls back to http-bridge and prints to stderr. | | `--mcp-client-budget ` | positive integer | unset | Sets `WorkspaceMcpBudget.clientBudget` and forwards it to the ACP child through `childEnvOverrides`. | | `--mcp-budget-mode ` | `off` / `warn` / `enforce` | `warn` when budget is set, otherwise `off` | Sets `WorkspaceMcpBudget.mode`; `enforce` requires `--mcp-client-budget`. | diff --git a/docs/developers/daemon/20-quickstart-operations.md b/docs/developers/daemon/20-quickstart-operations.md index e52fd74ce77..0a6a8e2f26d 100644 --- a/docs/developers/daemon/20-quickstart-operations.md +++ b/docs/developers/daemon/20-quickstart-operations.md @@ -82,6 +82,7 @@ The CLI is defined in **`packages/cli/src/commands/serve.ts`**: | `--max-total-sessions ` | number | derived for multiple startup/restored workspaces | - | Daemon-wide active session cap. When omitted, a finite default is derived once from the per-workspace cap and startup/restored workspace count; dynamic registration does not recompute it. `0` means unlimited. | | `--memory-budget-mb ` | integer in `[1024, 1048576]` | 50% of cgroup/host memory | Observation only | Total memory budget for the daemon process tree, capped at resolved available memory. Reported under `limits.memory`; does not size any child. | | `--memory-pressure-mode ` | `off` \| `observe` | `observe` | Observation only | Reports `runtime.memory.pressure` in both modes; only `observe` raises the `daemon_memory_pressure` issue. Root process only. | +| `--child-heap-mode ` | `off \| observe \| enforce` | `observe` | Enforcing only under `enforce` | Sizes each ACP child by concurrent child count; refuses spawns the pool cannot cover. `observe` applies neither and counts would-be refusals. | | `--max-pending-prompts-per-session ` | number | `5` | - | Accepted but pending/running prompt cap per session. Excess prompt returns 503. `0` / `Infinity` means unlimited. Negative or non-integer values throw. | | `--workspace ` | string / repeatable | `process.cwd()` | - | Startup workspace runtime; repeat to register additional isolated runtimes. The first is primary. Each value **must be an absolute path, must exist, and must be a directory**. Boot canonicalizes every value via `canonicalizeWorkspace`. `POST /session` with a mismatched `cwd` returns `400 workspace_mismatch`. | | `--max-connections ` | number | `256` | - | Listener-level `server.maxConnections`. `0` / `Infinity` means unlimited. `NaN` / negative values fail boot to avoid fail-open behavior. | diff --git a/docs/developers/qwen-serve-protocol.md b/docs/developers/qwen-serve-protocol.md index 00c2b239ffa..29a7283d53a 100644 --- a/docs/developers/qwen-serve-protocol.md +++ b/docs/developers/qwen-serve-protocol.md @@ -618,7 +618,7 @@ runtime routes return `503`. `runtime.activity` reports daemon-wide prompt activity. `activePrompts` counts sessions with an in-flight prompt. `pendingPrompts` counts all accepted prompts that have not settled yet, including the running prompt and FIFO-waiting prompts. `queuedPrompts` counts FIFO-waiting prompts that have been accepted but not dispatched. `lastActivityAt` is the ISO 8601 timestamp of the last prompt start/end or session spawn; `null` when the daemon has never processed any activity since boot. `idleSinceMs` is computed from `lastActivityAt` at response generation time. -`limits.memory` is additive and reports the daemon's resolved memory figures: a required `enforced: false`, `configuredBudgetMb`, `effectiveBudgetMb` (the configured value capped at resolved cgroup/host memory), `budgetSource` (`flag` / `derived`), `availableMemoryMb`, `availableMemorySource` (`constrained` / `host`), `insufficientMemory`, and a `modeled` object holding `rootReserveMb`, `childPoolMb`, `minChildHeapMb`, `maxChildHeapMb`, and `legacyChildCeilingMb` (a conservative model of the ceiling an ACP child receives today, which can sit below the real figure). `runtime.memory` additionally reports `registeredWorkspaces` (the registration count — non-removed workspace entries, including draining, transitioning, or blocked ones; not a live-child count), `activeAcpChildren` (daemon-managed ACP children with a live, non-dying channel — includes transitioning or blocked entries, but excludes a workspace whose kill has started even if the child has not exited; not channel workers, MCP descendants, or unattached spawn reservations), `childRssCoverage` (`active_children` — every ACP child with a live channel, which is the set `activeAcpChildren` counts; older daemons send `primary_only`), a `children` object described below, and a `modeled` object holding `recommendedShareAtRegisteredMb` (`null` when no workspace is registered) and `recommendedShareAtActiveMb` (`null` when no child is active). Each share is capped at the legacy child ceiling, and floored at the minimum child heap only when the ceiling allows — on a small host the ceiling sits below the floor, so share × count can exceed the child pool. Read a share as advisory, not a partition of the pool. All of it is observation: no child spawn argument derives from these values, and no request is refused on their basis. On the normal `runQwenServe` path the budget is resolved before the bootstrap app is created, so `limits.memory` is already populated during the bootstrap window. It is `null` only on paths that resolve no budget (such as direct-embed bypassing `runQwenServeImpl`). The SDK type allows `null`, so correct clients cope. +`limits.memory` is additive and reports the daemon's resolved memory figures: `enforced` (a boolean now, not the literal `false` it was while nothing applied these figures — it is `true` only under `--child-heap-mode enforce`, and stays `false` under `observe`, which computes everything and applies nothing), a `childHeap` object (`mode`, and `refusals` — spawns refused, or under `observe` that _would_ have been refused, which is the signal for whether enabling `enforce` is safe on a given host; `null` when the daemon built no policy), `configuredBudgetMb`, `effectiveBudgetMb` (the configured value capped at resolved cgroup/host memory), `budgetSource` (`flag` / `derived`), `availableMemoryMb`, `availableMemorySource` (`constrained` / `host`), `insufficientMemory`, and a `modeled` object holding `rootReserveMb`, `childPoolMb`, `minChildHeapMb`, `maxChildHeapMb`, and `legacyChildCeilingMb` (a conservative model of the ceiling an ACP child receives today, which can sit below the real figure). `runtime.memory` additionally reports `registeredWorkspaces` (the registration count — non-removed workspace entries, including draining, transitioning, or blocked ones; not a live-child count), `activeAcpChildren` (daemon-managed ACP children with a live, non-dying channel — includes transitioning or blocked entries, but excludes a workspace whose kill has started even if the child has not exited; not channel workers, MCP descendants, or unattached spawn reservations), `childRssCoverage` (`active_children` — every ACP child with a live channel, which is the set `activeAcpChildren` counts; older daemons send `primary_only`), a `children` object described below, and a `modeled` object holding `recommendedShareAtRegisteredMb` (`null` when no workspace is registered) and `recommendedShareAtActiveMb` (`null` when no child is active). Each share is capped at the legacy child ceiling, and floored at the minimum child heap only when the ceiling allows — on a small host the ceiling sits below the floor, so share × count can exceed the child pool. Read a share as advisory, not a partition of the pool. Whether any of it is applied is exactly what `enforced` reports. Under `off` and `observe` it remains pure observation: no child spawn argument derives from these values and no request is refused on their basis. Under `enforce` a spawning ACP child's `--max-old-space-size` is the child pool divided by the children concurrently committed at that moment, and a spawn is refused with `child_heap_pool_exhausted` (REST 503 with `Retry-After`, ACP `errorKind: 'child_heap_pool_exhausted'`, retryable) once the pool cannot cover another child at `modeled.minChildHeapMb`. That refusal surfaces as a failure to open a new session in the affected workspace — registration is never refused, because registration allocates nothing — and clears when any child exits. On the normal `runQwenServe` path the budget is resolved before the bootstrap app is created, so `limits.memory` is already populated during the bootstrap window. It is `null` only on paths that resolve no budget (such as direct-embed bypassing `runQwenServeImpl`). The SDK type allows `null`, so correct clients cope. `runtime.memory.children` is additive within that block and reports aggregate RSS across the children `childRssCoverage` names: `rssBytes` (their summed self-reported RSS), `sampled` (how many produced a reading), and `oldestReadingAgeMs` (the age of the oldest reading in the sum, so a caller can tell how far apart its parts were taken). The denominator for `sampled` is the sibling `activeAcpChildren`, not repeated inside the block; when `sampled` is lower, `rssBytes` is a floor rather than a total. Sampling is gated on an active SSE/WS watcher, so a status request against a daemon nobody is streaming from reports `sampled: 0` even with live children — `activeAcpChildren` beside it makes that gap visible, and `rssBytes: 0` with `sampled: 0` never means a measured zero. `oldestReadingAgeMs` is `null` when nothing was sampled and also when every contributor is a bridge predating the field, so it never means "fresh". Read the sum as an over-count and an under-count at once: summing per-process RSS double-counts pages the children share, while each child reports only its own process, so its MCP descendants and every channel worker are missing. It is not the daemon tree's memory. The field is optional in the SDK mirror because daemons reporting `primary_only` never send it. diff --git a/docs/users/qwen-serve.md b/docs/users/qwen-serve.md index 2d85e663681..8f631118ed0 100644 --- a/docs/users/qwen-serve.md +++ b/docs/users/qwen-serve.md @@ -397,6 +397,7 @@ Notes: | `--max-connections ` | `256` | Listener-level TCP connection cap (`server.maxConnections`). Bounds raw socket count irrespective of session count — slow / phantom SSE clients get rejected at accept time once full. Raise alongside `--max-sessions` if your deployment expects many SSE subscribers per session. | | `--memory-budget-mb ` | 50% of cgroup/host | Total memory budget in MB for the whole daemon process tree. When unset, derived as 50% of the cgroup limit or host memory; either way the effective value is capped at resolved available memory, and both the configured and effective figures are reported. Currently observation only — it does not change how any `qwen --acp` child is sized. Resolved figures appear under `limits.memory` in `GET /daemon/status`, alongside registered and live child counts and advisory per-child shares under `runtime.memory`. A host too small for the minimum reports `insufficientMemory` rather than being clamped upward; because the derived fraction is 50%, any host under ~2 GB trips this. Pass an explicit `--memory-budget-mb 1024` on such a host to override the derived figure (the flag still requires at least 1024 MB of available memory to clear the warning). Must be an integer in `[1024, 1048576]`. | | `--memory-pressure-mode ` | `observe` | Whether the daemon turns its own memory reading into a verdict. `observe` (default) reports the pressure level under `runtime.memory.pressure` in `GET /daemon/status` and raises a `daemon_memory_pressure` issue — a `warning`, so the overall `status` leaves `ok` — whenever the level leaves `normal`. `off` still reports every figure, including the level, but raises no issue, so the overall `status` is unchanged; use it while calibrating, or if you alert on the top-level status. The level is the worse of two ratios: RSS against available memory (what the cgroup OOM killer watches) and V8 heap used against this process's heap ceiling. It covers the daemon root process only; compare it against `runtime.memory.children.rssBytes` for the children. Nothing remediates in either mode. One of `off`, `observe`. | +| `--child-heap-mode ` | `observe` | What the daemon does with the per-child heap share it derives from `--memory-budget-mb`. `enforce` gives each `qwen --acp` child a `--max-old-space-size` equal to the child pool divided by the children live when it spawns, and refuses a spawn once the pool cannot cover another child at the 512 MB minimum — that refusal appears as a failure to open a new session in that workspace, and clears when any child exits. `observe` (default) computes the same share and the same decision but applies **neither**, reporting how many spawns would have been refused under `limits.memory.childHeap.refusals`; confirm that count is zero before switching to `enforce`. `off` computes nothing. Registration is never refused — a registered workspace with no session has no child. | | `--event-ring-size ` | `8000` | Per-session SSE replay ring depth (#3803 §02 target). Sets the backlog available to `GET /session/:id/events` with `Last-Event-ID: N`. Larger = more reconnect headroom at the cost of a few hundred KB extra RAM per session. SDK clients can additionally request a larger per-subscriber backlog cap on a specific subscription via `?maxQueued=N` (range `[16, 2048]`, default 256). Daemons also emit a non-terminal `slow_client_warning` SSE frame at 75% queue fill so clients can drain / reconnect before getting evicted. Pre-flight `caps.features.slow_client_warning`. | | `--compacted-replay-max-bytes ` | `4194304` | Per-live-session byte cap for the retained replay events in the bounded snapshot returned by `POST /session/:id/load`. The cap applies to `compactedReplay`; the current in-flight `liveJournal` is separately capped by `--max-journal-events` and `--max-journal-bytes`. Values must be positive safe integers; invalid values fail at boot, and the hard ceiling is 256 MiB. When older retained replay is dropped, the snapshot begins with `history_truncated`. This does not limit the on-disk transcript. | | `--max-journal-events ` | `10000` | Per-session cap on the number of raw events retained in the in-flight live journal (the current unfinished turn). When exceeded, the oldest journal entries are dropped and a `history_truncated` marker is prepended. Must be a positive safe integer. | diff --git a/packages/acp-bridge/src/spawnChannel.ts b/packages/acp-bridge/src/spawnChannel.ts index 9e040745f57..9c6cb5d0a4d 100644 --- a/packages/acp-bridge/src/spawnChannel.ts +++ b/packages/acp-bridge/src/spawnChannel.ts @@ -182,13 +182,15 @@ export function createSpawnChannelFactory( // figure decided on and the figure reported are the same number. const concurrentChildren = processRegistry.committedProcessCount; const decision = policy?.decide(concurrentChildren); - const enforced = policy?.snapshot().enforced ?? false; - if (decision?.refuse && enforced) { - const { childPoolMb, minChildHeapMb } = policy!.snapshot(); + // One snapshot for the whole decision, so the mode that gates the + // refusal is the same mode that gates whether the share is applied. + const snapshot = policy?.snapshot(); + const enforced = snapshot?.enforced ?? false; + if (snapshot && enforced && decision?.refuse) { throw new ChildHeapPoolExhaustedError( - childPoolMb, + snapshot.childPoolMb, concurrentChildren, - minChildHeapMb, + snapshot.minChildHeapMb, ); } // `observe` computed a share above and must not apply it: passing the diff --git a/packages/cli/src/serve/run-qwen-serve.test.ts b/packages/cli/src/serve/run-qwen-serve.test.ts index 2d2331bbed5..32a38c2f37e 100644 --- a/packages/cli/src/serve/run-qwen-serve.test.ts +++ b/packages/cli/src/serve/run-qwen-serve.test.ts @@ -1992,7 +1992,11 @@ describe('runQwenServe memory budget', () => { const body = (await res.json()) as { limits: { memory: { - enforced: false; + enforced: boolean; + childHeap: { + mode: string; + refusals: number; + } | null; configuredBudgetMb: number; effectiveBudgetMb: number; budgetSource: string; @@ -2042,6 +2046,12 @@ describe('runQwenServe memory budget', () => { const memory = body.limits.memory; expect(memory).not.toBeNull(); + // The child-heap policy reached status on a daemon that really booted. + // Default is `observe`, so it computed a share and applied nothing — + // `enforced` has to stay false or the field means "the feature exists" + // rather than "children are being sized by this". + expect(memory?.enforced).toBe(false); + expect(memory?.childHeap).toEqual({ mode: 'observe', refusals: 0 }); // Nothing in this section is applied, and the wire says so. expect(memory?.enforced).toBe(false); expect(memory?.configuredBudgetMb).toBe(4096); From 5e0a8da794de8144302cfe4e57043ce1c802d329 Mon Sep 17 00:00:00 2001 From: jinye Date: Tue, 4 Aug 2026 11:35:47 +0800 Subject: [PATCH 4/6] test(serve): cover both branches of the enforced tripwire MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `enforced` was only ever asserted false — the unit tests build no policy and the end-to-end daemon runs the default `observe` mode, so the branch that makes the field worth having was untested. Hardcoding it back to `false` passed everything. Also pins `childHeap: null` as distinct from a policy in `off` mode: the first says no policy exists (direct-embed, or the bootstrap window before the runtime is built), the second says one exists and computes nothing. Refs #8182. Co-Authored-By: Claude Opus 5 --- packages/cli/src/serve/daemon-status.test.ts | 47 ++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/packages/cli/src/serve/daemon-status.test.ts b/packages/cli/src/serve/daemon-status.test.ts index 38af7276bb1..23334a1f277 100644 --- a/packages/cli/src/serve/daemon-status.test.ts +++ b/packages/cli/src/serve/daemon-status.test.ts @@ -25,6 +25,7 @@ import type { ChannelWorkerSnapshot } from './channel-worker-supervisor.js'; import type { RateLimiterInstance, RateLimitTier } from './rate-limit.js'; import type { DaemonWorkspaceService } from './workspace-service/index.js'; import type { DaemonLogger } from './daemon-logger.js'; +import { createChildHeapPolicy } from '@qwen-code/acp-bridge/childHeapPolicy'; import { resolveDaemonMemoryBudget } from '@qwen-code/acp-bridge/daemonMemoryBudget'; const BASE_WORKSPACE = '/work/status'; @@ -130,6 +131,52 @@ describe('buildDaemonStatusResponse', () => { expect(response.limits.maxTotalSessions).toBe(50); }); + it('reports enforced only when a spawn argument really derives from the budget', async () => { + // The tripwire field. #8245 made it a required literal `false` so a client + // could never mistake this section for enforcement that had not shipped; + // it is a boolean now, and the two branches must be checked separately or + // it degrades into "the feature exists". + const budget = resolveDaemonMemoryBudget({ availableMemoryMb: 8_192 }); + + const observing = makeOptions(); + observing.opts.daemonMemoryBudget = budget; + observing.getChildHeapPolicySnapshot = () => + createChildHeapPolicy({ budget, mode: 'observe' }).snapshot(); + const observed = await buildDaemonStatusResponse('summary', observing); + // Computes everything, applies nothing — so `false`, not `true`. + expect(observed.limits.memory).toMatchObject({ + enforced: false, + childHeap: { mode: 'observe', refusals: 0 }, + }); + + const enforcing = makeOptions(); + enforcing.opts.daemonMemoryBudget = budget; + const policy = createChildHeapPolicy({ budget, mode: 'enforce' }); + // Drive one refusal through so the counter is not trivially zero here. + policy.decide(10_000); + enforcing.getChildHeapPolicySnapshot = () => policy.snapshot(); + const enforced = await buildDaemonStatusResponse('summary', enforcing); + expect(enforced.limits.memory).toMatchObject({ + enforced: true, + childHeap: { mode: 'enforce', refusals: 1 }, + }); + }); + + it('reports no child-heap policy as null rather than as a disabled one', async () => { + // Direct-embed and the bootstrap window build no policy. `null` says + // "there is no policy", which a client must not read as "mode off". + const options = makeOptions(); + options.opts.daemonMemoryBudget = resolveDaemonMemoryBudget({ + availableMemoryMb: 8_192, + }); + const response = await buildDaemonStatusResponse('summary', options); + + expect(response.limits.memory).toMatchObject({ + enforced: false, + childHeap: null, + }); + }); + it('reports the resolved memory budget in daemon status limits', () => { const options = makeOptions(); options.opts.daemonMemoryBudget = resolveDaemonMemoryBudget({ From c0dc50a7f06f3427042dc26cc08bb39a64085392 Mon Sep 17 00:00:00 2001 From: jinye Date: Tue, 4 Aug 2026 14:38:21 +0800 Subject: [PATCH 5/6] fix(serve): partition the child pool so granted ceilings stay inside it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review was right that the previous design did not deliver the aggregate bound it claimed. Sizing each child by the count live at *its* spawn bounds the child count but not the memory: V8 cannot lower a running child's ceiling, so grants accumulate as P + P/2 + P/3 + ... = P x H(n). Reproduced exactly — 9557 MB authorised against a 3687 MB pool at seven children on an 8 GB host, and 61355 MB against 15360 MB at the limit on 32 GB. That is 2.6x and 4x the pool, which is what the policy exists to prevent. Grant accounting alone does not fix it: the first child would take the whole pool and the second would be refused immediately. Keeping the invariant requires early children not to receive the whole pool, so the ceiling is now a fixed partition — childPoolMb / maxConcurrentChildren, constant for every child, with maxConcurrentChildren itself derived from the pool and capped at MAX_DAEMON_WORKSPACES. The sum is then n x ceiling <= pool by construction, with no ledger of outstanding grants and no dependence on arrival order. Tested as an invariant across four host sizes: fill the daemon to its admission limit and the authorised total still fits. The cost is deliberate and now documented rather than hidden: a lone workspace on a 32 GB host gets 614 MB rather than the pool, because any child may still be running when the house fills. An 8 GB host admits seven concurrent children at 526 MB each. Also from review: - The policy is no longer built for an injected `deps.bridge`. That bridge carries its own channel and never reaches the factory the policy rides on, so status could report `enforced: true` while nothing was being sized. - Both transport mappings now have direct tests. They are hand-written beside each other and drift silently; the spawn-policy tests cannot catch a wire regression. - Swept the "does not size any child" claim, which enforce makes false, out of the CLI help text, ServeOptions docs, the two operator tables, and the e2e header comment. The 17-configuration table realigns wholesale because that cell was its widest — whitespace only. Refs #8182. Co-Authored-By: Claude Opus 5 --- docs/developers/daemon/17-configuration.md | 70 ++++----- .../daemon/20-quickstart-operations.md | 2 +- .../acp-bridge/src/child-heap-policy.test.ts | 134 +++++++++++------- packages/acp-bridge/src/child-heap-policy.ts | 55 +++++-- packages/acp-bridge/src/spawnChannel.test.ts | 20 +-- packages/cli/src/commands/serve.ts | 4 +- .../src/serve/acp-http/dispatch-error.test.ts | 15 ++ packages/cli/src/serve/run-qwen-serve.test.ts | 3 +- packages/cli/src/serve/run-qwen-serve.ts | 17 ++- .../src/serve/server/error-response.test.ts | 33 +++++ packages/cli/src/serve/types.ts | 3 +- 11 files changed, 232 insertions(+), 124 deletions(-) diff --git a/docs/developers/daemon/17-configuration.md b/docs/developers/daemon/17-configuration.md index 376df2a2b09..ff7eec5bcfe 100644 --- a/docs/developers/daemon/17-configuration.md +++ b/docs/developers/daemon/17-configuration.md @@ -6,41 +6,41 @@ This page collects every setting that affects the `qwen serve` daemon and its ad ## CLI flags (`qwen serve`) -| Flag | Type | Default | Effect | -| --------------------------------------- | ---------------------------- | --------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `--hostname ` | string | `127.0.0.1` | Bind address. Loopback values: `127.0.0.1`, `localhost`, `::1`, `[::1]`. Non-loopback requires a bearer token at boot. `host:port` input is rejected with guidance to use `--port`. | -| `--port ` | number | `4170` | Listen port; `0` means ephemeral. | -| `--token ` | string | env | Bearer token. Overrides `QWEN_SERVER_TOKEN` and is trimmed at boot. It appears in the process command line, so prefer env in deployments. | -| `--require-auth` | boolean | `false` | Extends bearer auth to loopback and `/health`; boot refuses to start without a token. | -| `--workspace ` | absolute path / repeatable | `process.cwd()` | Startup workspace runtime; repeat to register additional isolated runtimes. The first is primary. Every value must be absolute and a directory; canonicalized at boot. | -| `--memory-project-scope ` | `git-root` / `workspace` | `git-root` | Project-memory partitioning. `git-root` shares memory among workspaces at the same Git root; `workspace` isolates by exact workspace directory. Overrides `QWEN_CODE_MEMORY_PROJECT_SCOPE`. | -| `--max-sessions ` | number | `32` | Per-workspace active session cap. `0` / `Infinity` means unlimited; `NaN` / negative values throw. | -| `--max-total-sessions ` | number | derived for multiple startup/restored workspaces | Daemon-wide active session cap. When omitted, a finite default is derived once from the per-workspace cap and startup/restored workspace count. `0` / `Infinity` means unlimited. | -| `--max-pending-prompts-per-session ` | number | `5` | Accepted but pending/running prompt cap per session. Excess prompt returns 503. `0` / `Infinity` means unlimited; negative or non-integer values throw. | -| `--max-connections ` | number | `256` | HTTP listener `server.maxConnections`; `0` / `Infinity` means unlimited. | -| `--enable-session-shell` | boolean | `false` | Enables direct `POST /session/:id/shell` execution. Requires bearer token, and every call must carry a session-bound `X-Qwen-Client-Id`. | -| `--event-ring-size ` | number | `8000` | Per-session SSE replay ring; soft cap is `1_000_000`. | -| `--compacted-replay-max-bytes ` | positive integer | `4194304` | Byte cap for the bounded in-memory replay snapshot returned by `POST /session/:id/load`; hard cap is `268435456`. | -| `--memory-budget-mb ` | integer in `[1024, 1048576]` | 50% of cgroup-constrained or host memory, capped at the flag maximum (1048576 MB) | Total memory budget for the daemon process tree, capped at resolved available memory. Observed and reported under `limits.memory` in daemon status; it does not size any child process. Boot rejects out-of-range values. | -| `--memory-pressure-mode ` | `off` \| `observe` | `observe` | Whether the daemon derives a memory-pressure level from its own RSS and V8 heap. Both modes report `runtime.memory.pressure`; only `observe` raises `daemon_memory_pressure`. Root process only; no remediation. | -| `--child-heap-mode ` | `off \| observe \| enforce` | `observe` | How the per-child heap share derived from the budget is used. `enforce` sizes each child by the live child count and refuses a spawn the pool cannot cover; `observe` computes both but applies neither. | -| `--http-bridge` | boolean | `true` | Stage 1 bridge mode. `--no-http-bridge` still falls back to http-bridge and prints to stderr. | -| `--mcp-client-budget ` | positive integer | unset | Sets `WorkspaceMcpBudget.clientBudget` and forwards it to the ACP child through `childEnvOverrides`. | -| `--mcp-budget-mode ` | `off` / `warn` / `enforce` | `warn` when budget is set, otherwise `off` | Sets `WorkspaceMcpBudget.mode`; `enforce` requires `--mcp-client-budget`. | -| `--allow-origin ` | repeatable string | unset | Cross-origin allowlist that replaces the default CORS denial. `*` allows any origin but requires a token. | -| `--allow-private-auth-base-url` | boolean | `false` | Allows `/workspace/auth/provider` to install localhost / private-network auth provider `baseUrl`; use only in trusted local development. | -| `--prompt-deadline-ms ` | positive integer | unset | Server-side prompt wallclock limit in ms. Timeout aborts and returns an error. | -| `--writer-idle-timeout-ms ` | positive integer | unset | Per-SSE-connection idle timeout in ms. The daemon closes the SSE connection when no event is sent for this duration. | -| `--channel-idle-timeout-ms ` | non-negative integer | `0` | How long to keep the ACP child alive after the last session closes. `0` means reclaim immediately. | -| `--initialize-timeout-ms ` | positive integer | `10000` | ACP child request timeout, including the initialize handshake (ms). | -| `--session-reap-interval-ms ` | non-negative integer | `60000` | Session reaper scan interval; `0` disables it. | -| `--session-idle-timeout-ms ` | non-negative integer | `1800000` | Disconnected-session idle reaping time; `0` disables it. | -| `--rate-limit` / `--no-rate-limit` | boolean | env / off | Enables per-tier HTTP rate limiting for prompt, mutation, and read routes. | -| `--rate-limit-prompt ` | positive integer | `10` | Prompt request limit per window; requires rate limiting to be enabled. | -| `--rate-limit-mutation ` | positive integer | `30` | Mutation request limit per window; requires rate limiting to be enabled. | -| `--rate-limit-read ` | positive integer | `120` | Read request limit per window; requires rate limiting to be enabled. | -| `--rate-limit-window-ms ` | integer `>= 1000` | `60000` | Rate limit window length; requires rate limiting to be enabled. | -| no flag | - | - | `QWEN_SERVE_NO_MCP_POOL=1` fully disables the pool. | +| Flag | Type | Default | Effect | +| --------------------------------------- | ---------------------------- | --------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `--hostname ` | string | `127.0.0.1` | Bind address. Loopback values: `127.0.0.1`, `localhost`, `::1`, `[::1]`. Non-loopback requires a bearer token at boot. `host:port` input is rejected with guidance to use `--port`. | +| `--port ` | number | `4170` | Listen port; `0` means ephemeral. | +| `--token ` | string | env | Bearer token. Overrides `QWEN_SERVER_TOKEN` and is trimmed at boot. It appears in the process command line, so prefer env in deployments. | +| `--require-auth` | boolean | `false` | Extends bearer auth to loopback and `/health`; boot refuses to start without a token. | +| `--workspace ` | absolute path / repeatable | `process.cwd()` | Startup workspace runtime; repeat to register additional isolated runtimes. The first is primary. Every value must be absolute and a directory; canonicalized at boot. | +| `--memory-project-scope ` | `git-root` / `workspace` | `git-root` | Project-memory partitioning. `git-root` shares memory among workspaces at the same Git root; `workspace` isolates by exact workspace directory. Overrides `QWEN_CODE_MEMORY_PROJECT_SCOPE`. | +| `--max-sessions ` | number | `32` | Per-workspace active session cap. `0` / `Infinity` means unlimited; `NaN` / negative values throw. | +| `--max-total-sessions ` | number | derived for multiple startup/restored workspaces | Daemon-wide active session cap. When omitted, a finite default is derived once from the per-workspace cap and startup/restored workspace count. `0` / `Infinity` means unlimited. | +| `--max-pending-prompts-per-session ` | number | `5` | Accepted but pending/running prompt cap per session. Excess prompt returns 503. `0` / `Infinity` means unlimited; negative or non-integer values throw. | +| `--max-connections ` | number | `256` | HTTP listener `server.maxConnections`; `0` / `Infinity` means unlimited. | +| `--enable-session-shell` | boolean | `false` | Enables direct `POST /session/:id/shell` execution. Requires bearer token, and every call must carry a session-bound `X-Qwen-Client-Id`. | +| `--event-ring-size ` | number | `8000` | Per-session SSE replay ring; soft cap is `1_000_000`. | +| `--compacted-replay-max-bytes ` | positive integer | `4194304` | Byte cap for the bounded in-memory replay snapshot returned by `POST /session/:id/load`; hard cap is `268435456`. | +| `--memory-budget-mb ` | integer in `[1024, 1048576]` | 50% of cgroup-constrained or host memory, capped at the flag maximum (1048576 MB) | Total memory budget for the daemon process tree, capped at resolved available memory. Reported under `limits.memory`; sizes each ACP child only under `--child-heap-mode enforce`. Boot rejects out-of-range values. | +| `--memory-pressure-mode ` | `off` \| `observe` | `observe` | Whether the daemon derives a memory-pressure level from its own RSS and V8 heap. Both modes report `runtime.memory.pressure`; only `observe` raises `daemon_memory_pressure`. Root process only; no remediation. | +| `--child-heap-mode ` | `off \| observe \| enforce` | `observe` | How the per-child heap share derived from the budget is used. `enforce` sizes each child by the live child count and refuses a spawn the pool cannot cover; `observe` computes both but applies neither. | +| `--http-bridge` | boolean | `true` | Stage 1 bridge mode. `--no-http-bridge` still falls back to http-bridge and prints to stderr. | +| `--mcp-client-budget ` | positive integer | unset | Sets `WorkspaceMcpBudget.clientBudget` and forwards it to the ACP child through `childEnvOverrides`. | +| `--mcp-budget-mode ` | `off` / `warn` / `enforce` | `warn` when budget is set, otherwise `off` | Sets `WorkspaceMcpBudget.mode`; `enforce` requires `--mcp-client-budget`. | +| `--allow-origin ` | repeatable string | unset | Cross-origin allowlist that replaces the default CORS denial. `*` allows any origin but requires a token. | +| `--allow-private-auth-base-url` | boolean | `false` | Allows `/workspace/auth/provider` to install localhost / private-network auth provider `baseUrl`; use only in trusted local development. | +| `--prompt-deadline-ms ` | positive integer | unset | Server-side prompt wallclock limit in ms. Timeout aborts and returns an error. | +| `--writer-idle-timeout-ms ` | positive integer | unset | Per-SSE-connection idle timeout in ms. The daemon closes the SSE connection when no event is sent for this duration. | +| `--channel-idle-timeout-ms ` | non-negative integer | `0` | How long to keep the ACP child alive after the last session closes. `0` means reclaim immediately. | +| `--initialize-timeout-ms ` | positive integer | `10000` | ACP child request timeout, including the initialize handshake (ms). | +| `--session-reap-interval-ms ` | non-negative integer | `60000` | Session reaper scan interval; `0` disables it. | +| `--session-idle-timeout-ms ` | non-negative integer | `1800000` | Disconnected-session idle reaping time; `0` disables it. | +| `--rate-limit` / `--no-rate-limit` | boolean | env / off | Enables per-tier HTTP rate limiting for prompt, mutation, and read routes. | +| `--rate-limit-prompt ` | positive integer | `10` | Prompt request limit per window; requires rate limiting to be enabled. | +| `--rate-limit-mutation ` | positive integer | `30` | Mutation request limit per window; requires rate limiting to be enabled. | +| `--rate-limit-read ` | positive integer | `120` | Read request limit per window; requires rate limiting to be enabled. | +| `--rate-limit-window-ms ` | integer `>= 1000` | `60000` | Rate limit window length; requires rate limiting to be enabled. | +| no flag | - | - | `QWEN_SERVE_NO_MCP_POOL=1` fully disables the pool. | ## Environment variables diff --git a/docs/developers/daemon/20-quickstart-operations.md b/docs/developers/daemon/20-quickstart-operations.md index 0a6a8e2f26d..aa3eaf564dd 100644 --- a/docs/developers/daemon/20-quickstart-operations.md +++ b/docs/developers/daemon/20-quickstart-operations.md @@ -80,7 +80,7 @@ The CLI is defined in **`packages/cli/src/commands/serve.ts`**: | `--token ` | string | env / none | Non-loopback and `--require-auth` | Bearer token; trimmed once. **It appears in `/proc//cmdline`, so prefer `QWEN_SERVER_TOKEN`**. Boot stderr also warns about this. | | `--max-sessions ` | number | `32` | - | Per-workspace active session cap. Excess spawn returns 503. `0` means unlimited. `NaN` / negative values throw. | | `--max-total-sessions ` | number | derived for multiple startup/restored workspaces | - | Daemon-wide active session cap. When omitted, a finite default is derived once from the per-workspace cap and startup/restored workspace count; dynamic registration does not recompute it. `0` means unlimited. | -| `--memory-budget-mb ` | integer in `[1024, 1048576]` | 50% of cgroup/host memory | Observation only | Total memory budget for the daemon process tree, capped at resolved available memory. Reported under `limits.memory`; does not size any child. | +| `--memory-budget-mb ` | integer in `[1024, 1048576]` | 50% of cgroup/host memory | Observation only | Total memory budget for the daemon process tree, capped at resolved available memory. Reported under `limits.memory`. Sizes children only under `--child-heap-mode enforce`. | | `--memory-pressure-mode ` | `off` \| `observe` | `observe` | Observation only | Reports `runtime.memory.pressure` in both modes; only `observe` raises the `daemon_memory_pressure` issue. Root process only. | | `--child-heap-mode ` | `off \| observe \| enforce` | `observe` | Enforcing only under `enforce` | Sizes each ACP child by concurrent child count; refuses spawns the pool cannot cover. `observe` applies neither and counts would-be refusals. | | `--max-pending-prompts-per-session ` | number | `5` | - | Accepted but pending/running prompt cap per session. Excess prompt returns 503. `0` / `Infinity` means unlimited. Negative or non-integer values throw. | diff --git a/packages/acp-bridge/src/child-heap-policy.test.ts b/packages/acp-bridge/src/child-heap-policy.test.ts index b3cf71c18b6..57debb7287b 100644 --- a/packages/acp-bridge/src/child-heap-policy.test.ts +++ b/packages/acp-bridge/src/child-heap-policy.test.ts @@ -11,52 +11,77 @@ import { resolveDaemonMemoryBudget, } from './daemon-memory-budget.js'; -// 8 GB of available memory, so the derived pool is a realistic size and the -// refusal boundary lands at a child count a real daemon could reach. -const budget = resolveDaemonMemoryBudget({ availableMemoryMb: 8_192 }); -const poolMb = budget.childPoolMb; -// The last count the pool can still cover at the floor, and the first it cannot. -const lastFitting = Math.floor(poolMb / MIN_CHILD_HEAP_MB); - describe('createChildHeapPolicy', () => { - it('refuses exactly when the pool can no longer cover another child', () => { - const policy = createChildHeapPolicy({ budget, mode: 'enforce' }); + it.each([2_048, 8_192, 32_768, 262_144])( + 'keeps the sum of every admissible ceiling inside the pool (%i MB host)', + (availableMemoryMb) => { + // THE invariant. A per-spawn share bounds the child count but not the + // memory — grants accumulate as P x H(n) because V8 cannot lower a + // running child's ceiling — so this asserts the property that claim + // actually needs: fill the daemon to its admission limit and the + // authorised total still fits the pool. + const b = resolveDaemonMemoryBudget({ availableMemoryMb }); + const policy = createChildHeapPolicy({ budget: b, mode: 'enforce' }); + const { maxConcurrentChildren, perChildCeilingMb } = policy.snapshot(); - expect(policy.decide(lastFitting).refuse).toBe(false); - expect(policy.decide(lastFitting + 1).refuse).toBe(true); + let granted = 0; + for (let n = 1; n <= maxConcurrentChildren; n++) { + const decision = policy.decide(n); + expect(decision.refuse).toBe(false); + granted += decision.ceilingMb!; + } + expect(granted).toBeLessThanOrEqual(b.childPoolMb); + expect(granted).toBe(maxConcurrentChildren * perChildCeilingMb); + // And the child past the limit is refused, which is what holds the sum. + expect(policy.decide(maxConcurrentChildren + 1).refuse).toBe(true); + }, + ); - // The boundary is the unclamped quotient, not the returned share. Past - // the boundary the share saturates at the floor and stops carrying any - // information: "barely does not fit" and "wildly does not fit" both read - // as 512, so a refusal derived from the share could never tell them apart. - expect(policy.decide(lastFitting).ceilingMb).toBeGreaterThan( - MIN_CHILD_HEAP_MB, - ); - expect(policy.decide(lastFitting + 1).ceilingMb).toBe(MIN_CHILD_HEAP_MB); - expect(policy.decide(lastFitting * 4).ceilingMb).toBe(MIN_CHILD_HEAP_MB); + it('hands every child the same ceiling regardless of how many are live', () => { + // The property the invariant rests on: a ceiling that shrank as children + // arrived would leave the early, larger grants outstanding and unbounded. + const b = resolveDaemonMemoryBudget({ availableMemoryMb: 8_192 }); + const policy = createChildHeapPolicy({ budget: b, mode: 'enforce' }); + const ceilings = [1, 2, 3, 7].map((n) => policy.decide(n).ceilingMb); + expect(new Set(ceilings).size).toBe(1); + expect(ceilings[0]).toBeGreaterThanOrEqual(MIN_CHILD_HEAP_MB); }); - it('shrinks the share as children arrive, and never below the floor', () => { - const policy = createChildHeapPolicy({ budget, mode: 'enforce' }); + it('sizes an 8 GB host for seven concurrent children', () => { + // Pinned deliberately: this is the number an operator plans against, and + // it is the cost of the invariant above — the eighth concurrent session + // is refused even though real RSS would likely have fit it. + const policy = createChildHeapPolicy({ + budget: resolveDaemonMemoryBudget({ availableMemoryMb: 8_192 }), + mode: 'enforce', + }); + expect(policy.snapshot()).toMatchObject({ + childPoolMb: 3_687, + maxConcurrentChildren: 7, + perChildCeilingMb: 526, + }); + }); - const one = policy.decide(1).ceilingMb!; - const two = policy.decide(2).ceilingMb!; - const four = policy.decide(4).ceilingMb!; - expect(one).toBeGreaterThan(two); - expect(two).toBeGreaterThan(four); - expect(four).toBeGreaterThanOrEqual(MIN_CHILD_HEAP_MB); - // Concurrency, not registration: this is a share of the pool at the count - // passed in, so it is capped by the legacy ceiling rather than the pool. - expect(one).toBeLessThanOrEqual(budget.legacyChildCeilingMb); + it('never admits more than the repository workspace maximum', () => { + // A large host divides by MAX_DAEMON_WORKSPACES rather than by + // pool/512, so the ceiling is 614 MB and not the 512 MB floor. + const policy = createChildHeapPolicy({ + budget: resolveDaemonMemoryBudget({ availableMemoryMb: 32_768 }), + mode: 'enforce', + }); + expect(policy.snapshot()).toMatchObject({ + maxConcurrentChildren: 25, + perChildCeilingMb: 614, + }); }); it('computes in observe mode but reports nothing as enforced', () => { - const observe = createChildHeapPolicy({ budget, mode: 'observe' }); + const b = resolveDaemonMemoryBudget({ availableMemoryMb: 8_192 }); + const observe = createChildHeapPolicy({ budget: b, mode: 'observe' }); + const over = observe.snapshot().maxConcurrentChildren + 1; - // The point of `observe`: the numbers exist, so a caller can be wired up - // and tested, while `enforced` stays false because nothing is applied. - const decision = observe.decide(lastFitting + 1); - expect(decision.ceilingMb).toBe(MIN_CHILD_HEAP_MB); + const decision = observe.decide(over); + expect(decision.ceilingMb).toBe(observe.snapshot().perChildCeilingMb); expect(decision.refuse).toBe(true); expect(observe.snapshot()).toMatchObject({ mode: 'observe', @@ -66,37 +91,38 @@ describe('createChildHeapPolicy', () => { }); it('counts would-be refusals so calibration does not need a broken deployment', () => { - const policy = createChildHeapPolicy({ budget, mode: 'observe' }); + const b = resolveDaemonMemoryBudget({ availableMemoryMb: 8_192 }); + const policy = createChildHeapPolicy({ budget: b, mode: 'observe' }); + const limit = policy.snapshot().maxConcurrentChildren; expect(policy.snapshot().refusals).toBe(0); policy.decide(1); - policy.decide(lastFitting); + policy.decide(limit); expect(policy.snapshot().refusals).toBe(0); - policy.decide(lastFitting + 1); - policy.decide(lastFitting + 9); + policy.decide(limit + 1); + policy.decide(limit + 9); expect(policy.snapshot().refusals).toBe(2); }); it('computes nothing at all when off', () => { - const off = createChildHeapPolicy({ budget, mode: 'off' }); + const b = resolveDaemonMemoryBudget({ availableMemoryMb: 8_192 }); + const off = createChildHeapPolicy({ budget: b, mode: 'off' }); // Not "a share of zero" — no share, so the caller keeps the historical - // host-derived ceiling. And an off policy must never accrue refusals, - // or the calibration counter would report on a daemon that never applied - // the policy in the first place. - expect(off.decide(lastFitting + 1)).toEqual({ - ceilingMb: undefined, - refuse: false, - }); + // host-derived ceiling. And an off policy must never accrue refusals. + expect(off.decide(9_999)).toEqual({ ceilingMb: undefined, refuse: false }); expect(off.snapshot()).toMatchObject({ enforced: false, refusals: 0 }); }); - it('treats a zero or negative count as one child', () => { - // Defensive: the caller reads a live count that should always include the - // spawn being admitted, but a 0 would otherwise divide the pool by zero. - const policy = createChildHeapPolicy({ budget, mode: 'enforce' }); - expect(policy.decide(0)).toEqual(policy.decide(1)); - expect(Number.isFinite(policy.decide(0).ceilingMb!)).toBe(true); + it('always admits at least one child, however small the pool', () => { + // A pool below the floor would otherwise divide to zero and refuse every + // spawn, bricking a daemon that works today. + const tiny = createChildHeapPolicy({ + budget: resolveDaemonMemoryBudget({ availableMemoryMb: 1_024 }), + mode: 'enforce', + }); + expect(tiny.snapshot().maxConcurrentChildren).toBeGreaterThanOrEqual(1); + expect(tiny.decide(1).refuse).toBe(false); }); }); diff --git a/packages/acp-bridge/src/child-heap-policy.ts b/packages/acp-bridge/src/child-heap-policy.ts index d89fe930af9..83d454aaf9b 100644 --- a/packages/acp-bridge/src/child-heap-policy.ts +++ b/packages/acp-bridge/src/child-heap-policy.ts @@ -6,9 +6,9 @@ import { MIN_CHILD_HEAP_MB, - recommendedChildShareMb, type DaemonMemoryBudget, } from './daemon-memory-budget.js'; +import { MAX_DAEMON_WORKSPACES } from './channel-control-timeouts.js'; /** * What the daemon does with the child-heap share it computes. @@ -42,6 +42,10 @@ export interface ChildHeapPolicySnapshot { enforced: boolean; childPoolMb: number; minChildHeapMb: number; + /** Children admitted concurrently. `perChildCeilingMb * this <= childPoolMb`. */ + maxConcurrentChildren: number; + /** The constant every admitted child receives. */ + perChildCeilingMb: number; /** * Spawns this policy refused, or would have refused under `enforce`. The * calibration signal: non-zero under `observe` means enforcement would have @@ -67,26 +71,45 @@ export function createChildHeapPolicy(options: { const { budget, mode } = options; let refusals = 0; + // A FIXED partition, not a share of the pool divided by the children live at + // this instant. The difference is the whole contract. + // + // A per-spawn share bounds the child *count* but not the memory: V8 cannot + // lower a running child's ceiling, so grants accumulate as + // P + P/2 + P/3 + ... = P x H(n) — 2.6x the pool at seven children on an + // 8 GB host, 4x at twenty-five on 32 GB. That authorises more old space than + // the host has, which is what this policy exists to stop. + // + // Holding the ceiling constant makes the sum n * ceiling, and admitting at + // most `maxConcurrentChildren` makes that <= childPoolMb by construction — + // no ledger of outstanding grants, and no dependence on the order children + // happened to arrive in. + // + // The cost is real and deliberate: a lone workspace on a 32 GB host gets + // 614 MB rather than the whole pool. Every admitted child is sized for a + // full house, because any child may still be running when the house fills. + const maxConcurrentChildren = Math.max( + 1, + Math.min( + Math.floor(budget.childPoolMb / MIN_CHILD_HEAP_MB), + MAX_DAEMON_WORKSPACES, + ), + ); + const perChildCeilingMb = Math.min( + Math.floor(budget.childPoolMb / maxConcurrentChildren), + budget.legacyChildCeilingMb, + ); + return { decide(concurrentChildren) { if (mode === 'off') return { ceilingMb: undefined, refuse: false }; - const children = Math.max(concurrentChildren, 1); - // `recommendedChildShareMb` clamps UP to MIN_CHILD_HEAP_MB, so past the - // point where the pool stops covering the count its answer saturates at - // 512 and can no longer say "will not fit" — a 600 MB pool split four - // ways still returns 512. Derive the refusal from the unclamped - // quotient, or the clamp silently authorises the very overcommit this - // policy exists to bound. - const refuse = - Math.floor(budget.childPoolMb / children) < MIN_CHILD_HEAP_MB; + // Refuse on the count, since the ceiling no longer varies with it. This + // is the only thing keeping the sum inside the pool. + const refuse = concurrentChildren > maxConcurrentChildren; if (refuse) refusals += 1; - return { - // Reported in both modes; only `enforce` lets the caller apply it. - ceilingMb: recommendedChildShareMb(budget, children), - refuse, - }; + return { ceilingMb: perChildCeilingMb, refuse }; }, snapshot() { @@ -95,6 +118,8 @@ export function createChildHeapPolicy(options: { enforced: mode === 'enforce', childPoolMb: budget.childPoolMb, minChildHeapMb: MIN_CHILD_HEAP_MB, + maxConcurrentChildren, + perChildCeilingMb, refusals, }; }, diff --git a/packages/acp-bridge/src/spawnChannel.test.ts b/packages/acp-bridge/src/spawnChannel.test.ts index 5115d822373..75546b8bfb6 100644 --- a/packages/acp-bridge/src/spawnChannel.test.ts +++ b/packages/acp-bridge/src/spawnChannel.test.ts @@ -260,26 +260,26 @@ describe('createSpawnChannelFactory child-heap admission', () => { return argv?.find((a) => a.startsWith('--max-old-space-size=')); }; - it('applies the share under enforce and shrinks it as children accumulate', async () => { + it('applies the same partitioned ceiling to every child under enforce', async () => { const processRegistry = new ProcessRegistry(); + const policy = createChildHeapPolicy({ budget, mode: 'enforce' }); const factory = createSpawnChannelFactory({ processRegistry, - childHeapPolicy: createChildHeapPolicy({ budget, mode: 'enforce' }), + childHeapPolicy: policy, }); + const { perChildCeilingMb } = policy.snapshot(); await factory('/tmp/a'); const first = Number(heapArg()!.split('=')[1]); - // First child alone gets the whole pool, capped by the legacy ceiling. - expect(first).toBe( - Math.min(budget.childPoolMb, budget.legacyChildCeilingMb), - ); - mockSpawn.mockClear(); await factory('/tmp/b'); const second = Number(heapArg()!.split('=')[1]); - // Two live children now, so the second is sized for two — the whole point - // of keying on concurrency rather than on the host. - expect(second).toBeLessThan(first); + + // Constant, not a share of the pool at this instant. A ceiling that shrank + // as children arrived would leave the earlier, larger grants outstanding + // and the authorised total unbounded — V8 cannot lower a running child. + expect(first).toBe(perChildCeilingMb); + expect(second).toBe(perChildCeilingMb); }); it('computes but applies nothing under observe', async () => { diff --git a/packages/cli/src/commands/serve.ts b/packages/cli/src/commands/serve.ts index 80e7b615e66..80b405fb574 100644 --- a/packages/cli/src/commands/serve.ts +++ b/packages/cli/src/commands/serve.ts @@ -332,7 +332,9 @@ export const serveCommand: CommandModule = { 'derived as 50% of cgroup-constrained ' + 'or host memory, and capped at the resolved available memory either ' + 'way. Currently observed and reported under `limits.memory` in daemon ' + - 'status; it does not yet size any child process. Must be an integer ' + + 'status. Under `--child-heap-mode enforce` it also sizes every ' + + '`qwen --acp` child and bounds how many run at once; under the ' + + 'default `observe` it sizes nothing. Must be an integer ' + 'in [1024, 1048576].', }) .option('memory-pressure-mode', { diff --git a/packages/cli/src/serve/acp-http/dispatch-error.test.ts b/packages/cli/src/serve/acp-http/dispatch-error.test.ts index ed0d4a1f7fa..4885f6447ef 100644 --- a/packages/cli/src/serve/acp-http/dispatch-error.test.ts +++ b/packages/cli/src/serve/acp-http/dispatch-error.test.ts @@ -6,6 +6,7 @@ import { describe, expect, it } from 'vitest'; import { DaemonDrainingError } from '../server/session-archive.js'; +import { ChildHeapPoolExhaustedError } from '../acp-session-bridge.js'; import { toRpcError } from './dispatch.js'; import { RPC } from './json-rpc.js'; @@ -18,4 +19,18 @@ describe('toRpcError', () => { data: { errorKind: 'daemon_draining' }, }); }); + + it('maps a child heap pool refusal to a retryable 503-equivalent', () => { + // The ACP mapping is hand-written beside the REST one and drifts silently + // otherwise; both carry the same contract over different transports. + const rpc = toRpcError(new ChildHeapPoolExhaustedError(3_687, 8, 512)); + expect(rpc.code).toBe(RPC.INTERNAL_ERROR); + expect(rpc.data).toMatchObject({ + errorKind: 'child_heap_pool_exhausted', + childPoolMb: 3_687, + concurrentChildren: 8, + httpStatus: 503, + retryable: true, + }); + }); }); diff --git a/packages/cli/src/serve/run-qwen-serve.test.ts b/packages/cli/src/serve/run-qwen-serve.test.ts index 32a38c2f37e..79ee8f3501f 100644 --- a/packages/cli/src/serve/run-qwen-serve.test.ts +++ b/packages/cli/src/serve/run-qwen-serve.test.ts @@ -1949,7 +1949,8 @@ describe('runQwenServe permissionResponseTimeoutMs validation', () => { }); /** - * The budget is resolved at boot and reported; it does not size any child yet. + * The budget is resolved at boot and reported. Whether it also sizes a child + * depends on `childHeapMode`, which defaults to `observe` and sizes nothing. * The only boot-time behavior is rejecting an out-of-range flag value. */ describe('runQwenServe memory budget', () => { diff --git a/packages/cli/src/serve/run-qwen-serve.ts b/packages/cli/src/serve/run-qwen-serve.ts index dda9c18c5a8..2811c7df945 100644 --- a/packages/cli/src/serve/run-qwen-serve.ts +++ b/packages/cli/src/serve/run-qwen-serve.ts @@ -3497,12 +3497,17 @@ async function runQwenServeImpl( // One policy for the whole daemon, beside the one registry it reads. Both // must be shared: a per-factory registry would report a concurrent count // of 1 on every spawn and hand each child the entire pool. - const childHeapPolicy: ChildHeapPolicy | undefined = opts.daemonMemoryBudget - ? createChildHeapPolicy({ - budget: opts.daemonMemoryBudget, - mode: opts.childHeapMode ?? 'observe', - }) - : undefined; + // Not built for an injected bridge: `deps.bridge` brings its own channel + // and never goes through the factory this policy rides on, so a policy + // here would size nothing while `limits.memory.enforced` claimed + // otherwise — a status field asserting enforcement that is not happening. + const childHeapPolicy: ChildHeapPolicy | undefined = + opts.daemonMemoryBudget && !deps.bridge + ? createChildHeapPolicy({ + budget: opts.daemonMemoryBudget, + mode: opts.childHeapMode ?? 'observe', + }) + : undefined; managedChildHeapPolicy = childHeapPolicy; const fsFactory = runtime.resolveBridgeFsFactory({ // Secondary roots share a write-capable factory only after their own diff --git a/packages/cli/src/serve/server/error-response.test.ts b/packages/cli/src/serve/server/error-response.test.ts index 9dadb02ec09..bf6653f6750 100644 --- a/packages/cli/src/serve/server/error-response.test.ts +++ b/packages/cli/src/serve/server/error-response.test.ts @@ -13,6 +13,7 @@ import { SessionWriterUnavailableError, } from '@qwen-code/qwen-code-core'; import { sendBridgeError } from './error-response.js'; +import { ChildHeapPoolExhaustedError } from '../acp-session-bridge.js'; import { DaemonDrainingError } from './session-archive.js'; function responseMock(): { @@ -28,6 +29,38 @@ function responseMock(): { return { response: response as unknown as Response, status, json }; } +describe('sendBridgeError child heap pool exhaustion', () => { + it('maps the spawn refusal to a retryable 503 with its figures', () => { + // The spawn-policy tests exercise the throw; nothing exercised the wire + // shape, so a transport regression here would have shipped silently. + const status = vi.fn(); + const json = vi.fn(); + const set = vi.fn(); + const response = { status, json, set }; + status.mockReturnValue(response); + json.mockReturnValue(response); + set.mockReturnValue(response); + + sendBridgeError( + response as unknown as Response, + new ChildHeapPoolExhaustedError(3_687, 8, 512), + ); + + expect(status).toHaveBeenCalledWith(503); + // Retryable without operator action: the condition clears the moment any + // child exits, so the header is part of the contract, not decoration. + expect(set).toHaveBeenCalledWith('Retry-After', '5'); + expect(json).toHaveBeenCalledWith( + expect.objectContaining({ + code: 'child_heap_pool_exhausted', + childPoolMb: 3_687, + concurrentChildren: 8, + minChildHeapMb: 512, + }), + ); + }); +}); + describe('sendBridgeError session writer errors', () => { it('maps sealed session maintenance to daemon_draining', () => { const { response, status, json } = responseMock(); diff --git a/packages/cli/src/serve/types.ts b/packages/cli/src/serve/types.ts index aca1ce44f2e..26f92fcd7cf 100644 --- a/packages/cli/src/serve/types.ts +++ b/packages/cli/src/serve/types.ts @@ -228,7 +228,8 @@ export interface ServeOptions { * Total memory budget in MB for the whole daemon process tree — the root * plus every `qwen --acp` child it spawns. When unset, derived as half of * the cgroup-constrained or host memory. Currently observed and reported - * only; it does not yet size any child. + * only until `childHeapMode` is `enforce`, which sizes every ACP child from + * it and refuses spawns the child pool cannot cover. */ memoryBudgetMb?: number; /** From 25e69d2c0f24716047c4369994f0b7c0c1e28bdf Mon Sep 17 00:00:00 2001 From: jinye Date: Tue, 4 Aug 2026 15:59:41 +0800 Subject: [PATCH 6/6] refactor(serve): model the child heap partition, defer applying it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review established that the refusal counter cannot tell an operator whether enforcement is safe, and that is the ground the enforcing mode stood on. While observing, children run on the host-derived ceiling (16384 MB on a 32 GB host), so a workload needing 2 GB of old space is healthy with zero refusals and OOMs the moment a 614 MB partition is applied. The counter measures admission pressure, not ceiling adequacy. Rather than ship a switch with no safe way to decide when to turn it on, `enforce` is removed. `--child-heap-mode` is `off | observe`, and the mode that would apply the partition arrives with the measurement that justifies it: peak old-space per child, compared against the modeled ceiling. That is a real measurement chain — the child reports rss and cpu today, and `--max-old-space-size` bounds old space specifically, so neither rss nor heapUsed answers the question. With nothing applying the partition, the machinery that existed only to apply it goes too rather than shipping unreachable: `getAcpMemoryArgs(explicitMb?)`, `ChildHeapPoolExhaustedError` and both transport mappings, and `limits.memory.enforced` reverts to the required literal `false` it was before. The spawn path is untouched again; the factory asks the policy what it would decide purely so the count is real. Also fixes the zero-pool defect review found, which the removed clamp caused: forcing at least one admissible child on a 512 MB host — where the root reserve consumes the whole 256 MB budget — produced a ceiling of 0, and `--max-old-space-size=0` is V8's *default* heap, not a zero ceiling. A pool that cannot cover one child at the floor now reports `maxConcurrentChildren: 0` and `perChildCeilingMb: null`, and the test that enshrined the old behaviour is inverted. Status now publishes `maxConcurrentChildren` and `perChildCeilingMb`, so an operator can judge the partition against their own workload — the substitute for a counter that cannot judge it for them. Every claim that a zero refusal count means the partition is safe to apply is removed from the flag help, the operator docs, the protocol doc, and the design doc. Refs #8182. Co-Authored-By: Claude Opus 5 --- ...daemon-capacity-model-and-memory-bounds.md | 4 +- docs/developers/daemon/17-configuration.md | 70 ++++---- .../daemon/20-quickstart-operations.md | 4 +- docs/developers/qwen-serve-protocol.md | 2 +- docs/users/qwen-serve.md | 2 +- packages/acp-bridge/src/bridgeErrors.ts | 34 ---- .../acp-bridge/src/child-heap-policy.test.ts | 149 ++++++++---------- packages/acp-bridge/src/child-heap-policy.ts | 119 +++++++------- packages/acp-bridge/src/spawnChannel.test.ts | 133 ++-------------- packages/acp-bridge/src/spawnChannel.ts | 57 +------ packages/cli/src/commands/serve.test.ts | 13 +- packages/cli/src/commands/serve.ts | 30 ++-- .../src/serve/acp-http/dispatch-error.test.ts | 15 -- packages/cli/src/serve/acp-http/dispatch.ts | 13 -- packages/cli/src/serve/acp-session-bridge.ts | 1 - packages/cli/src/serve/daemon-status.test.ts | 43 +++-- packages/cli/src/serve/daemon-status.ts | 45 +++--- packages/cli/src/serve/fast-path.test.ts | 12 +- packages/cli/src/serve/fast-path.ts | 6 +- .../src/serve/server/error-response.test.ts | 33 ---- .../cli/src/serve/server/error-response.ts | 21 --- packages/cli/src/serve/types.ts | 27 ++-- packages/sdk-typescript/src/daemon/types.ts | 23 ++- 23 files changed, 273 insertions(+), 583 deletions(-) diff --git a/docs/design/2026-07-31-daemon-capacity-model-and-memory-bounds.md b/docs/design/2026-07-31-daemon-capacity-model-and-memory-bounds.md index 78988b7d97d..7af90f826d6 100644 --- a/docs/design/2026-07-31-daemon-capacity-model-and-memory-bounds.md +++ b/docs/design/2026-07-31-daemon-capacity-model-and-memory-bounds.md @@ -113,9 +113,9 @@ The real control is admission at spawn time keyed on concurrently live children, - **It must never raise a ceiling.** Clamping to `legacyChildCeilingMb` is what makes the policy safe to apply unconditionally; without it the minimum-budget constant and an over-large explicit flag both inflate the share. - **The spawn path has a trap.** `getAcpMemoryArgs()` emits `--max-old-space-size` only when its computed target exceeds the _spawning daemon's own_ `heap_size_limit` (`spawnChannel.ts:27-34`). A budget-derived share is normally below that, so a naive change is silently dropped and the overcommit returns. The regression test must assert the flag survives a value below the test process's own limit. -**Shipped as `--child-heap-mode`.** The share is keyed on children _concurrently committed_ at spawn time — read from the shared `ProcessRegistry` after `reserve()`, so racing spawns see each other — never on registrations, which allocate nothing. The trap above is handled by an explicit-value parameter that bypasses both the module cache and the raise-only guard, with a regression test asserting a 614 MB share survives against a multi-GB runner. +**Modeled, not yet applied, as `--child-heap-mode`.** A first attempt sized each child by the count live at _its_ spawn; review showed that bounds the child count but not the memory, since V8 cannot lower a running child's ceiling and grants accumulate as P x H(n) — 2.6x the pool at seven children on 8 GB. The model is now a fixed partition: one constant ceiling for every child, with admission capped so the total stays inside the pool by construction. -The compatibility point is why the default is `observe`: it computes the share and the admission decision, applies neither, and counts the refusals that would have happened. Enforcement is a deliberate operator action taken once that count shows it is safe — notably for channel swaps, where the dying child is counted alongside its replacement and a saturated pool could otherwise refuse a restart. `enforce` also supplies the aggregate bound a per-child ceiling alone cannot: refusing once the pool cannot cover another child at the floor caps concurrent children at `childPoolMb / 512`. +Applying it is deliberately deferred. The compatibility point above is why: enforcing changes child GC and OOM behaviour, and nothing yet tells an operator beforehand whether their workload fits the ceiling. The refusal count cannot — children run on the host-derived ceiling while observing, so it measures admission pressure, not ceiling adequacy. The enforcing mode ships with the measurement that justifies it: peak old-space per child, compared against the modeled ceiling. ### Part 2 — Observe, with a denominator, before enforcing diff --git a/docs/developers/daemon/17-configuration.md b/docs/developers/daemon/17-configuration.md index ff7eec5bcfe..47dfd0423d5 100644 --- a/docs/developers/daemon/17-configuration.md +++ b/docs/developers/daemon/17-configuration.md @@ -6,41 +6,41 @@ This page collects every setting that affects the `qwen serve` daemon and its ad ## CLI flags (`qwen serve`) -| Flag | Type | Default | Effect | -| --------------------------------------- | ---------------------------- | --------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `--hostname ` | string | `127.0.0.1` | Bind address. Loopback values: `127.0.0.1`, `localhost`, `::1`, `[::1]`. Non-loopback requires a bearer token at boot. `host:port` input is rejected with guidance to use `--port`. | -| `--port ` | number | `4170` | Listen port; `0` means ephemeral. | -| `--token ` | string | env | Bearer token. Overrides `QWEN_SERVER_TOKEN` and is trimmed at boot. It appears in the process command line, so prefer env in deployments. | -| `--require-auth` | boolean | `false` | Extends bearer auth to loopback and `/health`; boot refuses to start without a token. | -| `--workspace ` | absolute path / repeatable | `process.cwd()` | Startup workspace runtime; repeat to register additional isolated runtimes. The first is primary. Every value must be absolute and a directory; canonicalized at boot. | -| `--memory-project-scope ` | `git-root` / `workspace` | `git-root` | Project-memory partitioning. `git-root` shares memory among workspaces at the same Git root; `workspace` isolates by exact workspace directory. Overrides `QWEN_CODE_MEMORY_PROJECT_SCOPE`. | -| `--max-sessions ` | number | `32` | Per-workspace active session cap. `0` / `Infinity` means unlimited; `NaN` / negative values throw. | -| `--max-total-sessions ` | number | derived for multiple startup/restored workspaces | Daemon-wide active session cap. When omitted, a finite default is derived once from the per-workspace cap and startup/restored workspace count. `0` / `Infinity` means unlimited. | -| `--max-pending-prompts-per-session ` | number | `5` | Accepted but pending/running prompt cap per session. Excess prompt returns 503. `0` / `Infinity` means unlimited; negative or non-integer values throw. | -| `--max-connections ` | number | `256` | HTTP listener `server.maxConnections`; `0` / `Infinity` means unlimited. | -| `--enable-session-shell` | boolean | `false` | Enables direct `POST /session/:id/shell` execution. Requires bearer token, and every call must carry a session-bound `X-Qwen-Client-Id`. | -| `--event-ring-size ` | number | `8000` | Per-session SSE replay ring; soft cap is `1_000_000`. | -| `--compacted-replay-max-bytes ` | positive integer | `4194304` | Byte cap for the bounded in-memory replay snapshot returned by `POST /session/:id/load`; hard cap is `268435456`. | -| `--memory-budget-mb ` | integer in `[1024, 1048576]` | 50% of cgroup-constrained or host memory, capped at the flag maximum (1048576 MB) | Total memory budget for the daemon process tree, capped at resolved available memory. Reported under `limits.memory`; sizes each ACP child only under `--child-heap-mode enforce`. Boot rejects out-of-range values. | -| `--memory-pressure-mode ` | `off` \| `observe` | `observe` | Whether the daemon derives a memory-pressure level from its own RSS and V8 heap. Both modes report `runtime.memory.pressure`; only `observe` raises `daemon_memory_pressure`. Root process only; no remediation. | -| `--child-heap-mode ` | `off \| observe \| enforce` | `observe` | How the per-child heap share derived from the budget is used. `enforce` sizes each child by the live child count and refuses a spawn the pool cannot cover; `observe` computes both but applies neither. | -| `--http-bridge` | boolean | `true` | Stage 1 bridge mode. `--no-http-bridge` still falls back to http-bridge and prints to stderr. | -| `--mcp-client-budget ` | positive integer | unset | Sets `WorkspaceMcpBudget.clientBudget` and forwards it to the ACP child through `childEnvOverrides`. | -| `--mcp-budget-mode ` | `off` / `warn` / `enforce` | `warn` when budget is set, otherwise `off` | Sets `WorkspaceMcpBudget.mode`; `enforce` requires `--mcp-client-budget`. | -| `--allow-origin ` | repeatable string | unset | Cross-origin allowlist that replaces the default CORS denial. `*` allows any origin but requires a token. | -| `--allow-private-auth-base-url` | boolean | `false` | Allows `/workspace/auth/provider` to install localhost / private-network auth provider `baseUrl`; use only in trusted local development. | -| `--prompt-deadline-ms ` | positive integer | unset | Server-side prompt wallclock limit in ms. Timeout aborts and returns an error. | -| `--writer-idle-timeout-ms ` | positive integer | unset | Per-SSE-connection idle timeout in ms. The daemon closes the SSE connection when no event is sent for this duration. | -| `--channel-idle-timeout-ms ` | non-negative integer | `0` | How long to keep the ACP child alive after the last session closes. `0` means reclaim immediately. | -| `--initialize-timeout-ms ` | positive integer | `10000` | ACP child request timeout, including the initialize handshake (ms). | -| `--session-reap-interval-ms ` | non-negative integer | `60000` | Session reaper scan interval; `0` disables it. | -| `--session-idle-timeout-ms ` | non-negative integer | `1800000` | Disconnected-session idle reaping time; `0` disables it. | -| `--rate-limit` / `--no-rate-limit` | boolean | env / off | Enables per-tier HTTP rate limiting for prompt, mutation, and read routes. | -| `--rate-limit-prompt ` | positive integer | `10` | Prompt request limit per window; requires rate limiting to be enabled. | -| `--rate-limit-mutation ` | positive integer | `30` | Mutation request limit per window; requires rate limiting to be enabled. | -| `--rate-limit-read ` | positive integer | `120` | Read request limit per window; requires rate limiting to be enabled. | -| `--rate-limit-window-ms ` | integer `>= 1000` | `60000` | Rate limit window length; requires rate limiting to be enabled. | -| no flag | - | - | `QWEN_SERVE_NO_MCP_POOL=1` fully disables the pool. | +| Flag | Type | Default | Effect | +| --------------------------------------- | ---------------------------- | --------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `--hostname ` | string | `127.0.0.1` | Bind address. Loopback values: `127.0.0.1`, `localhost`, `::1`, `[::1]`. Non-loopback requires a bearer token at boot. `host:port` input is rejected with guidance to use `--port`. | +| `--port ` | number | `4170` | Listen port; `0` means ephemeral. | +| `--token ` | string | env | Bearer token. Overrides `QWEN_SERVER_TOKEN` and is trimmed at boot. It appears in the process command line, so prefer env in deployments. | +| `--require-auth` | boolean | `false` | Extends bearer auth to loopback and `/health`; boot refuses to start without a token. | +| `--workspace ` | absolute path / repeatable | `process.cwd()` | Startup workspace runtime; repeat to register additional isolated runtimes. The first is primary. Every value must be absolute and a directory; canonicalized at boot. | +| `--memory-project-scope ` | `git-root` / `workspace` | `git-root` | Project-memory partitioning. `git-root` shares memory among workspaces at the same Git root; `workspace` isolates by exact workspace directory. Overrides `QWEN_CODE_MEMORY_PROJECT_SCOPE`. | +| `--max-sessions ` | number | `32` | Per-workspace active session cap. `0` / `Infinity` means unlimited; `NaN` / negative values throw. | +| `--max-total-sessions ` | number | derived for multiple startup/restored workspaces | Daemon-wide active session cap. When omitted, a finite default is derived once from the per-workspace cap and startup/restored workspace count. `0` / `Infinity` means unlimited. | +| `--max-pending-prompts-per-session ` | number | `5` | Accepted but pending/running prompt cap per session. Excess prompt returns 503. `0` / `Infinity` means unlimited; negative or non-integer values throw. | +| `--max-connections ` | number | `256` | HTTP listener `server.maxConnections`; `0` / `Infinity` means unlimited. | +| `--enable-session-shell` | boolean | `false` | Enables direct `POST /session/:id/shell` execution. Requires bearer token, and every call must carry a session-bound `X-Qwen-Client-Id`. | +| `--event-ring-size ` | number | `8000` | Per-session SSE replay ring; soft cap is `1_000_000`. | +| `--compacted-replay-max-bytes ` | positive integer | `4194304` | Byte cap for the bounded in-memory replay snapshot returned by `POST /session/:id/load`; hard cap is `268435456`. | +| `--memory-budget-mb ` | integer in `[1024, 1048576]` | 50% of cgroup-constrained or host memory, capped at the flag maximum (1048576 MB) | Total memory budget for the daemon process tree, capped at resolved available memory. Reported under `limits.memory`, and modeled into a per-child partition. Nothing applies it. Boot rejects out-of-range values. | +| `--memory-pressure-mode ` | `off` \| `observe` | `observe` | Whether the daemon derives a memory-pressure level from its own RSS and V8 heap. Both modes report `runtime.memory.pressure`; only `observe` raises `daemon_memory_pressure`. Root process only; no remediation. | +| `--child-heap-mode ` | `off \| observe` | `observe` | Whether the daemon models a per-child heap partition of the budget. `observe` reports it and counts spawns past it; nothing is applied. | +| `--http-bridge` | boolean | `true` | Stage 1 bridge mode. `--no-http-bridge` still falls back to http-bridge and prints to stderr. | +| `--mcp-client-budget ` | positive integer | unset | Sets `WorkspaceMcpBudget.clientBudget` and forwards it to the ACP child through `childEnvOverrides`. | +| `--mcp-budget-mode ` | `off` / `warn` / `enforce` | `warn` when budget is set, otherwise `off` | Sets `WorkspaceMcpBudget.mode`; `enforce` requires `--mcp-client-budget`. | +| `--allow-origin ` | repeatable string | unset | Cross-origin allowlist that replaces the default CORS denial. `*` allows any origin but requires a token. | +| `--allow-private-auth-base-url` | boolean | `false` | Allows `/workspace/auth/provider` to install localhost / private-network auth provider `baseUrl`; use only in trusted local development. | +| `--prompt-deadline-ms ` | positive integer | unset | Server-side prompt wallclock limit in ms. Timeout aborts and returns an error. | +| `--writer-idle-timeout-ms ` | positive integer | unset | Per-SSE-connection idle timeout in ms. The daemon closes the SSE connection when no event is sent for this duration. | +| `--channel-idle-timeout-ms ` | non-negative integer | `0` | How long to keep the ACP child alive after the last session closes. `0` means reclaim immediately. | +| `--initialize-timeout-ms ` | positive integer | `10000` | ACP child request timeout, including the initialize handshake (ms). | +| `--session-reap-interval-ms ` | non-negative integer | `60000` | Session reaper scan interval; `0` disables it. | +| `--session-idle-timeout-ms ` | non-negative integer | `1800000` | Disconnected-session idle reaping time; `0` disables it. | +| `--rate-limit` / `--no-rate-limit` | boolean | env / off | Enables per-tier HTTP rate limiting for prompt, mutation, and read routes. | +| `--rate-limit-prompt ` | positive integer | `10` | Prompt request limit per window; requires rate limiting to be enabled. | +| `--rate-limit-mutation ` | positive integer | `30` | Mutation request limit per window; requires rate limiting to be enabled. | +| `--rate-limit-read ` | positive integer | `120` | Read request limit per window; requires rate limiting to be enabled. | +| `--rate-limit-window-ms ` | integer `>= 1000` | `60000` | Rate limit window length; requires rate limiting to be enabled. | +| no flag | - | - | `QWEN_SERVE_NO_MCP_POOL=1` fully disables the pool. | ## Environment variables diff --git a/docs/developers/daemon/20-quickstart-operations.md b/docs/developers/daemon/20-quickstart-operations.md index aa3eaf564dd..2319c53a1e7 100644 --- a/docs/developers/daemon/20-quickstart-operations.md +++ b/docs/developers/daemon/20-quickstart-operations.md @@ -80,9 +80,9 @@ The CLI is defined in **`packages/cli/src/commands/serve.ts`**: | `--token ` | string | env / none | Non-loopback and `--require-auth` | Bearer token; trimmed once. **It appears in `/proc//cmdline`, so prefer `QWEN_SERVER_TOKEN`**. Boot stderr also warns about this. | | `--max-sessions ` | number | `32` | - | Per-workspace active session cap. Excess spawn returns 503. `0` means unlimited. `NaN` / negative values throw. | | `--max-total-sessions ` | number | derived for multiple startup/restored workspaces | - | Daemon-wide active session cap. When omitted, a finite default is derived once from the per-workspace cap and startup/restored workspace count; dynamic registration does not recompute it. `0` means unlimited. | -| `--memory-budget-mb ` | integer in `[1024, 1048576]` | 50% of cgroup/host memory | Observation only | Total memory budget for the daemon process tree, capped at resolved available memory. Reported under `limits.memory`. Sizes children only under `--child-heap-mode enforce`. | +| `--memory-budget-mb ` | integer in `[1024, 1048576]` | 50% of cgroup/host memory | Observation only | Total memory budget for the daemon process tree, capped at resolved available memory. Reported under `limits.memory`; modeled into a partition that nothing applies. | | `--memory-pressure-mode ` | `off` \| `observe` | `observe` | Observation only | Reports `runtime.memory.pressure` in both modes; only `observe` raises the `daemon_memory_pressure` issue. Root process only. | -| `--child-heap-mode ` | `off \| observe \| enforce` | `observe` | Enforcing only under `enforce` | Sizes each ACP child by concurrent child count; refuses spawns the pool cannot cover. `observe` applies neither and counts would-be refusals. | +| `--child-heap-mode ` | `off \| observe` | `observe` | Observation only | Reports the modeled partition under `limits.memory.childHeap`; applies nothing and refuses nothing. | | `--max-pending-prompts-per-session ` | number | `5` | - | Accepted but pending/running prompt cap per session. Excess prompt returns 503. `0` / `Infinity` means unlimited. Negative or non-integer values throw. | | `--workspace ` | string / repeatable | `process.cwd()` | - | Startup workspace runtime; repeat to register additional isolated runtimes. The first is primary. Each value **must be an absolute path, must exist, and must be a directory**. Boot canonicalizes every value via `canonicalizeWorkspace`. `POST /session` with a mismatched `cwd` returns `400 workspace_mismatch`. | | `--max-connections ` | number | `256` | - | Listener-level `server.maxConnections`. `0` / `Infinity` means unlimited. `NaN` / negative values fail boot to avoid fail-open behavior. | diff --git a/docs/developers/qwen-serve-protocol.md b/docs/developers/qwen-serve-protocol.md index 29a7283d53a..e1cfccb4a71 100644 --- a/docs/developers/qwen-serve-protocol.md +++ b/docs/developers/qwen-serve-protocol.md @@ -618,7 +618,7 @@ runtime routes return `503`. `runtime.activity` reports daemon-wide prompt activity. `activePrompts` counts sessions with an in-flight prompt. `pendingPrompts` counts all accepted prompts that have not settled yet, including the running prompt and FIFO-waiting prompts. `queuedPrompts` counts FIFO-waiting prompts that have been accepted but not dispatched. `lastActivityAt` is the ISO 8601 timestamp of the last prompt start/end or session spawn; `null` when the daemon has never processed any activity since boot. `idleSinceMs` is computed from `lastActivityAt` at response generation time. -`limits.memory` is additive and reports the daemon's resolved memory figures: `enforced` (a boolean now, not the literal `false` it was while nothing applied these figures — it is `true` only under `--child-heap-mode enforce`, and stays `false` under `observe`, which computes everything and applies nothing), a `childHeap` object (`mode`, and `refusals` — spawns refused, or under `observe` that _would_ have been refused, which is the signal for whether enabling `enforce` is safe on a given host; `null` when the daemon built no policy), `configuredBudgetMb`, `effectiveBudgetMb` (the configured value capped at resolved cgroup/host memory), `budgetSource` (`flag` / `derived`), `availableMemoryMb`, `availableMemorySource` (`constrained` / `host`), `insufficientMemory`, and a `modeled` object holding `rootReserveMb`, `childPoolMb`, `minChildHeapMb`, `maxChildHeapMb`, and `legacyChildCeilingMb` (a conservative model of the ceiling an ACP child receives today, which can sit below the real figure). `runtime.memory` additionally reports `registeredWorkspaces` (the registration count — non-removed workspace entries, including draining, transitioning, or blocked ones; not a live-child count), `activeAcpChildren` (daemon-managed ACP children with a live, non-dying channel — includes transitioning or blocked entries, but excludes a workspace whose kill has started even if the child has not exited; not channel workers, MCP descendants, or unattached spawn reservations), `childRssCoverage` (`active_children` — every ACP child with a live channel, which is the set `activeAcpChildren` counts; older daemons send `primary_only`), a `children` object described below, and a `modeled` object holding `recommendedShareAtRegisteredMb` (`null` when no workspace is registered) and `recommendedShareAtActiveMb` (`null` when no child is active). Each share is capped at the legacy child ceiling, and floored at the minimum child heap only when the ceiling allows — on a small host the ceiling sits below the floor, so share × count can exceed the child pool. Read a share as advisory, not a partition of the pool. Whether any of it is applied is exactly what `enforced` reports. Under `off` and `observe` it remains pure observation: no child spawn argument derives from these values and no request is refused on their basis. Under `enforce` a spawning ACP child's `--max-old-space-size` is the child pool divided by the children concurrently committed at that moment, and a spawn is refused with `child_heap_pool_exhausted` (REST 503 with `Retry-After`, ACP `errorKind: 'child_heap_pool_exhausted'`, retryable) once the pool cannot cover another child at `modeled.minChildHeapMb`. That refusal surfaces as a failure to open a new session in the affected workspace — registration is never refused, because registration allocates nothing — and clears when any child exits. On the normal `runQwenServe` path the budget is resolved before the bootstrap app is created, so `limits.memory` is already populated during the bootstrap window. It is `null` only on paths that resolve no budget (such as direct-embed bypassing `runQwenServeImpl`). The SDK type allows `null`, so correct clients cope. +`limits.memory` is additive and reports the daemon's resolved memory figures: a required `enforced: false`, a `childHeap` object (`mode`, `maxConcurrentChildren`, `perChildCeilingMb` — `null` when no child is admissible, never 0 — and `refusals`, the spawns that would have exceeded the modeled limit), `configuredBudgetMb`, `effectiveBudgetMb` (the configured value capped at resolved cgroup/host memory), `budgetSource` (`flag` / `derived`), `availableMemoryMb`, `availableMemorySource` (`constrained` / `host`), `insufficientMemory`, and a `modeled` object holding `rootReserveMb`, `childPoolMb`, `minChildHeapMb`, `maxChildHeapMb`, and `legacyChildCeilingMb` (a conservative model of the ceiling an ACP child receives today, which can sit below the real figure). `runtime.memory` additionally reports `registeredWorkspaces` (the registration count — non-removed workspace entries, including draining, transitioning, or blocked ones; not a live-child count), `activeAcpChildren` (daemon-managed ACP children with a live, non-dying channel — includes transitioning or blocked entries, but excludes a workspace whose kill has started even if the child has not exited; not channel workers, MCP descendants, or unattached spawn reservations), `childRssCoverage` (`active_children` — every ACP child with a live channel, which is the set `activeAcpChildren` counts; older daemons send `primary_only`), a `children` object described below, and a `modeled` object holding `recommendedShareAtRegisteredMb` (`null` when no workspace is registered) and `recommendedShareAtActiveMb` (`null` when no child is active). Each share is capped at the legacy child ceiling, and floored at the minimum child heap only when the ceiling allows — on a small host the ceiling sits below the floor, so share × count can exceed the child pool. Read a share as advisory, not a partition of the pool. All of it is observation: no child spawn argument derives from these values, and no request is refused on their basis. `childHeap` models a fixed partition of `modeled.childPoolMb` — every child would receive the same `perChildCeilingMb`, so the modeled total stays inside the pool rather than accumulating as a per-spawn share would. Read `refusals` as admission pressure only: a count of 0 does **not** mean the partition is safe to apply, because children run on the much larger host-derived ceiling, so a workload needing more old space than `perChildCeilingMb` is healthy here and would only fail once the partition were applied. On the normal `runQwenServe` path the budget is resolved before the bootstrap app is created, so `limits.memory` is already populated during the bootstrap window. It is `null` only on paths that resolve no budget (such as direct-embed bypassing `runQwenServeImpl`). The SDK type allows `null`, so correct clients cope. `runtime.memory.children` is additive within that block and reports aggregate RSS across the children `childRssCoverage` names: `rssBytes` (their summed self-reported RSS), `sampled` (how many produced a reading), and `oldestReadingAgeMs` (the age of the oldest reading in the sum, so a caller can tell how far apart its parts were taken). The denominator for `sampled` is the sibling `activeAcpChildren`, not repeated inside the block; when `sampled` is lower, `rssBytes` is a floor rather than a total. Sampling is gated on an active SSE/WS watcher, so a status request against a daemon nobody is streaming from reports `sampled: 0` even with live children — `activeAcpChildren` beside it makes that gap visible, and `rssBytes: 0` with `sampled: 0` never means a measured zero. `oldestReadingAgeMs` is `null` when nothing was sampled and also when every contributor is a bridge predating the field, so it never means "fresh". Read the sum as an over-count and an under-count at once: summing per-process RSS double-counts pages the children share, while each child reports only its own process, so its MCP descendants and every channel worker are missing. It is not the daemon tree's memory. The field is optional in the SDK mirror because daemons reporting `primary_only` never send it. diff --git a/docs/users/qwen-serve.md b/docs/users/qwen-serve.md index 8f631118ed0..2c91f2bc79d 100644 --- a/docs/users/qwen-serve.md +++ b/docs/users/qwen-serve.md @@ -397,7 +397,7 @@ Notes: | `--max-connections ` | `256` | Listener-level TCP connection cap (`server.maxConnections`). Bounds raw socket count irrespective of session count — slow / phantom SSE clients get rejected at accept time once full. Raise alongside `--max-sessions` if your deployment expects many SSE subscribers per session. | | `--memory-budget-mb ` | 50% of cgroup/host | Total memory budget in MB for the whole daemon process tree. When unset, derived as 50% of the cgroup limit or host memory; either way the effective value is capped at resolved available memory, and both the configured and effective figures are reported. Currently observation only — it does not change how any `qwen --acp` child is sized. Resolved figures appear under `limits.memory` in `GET /daemon/status`, alongside registered and live child counts and advisory per-child shares under `runtime.memory`. A host too small for the minimum reports `insufficientMemory` rather than being clamped upward; because the derived fraction is 50%, any host under ~2 GB trips this. Pass an explicit `--memory-budget-mb 1024` on such a host to override the derived figure (the flag still requires at least 1024 MB of available memory to clear the warning). Must be an integer in `[1024, 1048576]`. | | `--memory-pressure-mode ` | `observe` | Whether the daemon turns its own memory reading into a verdict. `observe` (default) reports the pressure level under `runtime.memory.pressure` in `GET /daemon/status` and raises a `daemon_memory_pressure` issue — a `warning`, so the overall `status` leaves `ok` — whenever the level leaves `normal`. `off` still reports every figure, including the level, but raises no issue, so the overall `status` is unchanged; use it while calibrating, or if you alert on the top-level status. The level is the worse of two ratios: RSS against available memory (what the cgroup OOM killer watches) and V8 heap used against this process's heap ceiling. It covers the daemon root process only; compare it against `runtime.memory.children.rssBytes` for the children. Nothing remediates in either mode. One of `off`, `observe`. | -| `--child-heap-mode ` | `observe` | What the daemon does with the per-child heap share it derives from `--memory-budget-mb`. `enforce` gives each `qwen --acp` child a `--max-old-space-size` equal to the child pool divided by the children live when it spawns, and refuses a spawn once the pool cannot cover another child at the 512 MB minimum — that refusal appears as a failure to open a new session in that workspace, and clears when any child exits. `observe` (default) computes the same share and the same decision but applies **neither**, reporting how many spawns would have been refused under `limits.memory.childHeap.refusals`; confirm that count is zero before switching to `enforce`. `off` computes nothing. Registration is never refused — a registered workspace with no session has no child. | +| `--child-heap-mode ` | `observe` | Whether the daemon models a per-child heap partition of `--memory-budget-mb`. `observe` (default) reports what it would apply — `limits.memory.childHeap.perChildCeilingMb` and `maxConcurrentChildren` — and counts spawns that would have exceeded the limit. **Nothing is applied**: no child is sized from the budget and no spawn is refused. `off` models nothing. A refusal count of 0 does **not** mean the partition would be safe to apply: children still run on the much larger host-derived ceiling, so a workload needing more old space than the modeled ceiling looks perfectly healthy here. Applying the partition ships with the measurement that can answer that. | | `--event-ring-size ` | `8000` | Per-session SSE replay ring depth (#3803 §02 target). Sets the backlog available to `GET /session/:id/events` with `Last-Event-ID: N`. Larger = more reconnect headroom at the cost of a few hundred KB extra RAM per session. SDK clients can additionally request a larger per-subscriber backlog cap on a specific subscription via `?maxQueued=N` (range `[16, 2048]`, default 256). Daemons also emit a non-terminal `slow_client_warning` SSE frame at 75% queue fill so clients can drain / reconnect before getting evicted. Pre-flight `caps.features.slow_client_warning`. | | `--compacted-replay-max-bytes ` | `4194304` | Per-live-session byte cap for the retained replay events in the bounded snapshot returned by `POST /session/:id/load`. The cap applies to `compactedReplay`; the current in-flight `liveJournal` is separately capped by `--max-journal-events` and `--max-journal-bytes`. Values must be positive safe integers; invalid values fail at boot, and the hard ceiling is 256 MiB. When older retained replay is dropped, the snapshot begins with `history_truncated`. This does not limit the on-disk transcript. | | `--max-journal-events ` | `10000` | Per-session cap on the number of raw events retained in the in-flight live journal (the current unfinished turn). When exceeded, the oldest journal entries are dropped and a `history_truncated` marker is prepended. Must be a positive safe integer. | diff --git a/packages/acp-bridge/src/bridgeErrors.ts b/packages/acp-bridge/src/bridgeErrors.ts index b2a6bb25e18..6ac2a5bf247 100644 --- a/packages/acp-bridge/src/bridgeErrors.ts +++ b/packages/acp-bridge/src/bridgeErrors.ts @@ -186,40 +186,6 @@ export class TotalSessionLimitExceededError extends Error { } } -/** - * Thrown at spawn time when the daemon's child pool cannot cover another ACP - * child at the minimum heap. Refusing here rather than at registration is - * deliberate: registration allocates nothing (a dormant workspace has no - * child), so this surfaces as "no new session in this workspace right now", - * which is both true and retryable — the condition clears as soon as another - * child exits. - * - * Only `--child-heap-mode enforce` throws it. Under `observe` the same - * condition is counted and reported instead, so a deployment can find out - * whether enforcement would have refused anything before it does. - */ -export class ChildHeapPoolExhaustedError extends Error { - readonly childPoolMb: number; - readonly concurrentChildren: number; - readonly minChildHeapMb: number; - constructor( - childPoolMb: number, - concurrentChildren: number, - minChildHeapMb: number, - ) { - super( - `Daemon child heap pool (${childPoolMb} MB) cannot cover ` + - `${concurrentChildren} concurrent children at the ${minChildHeapMb} MB ` + - `minimum. Wait for a child to exit, reduce concurrent workspaces, or ` + - `raise --memory-budget-mb.`, - ); - this.name = 'ChildHeapPoolExhaustedError'; - this.childPoolMb = childPoolMb; - this.concurrentChildren = concurrentChildren; - this.minChildHeapMb = minChildHeapMb; - } -} - /** * Thrown by `sendPrompt` when a session already has too many accepted * prompts waiting or running. The REST route maps this to 503 with diff --git a/packages/acp-bridge/src/child-heap-policy.test.ts b/packages/acp-bridge/src/child-heap-policy.test.ts index 57debb7287b..59900bfc67e 100644 --- a/packages/acp-bridge/src/child-heap-policy.test.ts +++ b/packages/acp-bridge/src/child-heap-policy.test.ts @@ -13,116 +13,91 @@ import { describe('createChildHeapPolicy', () => { it.each([2_048, 8_192, 32_768, 262_144])( - 'keeps the sum of every admissible ceiling inside the pool (%i MB host)', + 'models a partition whose total fits the pool (%i MB host)', (availableMemoryMb) => { - // THE invariant. A per-spawn share bounds the child count but not the - // memory — grants accumulate as P x H(n) because V8 cannot lower a - // running child's ceiling — so this asserts the property that claim - // actually needs: fill the daemon to its admission limit and the - // authorised total still fits the pool. + // The invariant a per-spawn share could not hold: sizing each child by + // the count live at *its* spawn accumulates grants as P x H(n), because + // V8 cannot lower a running child's ceiling. A constant ceiling makes + // the total n x ceiling, which admission keeps inside the pool. const b = resolveDaemonMemoryBudget({ availableMemoryMb }); - const policy = createChildHeapPolicy({ budget: b, mode: 'enforce' }); - const { maxConcurrentChildren, perChildCeilingMb } = policy.snapshot(); + const { maxConcurrentChildren, perChildCeilingMb } = + createChildHeapPolicy({ budget: b, mode: 'observe' }).snapshot(); - let granted = 0; - for (let n = 1; n <= maxConcurrentChildren; n++) { - const decision = policy.decide(n); - expect(decision.refuse).toBe(false); - granted += decision.ceilingMb!; - } - expect(granted).toBeLessThanOrEqual(b.childPoolMb); - expect(granted).toBe(maxConcurrentChildren * perChildCeilingMb); - // And the child past the limit is refused, which is what holds the sum. - expect(policy.decide(maxConcurrentChildren + 1).refuse).toBe(true); + expect(maxConcurrentChildren).toBeGreaterThan(0); + expect(perChildCeilingMb).toBeGreaterThanOrEqual(MIN_CHILD_HEAP_MB); + expect(maxConcurrentChildren * perChildCeilingMb!).toBeLessThanOrEqual( + b.childPoolMb, + ); }, ); - it('hands every child the same ceiling regardless of how many are live', () => { - // The property the invariant rests on: a ceiling that shrank as children - // arrived would leave the early, larger grants outstanding and unbounded. - const b = resolveDaemonMemoryBudget({ availableMemoryMb: 8_192 }); - const policy = createChildHeapPolicy({ budget: b, mode: 'enforce' }); - const ceilings = [1, 2, 3, 7].map((n) => policy.decide(n).ceilingMb); - expect(new Set(ceilings).size).toBe(1); - expect(ceilings[0]).toBeGreaterThanOrEqual(MIN_CHILD_HEAP_MB); + it('admits no child when the pool cannot cover one, and offers no ceiling', () => { + // A 512 MB host derives a 256 MB budget whose root reserve consumes all of + // it, leaving a pool of 0. Clamping the count up to 1 here produced a + // ceiling of 0 — and `--max-old-space-size=0` is not a zero ceiling, it is + // V8's *default* heap, so that would have modelled gigabytes against an + // empty pool. + const empty = resolveDaemonMemoryBudget({ availableMemoryMb: 512 }); + expect(empty.childPoolMb).toBe(0); + expect( + createChildHeapPolicy({ budget: empty, mode: 'observe' }).snapshot(), + ).toMatchObject({ maxConcurrentChildren: 0, perChildCeilingMb: null }); }); - it('sizes an 8 GB host for seven concurrent children', () => { - // Pinned deliberately: this is the number an operator plans against, and - // it is the cost of the invariant above — the eighth concurrent session - // is refused even though real RSS would likely have fit it. - const policy = createChildHeapPolicy({ - budget: resolveDaemonMemoryBudget({ availableMemoryMb: 8_192 }), - mode: 'enforce', - }); - expect(policy.snapshot()).toMatchObject({ - childPoolMb: 3_687, - maxConcurrentChildren: 7, - perChildCeilingMb: 526, - }); - }); - - it('never admits more than the repository workspace maximum', () => { - // A large host divides by MAX_DAEMON_WORKSPACES rather than by - // pool/512, so the ceiling is 614 MB and not the 512 MB floor. - const policy = createChildHeapPolicy({ - budget: resolveDaemonMemoryBudget({ availableMemoryMb: 32_768 }), - mode: 'enforce', - }); - expect(policy.snapshot()).toMatchObject({ - maxConcurrentChildren: 25, - perChildCeilingMb: 614, - }); + it('never models a ceiling below the documented minimum', () => { + // A 1024 MB host leaves a 256 MB pool — under the 512 MB floor, so still + // no admissible child rather than one child at half the minimum. + const small = resolveDaemonMemoryBudget({ availableMemoryMb: 1_024 }); + expect(small.childPoolMb).toBeLessThan(MIN_CHILD_HEAP_MB); + const snap = createChildHeapPolicy({ + budget: small, + mode: 'observe', + }).snapshot(); + expect(snap.maxConcurrentChildren).toBe(0); + expect(snap.perChildCeilingMb).toBeNull(); }); - it('computes in observe mode but reports nothing as enforced', () => { - const b = resolveDaemonMemoryBudget({ availableMemoryMb: 8_192 }); - const observe = createChildHeapPolicy({ budget: b, mode: 'observe' }); - const over = observe.snapshot().maxConcurrentChildren + 1; + it('sizes an 8 GB host for seven children, and a large host by the workspace cap', () => { + // Pinned: these are the numbers an operator plans against. The large host + // divides by MAX_DAEMON_WORKSPACES rather than pool/512, so the ceiling is + // 614 MB and not the floor. + expect( + createChildHeapPolicy({ + budget: resolveDaemonMemoryBudget({ availableMemoryMb: 8_192 }), + mode: 'observe', + }).snapshot(), + ).toMatchObject({ maxConcurrentChildren: 7, perChildCeilingMb: 526 }); - const decision = observe.decide(over); - expect(decision.ceilingMb).toBe(observe.snapshot().perChildCeilingMb); - expect(decision.refuse).toBe(true); - expect(observe.snapshot()).toMatchObject({ - mode: 'observe', - enforced: false, - refusals: 1, - }); + expect( + createChildHeapPolicy({ + budget: resolveDaemonMemoryBudget({ availableMemoryMb: 32_768 }), + mode: 'observe', + }).snapshot(), + ).toMatchObject({ maxConcurrentChildren: 25, perChildCeilingMb: 614 }); }); - it('counts would-be refusals so calibration does not need a broken deployment', () => { + it('counts spawns past the modeled limit', () => { const b = resolveDaemonMemoryBudget({ availableMemoryMb: 8_192 }); const policy = createChildHeapPolicy({ budget: b, mode: 'observe' }); const limit = policy.snapshot().maxConcurrentChildren; - expect(policy.snapshot().refusals).toBe(0); - policy.decide(1); - policy.decide(limit); + expect(policy.decide(1).refuse).toBe(false); + expect(policy.decide(limit).refuse).toBe(false); expect(policy.snapshot().refusals).toBe(0); - policy.decide(limit + 1); + expect(policy.decide(limit + 1).refuse).toBe(true); policy.decide(limit + 9); expect(policy.snapshot().refusals).toBe(2); }); - it('computes nothing at all when off', () => { - const b = resolveDaemonMemoryBudget({ availableMemoryMb: 8_192 }); - const off = createChildHeapPolicy({ budget: b, mode: 'off' }); - - // Not "a share of zero" — no share, so the caller keeps the historical - // host-derived ceiling. And an off policy must never accrue refusals. - expect(off.decide(9_999)).toEqual({ ceilingMb: undefined, refuse: false }); - expect(off.snapshot()).toMatchObject({ enforced: false, refusals: 0 }); - }); - - it('always admits at least one child, however small the pool', () => { - // A pool below the floor would otherwise divide to zero and refuse every - // spawn, bricking a daemon that works today. - const tiny = createChildHeapPolicy({ - budget: resolveDaemonMemoryBudget({ availableMemoryMb: 1_024 }), - mode: 'enforce', + it('models nothing at all when off', () => { + const off = createChildHeapPolicy({ + budget: resolveDaemonMemoryBudget({ availableMemoryMb: 8_192 }), + mode: 'off', }); - expect(tiny.snapshot().maxConcurrentChildren).toBeGreaterThanOrEqual(1); - expect(tiny.decide(1).refuse).toBe(false); + // An off policy must never accrue refusals, or the counter would report on + // a daemon that modelled nothing. + expect(off.decide(9_999).refuse).toBe(false); + expect(off.snapshot().refusals).toBe(0); }); }); diff --git a/packages/acp-bridge/src/child-heap-policy.ts b/packages/acp-bridge/src/child-heap-policy.ts index 83d454aaf9b..9404a217da1 100644 --- a/packages/acp-bridge/src/child-heap-policy.ts +++ b/packages/acp-bridge/src/child-heap-policy.ts @@ -11,46 +11,50 @@ import { import { MAX_DAEMON_WORKSPACES } from './channel-control-timeouts.js'; /** - * What the daemon does with the child-heap share it computes. + * Whether the daemon models a per-child heap partition. * - * `off` — do not compute it. Children get the historical host-derived ceiling. + * `off` — do not model it. * - * `observe` — compute the share and the admission decision, apply **neither**, - * and count the refusals that would have happened. Deliberately the default: - * the thresholds this policy divides by have never been checked against a real - * multi-workspace deployment, and a non-zero refusal count is the evidence - * that turning `enforce` on would have broken someone. + * `observe` — compute the partition and count the spawns it would have + * refused. Nothing is applied: no child receives a derived + * `--max-old-space-size`, and no spawn is refused. * - * `enforce` — pass the share to the child and refuse the spawn when the pool - * cannot cover another one. + * There is deliberately no `enforce` yet. Applying the partition needs a way + * to tell an operator beforehand whether their workload fits it, and that + * observation does not exist: `refusals` below counts admission pressure, not + * whether a child would have survived the ceiling. Enforcing on a signal that + * cannot answer the question it is being read for is how a healthy daemon gets + * switched into an OOM loop. The enforcing mode ships with the measurement + * that justifies it — peak old-space per child, compared against + * `perChildCeilingMb`. */ -export type ChildHeapMode = 'off' | 'observe' | 'enforce'; - -export interface ChildHeapDecision { - /** - * The share this child should receive, or `undefined` when the mode does not - * produce one. `undefined` means "spawn as before", never "zero". - */ - ceilingMb: number | undefined; - /** Whether the pool cannot cover this child. Only acted on under `enforce`. */ - refuse: boolean; -} +export type ChildHeapMode = 'off' | 'observe'; export interface ChildHeapPolicySnapshot { mode: ChildHeapMode; - /** True only under `enforce` — i.e. only when a spawn argument really derives from this. */ - enforced: boolean; childPoolMb: number; minChildHeapMb: number; - /** Children admitted concurrently. `perChildCeilingMb * this <= childPoolMb`. */ + /** + * Children the pool could host concurrently under the modeled partition. + * **0** when the pool cannot cover even one child at `minChildHeapMb` — a + * real state on a small host, and not the same as 1. + */ maxConcurrentChildren: number; - /** The constant every admitted child receives. */ - perChildCeilingMb: number; /** - * Spawns this policy refused, or would have refused under `enforce`. The - * calibration signal: non-zero under `observe` means enforcement would have - * failed a real spawn, including the channel-swap case where a replacement - * is counted alongside the process it replaces. + * The ceiling every child would receive. `null` when no child is + * admissible, never 0: `--max-old-space-size=0` means *V8's default heap*, + * so emitting a zero here would authorise gigabytes against an empty pool. + */ + perChildCeilingMb: number | null; + /** + * Spawns that would have been refused for exceeding + * `maxConcurrentChildren`. + * + * Read it as admission pressure and nothing more. In particular a count of + * 0 does **not** mean the partition is safe to apply: children currently + * run on the far larger host-derived ceiling, so a workload needing more + * old space than `perChildCeilingMb` is perfectly healthy here and would + * only fail once the partition were applied. */ refusals: number; } @@ -60,7 +64,7 @@ export interface ChildHeapPolicy { * @param concurrentChildren Children already committed *including this one* * — `ProcessRegistry.committedProcessCount` taken after `reserve()`. */ - decide(concurrentChildren: number): ChildHeapDecision; + decide(concurrentChildren: number): { refuse: boolean }; snapshot(): ChildHeapPolicySnapshot; } @@ -71,51 +75,40 @@ export function createChildHeapPolicy(options: { const { budget, mode } = options; let refusals = 0; - // A FIXED partition, not a share of the pool divided by the children live at - // this instant. The difference is the whole contract. - // - // A per-spawn share bounds the child *count* but not the memory: V8 cannot - // lower a running child's ceiling, so grants accumulate as - // P + P/2 + P/3 + ... = P x H(n) — 2.6x the pool at seven children on an - // 8 GB host, 4x at twenty-five on 32 GB. That authorises more old space than - // the host has, which is what this policy exists to stop. - // - // Holding the ceiling constant makes the sum n * ceiling, and admitting at - // most `maxConcurrentChildren` makes that <= childPoolMb by construction — - // no ledger of outstanding grants, and no dependence on the order children - // happened to arrive in. + // A FIXED partition, not a share of the pool divided by the children live + // at this instant. A per-spawn share bounds the child *count* but not the + // memory: V8 cannot lower a running child's ceiling, so grants accumulate + // as P + P/2 + P/3 + ... = P x H(n) — 2.6x the pool at seven children on an + // 8 GB host. Holding the ceiling constant makes the total n * ceiling, and + // admitting at most `maxConcurrentChildren` keeps that inside the pool by + // construction, with no ledger and no dependence on arrival order. // - // The cost is real and deliberate: a lone workspace on a 32 GB host gets - // 614 MB rather than the whole pool. Every admitted child is sized for a - // full house, because any child may still be running when the house fills. - const maxConcurrentChildren = Math.max( - 1, - Math.min( - Math.floor(budget.childPoolMb / MIN_CHILD_HEAP_MB), - MAX_DAEMON_WORKSPACES, - ), - ); - const perChildCeilingMb = Math.min( - Math.floor(budget.childPoolMb / maxConcurrentChildren), - budget.legacyChildCeilingMb, + // Not clamped to a minimum of one. A pool below `MIN_CHILD_HEAP_MB` hosts + // no child at all, and saying "1" there produced a ceiling of 0 — which V8 + // reads as its *default* heap, roughly 4 GB, against a pool of nothing. + const maxConcurrentChildren = Math.min( + Math.floor(budget.childPoolMb / MIN_CHILD_HEAP_MB), + MAX_DAEMON_WORKSPACES, ); + const perChildCeilingMb = + maxConcurrentChildren > 0 + ? Math.min( + Math.floor(budget.childPoolMb / maxConcurrentChildren), + budget.legacyChildCeilingMb, + ) + : null; return { decide(concurrentChildren) { - if (mode === 'off') return { ceilingMb: undefined, refuse: false }; - - // Refuse on the count, since the ceiling no longer varies with it. This - // is the only thing keeping the sum inside the pool. + if (mode === 'off') return { refuse: false }; const refuse = concurrentChildren > maxConcurrentChildren; if (refuse) refusals += 1; - - return { ceilingMb: perChildCeilingMb, refuse }; + return { refuse }; }, snapshot() { return { mode, - enforced: mode === 'enforce', childPoolMb: budget.childPoolMb, minChildHeapMb: MIN_CHILD_HEAP_MB, maxConcurrentChildren, diff --git a/packages/acp-bridge/src/spawnChannel.test.ts b/packages/acp-bridge/src/spawnChannel.test.ts index 75546b8bfb6..4d798b96494 100644 --- a/packages/acp-bridge/src/spawnChannel.test.ts +++ b/packages/acp-bridge/src/spawnChannel.test.ts @@ -35,14 +35,9 @@ import { EventEmitter } from 'node:events'; import type { ChildProcess } from 'node:child_process'; import { PassThrough } from 'node:stream'; -import { getHeapStatistics } from 'node:v8'; import { ProcessRegistry } from './process-registry.js'; import { createChildHeapPolicy } from './child-heap-policy.js'; -import { - MIN_CHILD_HEAP_MB, - resolveDaemonMemoryBudget, -} from './daemon-memory-budget.js'; -import { ChildHeapPoolExhaustedError } from './bridgeErrors.js'; +import { resolveDaemonMemoryBudget } from './daemon-memory-budget.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; const mockSpawn = vi.hoisted(() => vi.fn()); @@ -237,11 +232,8 @@ describe('createSpawnChannelFactory env policy', () => { }); }); -describe('createSpawnChannelFactory child-heap admission', () => { +describe('createSpawnChannelFactory child-heap observation', () => { const originalArgv1 = process.argv[1]; - // Big enough that several children fit — otherwise the very first spawn - // sits on the refusal boundary and there is no shrink to observe — but - // small enough that the boundary is still reachable in a few spawns. const budget = resolveDaemonMemoryBudget({ availableMemoryMb: 8_192 }); beforeEach(() => { @@ -255,90 +247,31 @@ describe('createSpawnChannelFactory child-heap admission', () => { delete process.env['QWEN_CLI_ENTRY']; }); - const heapArg = () => { - const argv = mockSpawn.mock.calls[0]?.[1] as string[] | undefined; - return argv?.find((a) => a.startsWith('--max-old-space-size=')); - }; - - it('applies the same partitioned ceiling to every child under enforce', async () => { - const processRegistry = new ProcessRegistry(); - const policy = createChildHeapPolicy({ budget, mode: 'enforce' }); + it('leaves argv byte-identical while counting what it would have refused', async () => { + const policy = createChildHeapPolicy({ budget, mode: 'observe' }); + const registry = new ProcessRegistry(); const factory = createSpawnChannelFactory({ - processRegistry, + processRegistry: registry, childHeapPolicy: policy, }); - const { perChildCeilingMb } = policy.snapshot(); + const limit = policy.snapshot().maxConcurrentChildren; - await factory('/tmp/a'); - const first = Number(heapArg()!.split('=')[1]); - mockSpawn.mockClear(); - await factory('/tmp/b'); - const second = Number(heapArg()!.split('=')[1]); - - // Constant, not a share of the pool at this instant. A ceiling that shrank - // as children arrived would leave the earlier, larger grants outstanding - // and the authorised total unbounded — V8 cannot lower a running child. - expect(first).toBe(perChildCeilingMb); - expect(second).toBe(perChildCeilingMb); - }); - - it('computes but applies nothing under observe', async () => { - const policy = createChildHeapPolicy({ budget, mode: 'observe' }); - const observeRegistry = new ProcessRegistry(); - await createSpawnChannelFactory({ - processRegistry: observeRegistry, - childHeapPolicy: policy, - })('/tmp/a'); - const observed = mockSpawn.mock.calls[0]?.[1] as string[]; + for (let i = 0; i < limit + 2; i++) await factory(`/tmp/w${i}`); + const observed = mockSpawn.mock.calls.at(-1)?.[1] as string[]; mockSpawn.mockClear(); - const bareRegistry = new ProcessRegistry(); - await createSpawnChannelFactory({ processRegistry: bareRegistry })( - '/tmp/a', + await createSpawnChannelFactory({ processRegistry: new ProcessRegistry() })( + '/tmp/w0', ); const bare = mockSpawn.mock.calls[0]?.[1] as string[]; - // Byte-identical argv: passing --max-old-space-size changes child GC and - // OOM behaviour, which a reporting mode must not do. + // Nothing applied: passing a derived --max-old-space-size would change the + // child's GC and OOM behaviour, which an observing mode may not do. expect(observed).toEqual(bare); - }); - - it('refuses under enforce once the pool cannot cover another child, and keeps the slot', async () => { - const processRegistry = new ProcessRegistry(); - const policy = createChildHeapPolicy({ budget, mode: 'enforce' }); - const factory = createSpawnChannelFactory({ - processRegistry, - childHeapPolicy: policy, - }); - - const fits = Math.floor(budget.childPoolMb / MIN_CHILD_HEAP_MB); - for (let i = 0; i < fits; i++) await factory(`/tmp/w${i}`); - expect(processRegistry.committedProcessCount).toBe(fits); - - await expect(factory('/tmp/over')).rejects.toBeInstanceOf( - ChildHeapPoolExhaustedError, - ); - // The refused spawn released its reservation. Leaking it would inflate - // every later count until the daemon refused everything. - expect(processRegistry.committedProcessCount).toBe(fits); - expect(policy.snapshot().refusals).toBe(1); - }); - - it('never refuses under observe, but counts what enforce would have', async () => { - const processRegistry = new ProcessRegistry(); - const policy = createChildHeapPolicy({ budget, mode: 'observe' }); - const factory = createSpawnChannelFactory({ - processRegistry, - childHeapPolicy: policy, - }); - - const fits = Math.floor(budget.childPoolMb / MIN_CHILD_HEAP_MB); - for (let i = 0; i < fits + 2; i++) await factory(`/tmp/w${i}`); - - // Everything spawned, and the counter is the calibration signal that - // enforcing here would have failed two real spawns. - expect(processRegistry.committedProcessCount).toBe(fits + 2); - expect(policy.snapshot()).toMatchObject({ enforced: false, refusals: 2 }); + // Every spawn still went through — and the two past the modeled limit are + // counted, which is the whole product of this mode. + expect(registry.committedProcessCount).toBe(limit + 2); + expect(policy.snapshot().refusals).toBe(2); }); }); @@ -640,36 +573,4 @@ describe('getAcpMemoryArgs', () => { expect(sizeMB).toBeLessThanOrEqual(16_384); } }); - - it('emits an explicit share even far below this process own heap limit', () => { - // THE regression guard for #8182. The no-argument path emits the flag only - // when it would RAISE the child above the spawning process's own limit. A - // budget-derived share is normally well below it — 614 MB against a - // multi-GB test runner — so routing it through that guard would drop the - // flag, silently restore the 25x overcommit, and break nothing else. If - // this assertion ever goes soft, the fix is gone. - const currentLimitMb = Math.floor( - getHeapStatistics().heap_size_limit / (1024 * 1024), - ); - expect(614).toBeLessThan(currentLimitMb); - expect(getAcpMemoryArgs(614)).toEqual([ - '--max-old-space-size=614', - '--expose-gc', - ]); - }); - - it('keeps the explicit path out of the module cache, in both directions', () => { - // The share depends on how many children are live right now, so caching it - // would pin the first spawn's answer for the process lifetime. Asserting - // both directions is what makes a cache-reset hook unnecessary. - const first = getAcpMemoryArgs(1_024); - const second = getAcpMemoryArgs(2_048); - expect(first).toEqual(['--max-old-space-size=1024', '--expose-gc']); - expect(second).toEqual(['--max-old-space-size=2048', '--expose-gc']); - - // And it neither poisons nor is poisoned by the cached default. - const derived = getAcpMemoryArgs(); - expect(derived).not.toContain('--max-old-space-size=2048'); - expect(getAcpMemoryArgs()).toBe(derived); - }); }); diff --git a/packages/acp-bridge/src/spawnChannel.ts b/packages/acp-bridge/src/spawnChannel.ts index 9c6cb5d0a4d..b45b433e9a2 100644 --- a/packages/acp-bridge/src/spawnChannel.ts +++ b/packages/acp-bridge/src/spawnChannel.ts @@ -14,30 +14,9 @@ import { ndJsonStream, type NdJsonStreamHooks } from './ndJsonStream.js'; import { MissingCliEntryError } from './status.js'; import { ProcessRegistry } from './process-registry.js'; import type { ChildHeapPolicy } from './child-heap-policy.js'; -import { ChildHeapPoolExhaustedError } from './bridgeErrors.js'; let cachedMemoryArgs: string[] | undefined; -/** - * V8 flags for a spawned ACP child. - * - * With no argument this is the historical behaviour: half of cgroup/host - * memory, capped at 16 GB, emitted only when it would *raise* the child above - * the spawning process's own heap limit, and cached for the process lifetime. - * Single-child callers — the interactive CLI, the IDE companion, direct-embed - * bridges — want exactly that and are unchanged. - * - * `explicitMb` is the daemon's budget-derived share, and deliberately bypasses - * **both** the cache and the raise-only guard. The cache, because the share - * depends on how many children are live right now rather than on the host. The - * guard, because a budget-derived share is normally *below* the daemon's own - * heap limit — so passing it through `targetMB > currentLimitMB` would drop - * the flag, silently restore the overcommit, and leave every test green. That - * failure mode is the whole reason this parameter exists; see #8182. - */ -export function getAcpMemoryArgs(explicitMb?: number): string[] { - if (explicitMb !== undefined) { - return [`--max-old-space-size=${explicitMb}`, '--expose-gc']; - } +export function getAcpMemoryArgs(): string[] { if (cachedMemoryArgs) return cachedMemoryArgs; const constrainedMemory = (process as { constrainedMemory?: () => number }) .constrainedMemory; @@ -175,35 +154,11 @@ export function createSpawnChannelFactory( // visible to any other spawn racing it, so the count below includes this // child and two concurrent spawns cannot both be told they are alone. const reservation = processRegistry.reserve(); - let memoryArgs: string[]; - try { - const policy = options.childHeapPolicy; - // Read the count once, while this reservation is still held, so the - // figure decided on and the figure reported are the same number. - const concurrentChildren = processRegistry.committedProcessCount; - const decision = policy?.decide(concurrentChildren); - // One snapshot for the whole decision, so the mode that gates the - // refusal is the same mode that gates whether the share is applied. - const snapshot = policy?.snapshot(); - const enforced = snapshot?.enforced ?? false; - if (snapshot && enforced && decision?.refuse) { - throw new ChildHeapPoolExhaustedError( - snapshot.childPoolMb, - concurrentChildren, - snapshot.minChildHeapMb, - ); - } - // `observe` computed a share above and must not apply it: passing the - // flag changes the child's GC and OOM behaviour, which is not something - // a reporting mode may do. Only `enforce` reaches the child. - memoryArgs = getAcpMemoryArgs(enforced ? decision?.ceilingMb : undefined); - } catch (error) { - // Covers both the refusal above and anything the policy throws. Leaking - // the reservation would inflate the count for every later spawn until - // the daemon refused everything. - reservation.cancel(); - throw error; - } + // Observation only: the policy is asked what it *would* decide so the + // refusal count is real, but nothing here acts on the answer — no derived + // ceiling reaches the child and no spawn is refused. + options.childHeapPolicy?.decide(processRegistry.committedProcessCount); + const memoryArgs = getAcpMemoryArgs(); let child; try { child = spawn( diff --git a/packages/cli/src/commands/serve.test.ts b/packages/cli/src/commands/serve.test.ts index b4a1cee177d..4f5450f9d52 100644 --- a/packages/cli/src/commands/serve.test.ts +++ b/packages/cli/src/commands/serve.test.ts @@ -348,14 +348,14 @@ describe('serve rate limit env parsing', () => { webShellMounted: false, }); - await startServeHandlerWithArgs('--no-web --child-heap-mode enforce'); + await startServeHandlerWithArgs('--no-web --child-heap-mode off'); expect(mockRunQwenServe).toHaveBeenCalledWith( - expect.objectContaining({ childHeapMode: 'enforce' }), + expect.objectContaining({ childHeapMode: 'off' }), ); }); - it('defaults the child heap mode to observe, never enforce', async () => { + it('defaults the child heap mode to observe, and rejects enforce outright', async () => { mockRunQwenServe.mockResolvedValueOnce({ url: 'http://127.0.0.1:4170/', webShellMounted: false, @@ -363,11 +363,14 @@ describe('serve rate limit env parsing', () => { await startServeHandlerWithArgs('--no-web'); - // The default is the safety property of this whole feature: enforcement - // must never switch itself on for a daemon that did not ask. expect(mockRunQwenServe).toHaveBeenCalledWith( expect.objectContaining({ childHeapMode: 'observe' }), ); + // `enforce` is not a value yet, and boot must say so rather than accept + // it: applying the partition needs an observation this daemon cannot make. + expect(() => buildParser().parseSync('--child-heap-mode enforce')).toThrow( + /Invalid values/, + ); }); it('defaults the memory pressure mode to observe', async () => { diff --git a/packages/cli/src/commands/serve.ts b/packages/cli/src/commands/serve.ts index 80b405fb574..05867013a4b 100644 --- a/packages/cli/src/commands/serve.ts +++ b/packages/cli/src/commands/serve.ts @@ -129,7 +129,7 @@ interface ServeArgs { 'mcp-client-budget'?: number; 'memory-budget-mb'?: number; 'memory-pressure-mode'?: 'off' | 'observe'; - 'child-heap-mode'?: 'off' | 'observe' | 'enforce'; + 'child-heap-mode'?: 'off' | 'observe'; 'mcp-budget-mode'?: 'enforce' | 'warn' | 'off'; 'allow-origin'?: string[]; 'allow-private-auth-base-url': boolean; @@ -332,9 +332,9 @@ export const serveCommand: CommandModule = { 'derived as 50% of cgroup-constrained ' + 'or host memory, and capped at the resolved available memory either ' + 'way. Currently observed and reported under `limits.memory` in daemon ' + - 'status. Under `--child-heap-mode enforce` it also sizes every ' + - '`qwen --acp` child and bounds how many run at once; under the ' + - 'default `observe` it sizes nothing. Must be an integer ' + + 'status, and modeled into a per-child partition reported under ' + + '`limits.memory.childHeap`. Nothing applies it: no child is sized ' + + 'from this budget. Must be an integer ' + 'in [1024, 1048576].', }) .option('memory-pressure-mode', { @@ -350,19 +350,19 @@ export const serveCommand: CommandModule = { 'either mode.', }) .option('child-heap-mode', { - choices: ['off', 'observe', 'enforce'] as const, + choices: ['off', 'observe'] as const, default: 'observe' as const, description: - 'What the daemon does with the per-child heap share it derives ' + - 'from the memory budget. `observe` (default) computes the share ' + - 'and the spawn-admission decision, applies neither, and reports ' + - 'how many spawns would have been refused — use that count to ' + - 'decide whether `enforce` is safe for your deployment. `enforce` ' + - 'passes the share to each `qwen --acp` child and refuses a spawn ' + - 'when the child pool cannot cover another at the minimum heap; ' + - 'that refusal surfaces as a failure to open a new session in the ' + - 'affected workspace, and clears when any child exits. `off` ' + - 'computes nothing and leaves children on the host-derived ceiling.', + 'Whether the daemon models a per-child heap partition of the ' + + 'memory budget. `observe` (default) reports the partition it would ' + + 'apply — `limits.memory.childHeap.perChildCeilingMb` and ' + + '`maxConcurrentChildren` — and counts spawns that would have ' + + 'exceeded it. Nothing is applied: no child is sized from the ' + + 'budget and no spawn is refused. `off` models nothing. Note a ' + + 'refusal count of 0 does NOT mean the partition would be safe to ' + + 'apply; children still run on the much larger host-derived ' + + 'ceiling, so a workload needing more old space than the modeled ' + + 'ceiling looks healthy here.', }) .option('mcp-client-budget', { type: 'number', diff --git a/packages/cli/src/serve/acp-http/dispatch-error.test.ts b/packages/cli/src/serve/acp-http/dispatch-error.test.ts index 4885f6447ef..ed0d4a1f7fa 100644 --- a/packages/cli/src/serve/acp-http/dispatch-error.test.ts +++ b/packages/cli/src/serve/acp-http/dispatch-error.test.ts @@ -6,7 +6,6 @@ import { describe, expect, it } from 'vitest'; import { DaemonDrainingError } from '../server/session-archive.js'; -import { ChildHeapPoolExhaustedError } from '../acp-session-bridge.js'; import { toRpcError } from './dispatch.js'; import { RPC } from './json-rpc.js'; @@ -19,18 +18,4 @@ describe('toRpcError', () => { data: { errorKind: 'daemon_draining' }, }); }); - - it('maps a child heap pool refusal to a retryable 503-equivalent', () => { - // The ACP mapping is hand-written beside the REST one and drifts silently - // otherwise; both carry the same contract over different transports. - const rpc = toRpcError(new ChildHeapPoolExhaustedError(3_687, 8, 512)); - expect(rpc.code).toBe(RPC.INTERNAL_ERROR); - expect(rpc.data).toMatchObject({ - errorKind: 'child_heap_pool_exhausted', - childPoolMb: 3_687, - concurrentChildren: 8, - httpStatus: 503, - retryable: true, - }); - }); }); diff --git a/packages/cli/src/serve/acp-http/dispatch.ts b/packages/cli/src/serve/acp-http/dispatch.ts index c25b9e29587..457005d1071 100644 --- a/packages/cli/src/serve/acp-http/dispatch.ts +++ b/packages/cli/src/serve/acp-http/dispatch.ts @@ -747,19 +747,6 @@ export function toRpcError(err: unknown): { retryable: true, }, }; - case 'ChildHeapPoolExhaustedError': - return { - code: RPC.INTERNAL_ERROR, - message: errMsg(err), - data: { - errorKind: 'child_heap_pool_exhausted', - childPoolMb: (err as { childPoolMb?: unknown }).childPoolMb, - concurrentChildren: (err as { concurrentChildren?: unknown }) - .concurrentChildren, - httpStatus: 503, - retryable: true, - }, - }; case 'TotalSessionLimitExceededError': return { code: RPC.INTERNAL_ERROR, diff --git a/packages/cli/src/serve/acp-session-bridge.ts b/packages/cli/src/serve/acp-session-bridge.ts index 6f820a8bc3f..dbec1b27e91 100644 --- a/packages/cli/src/serve/acp-session-bridge.ts +++ b/packages/cli/src/serve/acp-session-bridge.ts @@ -125,7 +125,6 @@ export { WorkspaceDrainingError, InvalidRewindTargetError, TotalSessionLimitExceededError, - ChildHeapPoolExhaustedError, NOT_CURRENTLY_GENERATING_CANCEL_MESSAGE, // Multi-client permission coordination errors. CancelSentinelCollisionError, diff --git a/packages/cli/src/serve/daemon-status.test.ts b/packages/cli/src/serve/daemon-status.test.ts index 23334a1f277..05252d3462a 100644 --- a/packages/cli/src/serve/daemon-status.test.ts +++ b/packages/cli/src/serve/daemon-status.test.ts @@ -131,34 +131,27 @@ describe('buildDaemonStatusResponse', () => { expect(response.limits.maxTotalSessions).toBe(50); }); - it('reports enforced only when a spawn argument really derives from the budget', async () => { - // The tripwire field. #8245 made it a required literal `false` so a client - // could never mistake this section for enforcement that had not shipped; - // it is a boolean now, and the two branches must be checked separately or - // it degrades into "the feature exists". + it('reports the modeled partition without claiming it is applied', async () => { const budget = resolveDaemonMemoryBudget({ availableMemoryMb: 8_192 }); + const options = makeOptions(); + options.opts.daemonMemoryBudget = budget; + const policy = createChildHeapPolicy({ budget, mode: 'observe' }); + policy.decide(10_000); // one would-be refusal, so the counter is not trivially 0 + options.getChildHeapPolicySnapshot = () => policy.snapshot(); + + const response = await buildDaemonStatusResponse('summary', options); - const observing = makeOptions(); - observing.opts.daemonMemoryBudget = budget; - observing.getChildHeapPolicySnapshot = () => - createChildHeapPolicy({ budget, mode: 'observe' }).snapshot(); - const observed = await buildDaemonStatusResponse('summary', observing); - // Computes everything, applies nothing — so `false`, not `true`. - expect(observed.limits.memory).toMatchObject({ + // The figures an operator needs to judge the partition for themselves — + // publishing them is the substitute for a refusal count that cannot say + // whether the ceiling would fit their workload. + expect(response.limits.memory).toMatchObject({ enforced: false, - childHeap: { mode: 'observe', refusals: 0 }, - }); - - const enforcing = makeOptions(); - enforcing.opts.daemonMemoryBudget = budget; - const policy = createChildHeapPolicy({ budget, mode: 'enforce' }); - // Drive one refusal through so the counter is not trivially zero here. - policy.decide(10_000); - enforcing.getChildHeapPolicySnapshot = () => policy.snapshot(); - const enforced = await buildDaemonStatusResponse('summary', enforcing); - expect(enforced.limits.memory).toMatchObject({ - enforced: true, - childHeap: { mode: 'enforce', refusals: 1 }, + childHeap: { + mode: 'observe', + maxConcurrentChildren: 7, + perChildCeilingMb: 526, + refusals: 1, + }, }); }); diff --git a/packages/cli/src/serve/daemon-status.ts b/packages/cli/src/serve/daemon-status.ts index 5d40b3b947c..a11fd4e8e10 100644 --- a/packages/cli/src/serve/daemon-status.ts +++ b/packages/cli/src/serve/daemon-status.ts @@ -193,26 +193,26 @@ interface DaemonStatusLimits { export interface DaemonStatusMemoryLimits { /** - * Whether a spawn argument actually derives from these numbers — i.e. only - * under `--child-heap-mode enforce`. - * - * This was a required literal `false` while the whole section was a model of - * a policy that had not shipped. It is a boolean now because the policy has, - * and it stays narrow on purpose: `observe` computes every figure below and - * applies none of them, so it still reports `false`. A client must be able - * to read this as "children are being sized by this", never as "the feature - * exists". + * False, and required. Every figure in this section is resolved input or a + * model of a policy that does not exist yet; nothing here is applied to a + * process. The flag exists so a client can never mistake the `limits` + * namespace for enforcement that has not shipped. + */ + enforced: false; + /** + * The per-child heap partition the daemon models but does not apply. + * `null` when no policy was built. */ - enforced: boolean; - /** How the derived per-child share is used. `null` when no policy was built. */ childHeap: { - mode: 'off' | 'observe' | 'enforce'; + mode: 'off' | 'observe'; + /** Children the pool could host at once. 0 when it cannot host one. */ + maxConcurrentChildren: number; + /** What each would receive. `null` when none is admissible — never 0. */ + perChildCeilingMb: number | null; /** - * Spawns refused, or — under `observe` — that would have been refused. - * The calibration signal for whether `enforce` is safe here: non-zero - * means enforcement would have failed a real spawn. Includes the - * channel-swap case, where a replacement child is counted alongside the - * process it is replacing. + * Spawns that would have exceeded `maxConcurrentChildren`. Admission + * pressure only: 0 does **not** mean the partition is safe to apply, + * because children still run on the much larger host-derived ceiling. */ refusals: number; } | null; @@ -250,11 +250,14 @@ export function toDaemonStatusMemoryLimits( ): DaemonStatusMemoryLimits | null { if (!budget) return null; return { - // Derived, never hardcoded: the whole point of the field is that a client - // can trust it to track what the daemon actually does. - enforced: childHeap?.enforced ?? false, + enforced: false, childHeap: childHeap - ? { mode: childHeap.mode, refusals: childHeap.refusals } + ? { + mode: childHeap.mode, + maxConcurrentChildren: childHeap.maxConcurrentChildren, + perChildCeilingMb: childHeap.perChildCeilingMb, + refusals: childHeap.refusals, + } : null, configuredBudgetMb: budget.configuredBudgetMb, effectiveBudgetMb: budget.effectiveBudgetMb, diff --git a/packages/cli/src/serve/fast-path.test.ts b/packages/cli/src/serve/fast-path.test.ts index 03ed7413032..fb7129f8aca 100644 --- a/packages/cli/src/serve/fast-path.test.ts +++ b/packages/cli/src/serve/fast-path.test.ts @@ -770,18 +770,18 @@ describe('serve fast path argument parsing', () => { it('parses --child-heap-mode and falls back on an unknown value', () => { for (const argv of [ - ['serve', '--child-heap-mode', 'enforce'], - ['serve', '--child-heap-mode=enforce'], + ['serve', '--child-heap-mode', 'off'], + ['serve', '--child-heap-mode=off'], ]) { expect(parseServeFastPathArgs(argv)).toMatchObject({ kind: 'serve', - options: { childHeapMode: 'enforce' }, + options: { childHeapMode: 'off' }, }); } - // Unlike memory-pressure-mode, `enforce` is a real value here — so the - // rejected sample has to be something else entirely. + // `enforce` is deliberately not a value yet, so it is the sample worth + // pinning: the fast path must defer to yargs rather than smuggle it in. expect( - parseServeFastPathArgs(['serve', '--child-heap-mode', 'warn']), + parseServeFastPathArgs(['serve', '--child-heap-mode', 'enforce']), ).toEqual({ kind: 'fallback' }); }); diff --git a/packages/cli/src/serve/fast-path.ts b/packages/cli/src/serve/fast-path.ts index dd106cc5d8f..e7ba58dd6cf 100644 --- a/packages/cli/src/serve/fast-path.ts +++ b/packages/cli/src/serve/fast-path.ts @@ -428,11 +428,7 @@ export function parseServeFastPathArgs( // Same reasoning as memory-pressure-mode: yargs `choices` already owns // the error message for a bad value, and letting an unknown string past // here would put a value in `ServeOptions` its own type forbids. - if ( - read.value !== 'off' && - read.value !== 'observe' && - read.value !== 'enforce' - ) { + if (read.value !== 'off' && read.value !== 'observe') { return { kind: 'fallback' }; } options.childHeapMode = read.value; diff --git a/packages/cli/src/serve/server/error-response.test.ts b/packages/cli/src/serve/server/error-response.test.ts index bf6653f6750..9dadb02ec09 100644 --- a/packages/cli/src/serve/server/error-response.test.ts +++ b/packages/cli/src/serve/server/error-response.test.ts @@ -13,7 +13,6 @@ import { SessionWriterUnavailableError, } from '@qwen-code/qwen-code-core'; import { sendBridgeError } from './error-response.js'; -import { ChildHeapPoolExhaustedError } from '../acp-session-bridge.js'; import { DaemonDrainingError } from './session-archive.js'; function responseMock(): { @@ -29,38 +28,6 @@ function responseMock(): { return { response: response as unknown as Response, status, json }; } -describe('sendBridgeError child heap pool exhaustion', () => { - it('maps the spawn refusal to a retryable 503 with its figures', () => { - // The spawn-policy tests exercise the throw; nothing exercised the wire - // shape, so a transport regression here would have shipped silently. - const status = vi.fn(); - const json = vi.fn(); - const set = vi.fn(); - const response = { status, json, set }; - status.mockReturnValue(response); - json.mockReturnValue(response); - set.mockReturnValue(response); - - sendBridgeError( - response as unknown as Response, - new ChildHeapPoolExhaustedError(3_687, 8, 512), - ); - - expect(status).toHaveBeenCalledWith(503); - // Retryable without operator action: the condition clears the moment any - // child exits, so the header is part of the contract, not decoration. - expect(set).toHaveBeenCalledWith('Retry-After', '5'); - expect(json).toHaveBeenCalledWith( - expect.objectContaining({ - code: 'child_heap_pool_exhausted', - childPoolMb: 3_687, - concurrentChildren: 8, - minChildHeapMb: 512, - }), - ); - }); -}); - describe('sendBridgeError session writer errors', () => { it('maps sealed session maintenance to daemon_draining', () => { const { response, status, json } = responseMock(); diff --git a/packages/cli/src/serve/server/error-response.ts b/packages/cli/src/serve/server/error-response.ts index e98a44d7f07..70b5cedfd69 100644 --- a/packages/cli/src/serve/server/error-response.ts +++ b/packages/cli/src/serve/server/error-response.ts @@ -49,7 +49,6 @@ import { WorkspaceMismatchError, WorkspaceDrainingError, TotalSessionLimitExceededError, - ChildHeapPoolExhaustedError, } from '../acp-session-bridge.js'; import type { DaemonLogger } from '../daemon-logger.js'; import { @@ -497,26 +496,6 @@ export function sendBridgeError( }); return; } - if (err instanceof ChildHeapPoolExhaustedError) { - daemonLog?.warn('child heap pool exhausted', { - ...(ctx?.route ? { route: ctx.route } : {}), - ...(ctx?.sessionId ? { sessionId: ctx.sessionId } : {}), - childPoolMb: err.childPoolMb, - concurrentChildren: err.concurrentChildren, - minChildHeapMb: err.minChildHeapMb, - }); - // Retryable in the same sense as the session limit above: the condition - // clears when any child exits, which needs no operator action. - res.set('Retry-After', '5'); - res.status(503).json({ - error: err.message, - code: 'child_heap_pool_exhausted', - childPoolMb: err.childPoolMb, - concurrentChildren: err.concurrentChildren, - minChildHeapMb: err.minChildHeapMb, - }); - return; - } if (err instanceof TotalSessionLimitExceededError) { const totalSessionError = err as TotalSessionLimitExceededError & { operation?: string; diff --git a/packages/cli/src/serve/types.ts b/packages/cli/src/serve/types.ts index 26f92fcd7cf..6a6066ee75b 100644 --- a/packages/cli/src/serve/types.ts +++ b/packages/cli/src/serve/types.ts @@ -247,25 +247,16 @@ export interface ServeOptions { */ memoryPressureMode?: 'off' | 'observe'; /** - * What the daemon does with the per-child heap share it derives from - * `memoryBudgetMb`. + * Whether the daemon models a per-child heap partition of the budget. * - * `observe` (default) computes the share and the spawn-admission decision - * and applies **neither**, counting the refusals that would have happened. - * That counter is the point: the divisor has never been checked against a - * real multi-workspace deployment, and a non-zero count says enforcement - * would have refused a real spawn — including the channel-swap case, where - * a replacement child is counted alongside the one it replaces. - * - * `enforce` passes the share to the child and refuses the spawn when the - * pool cannot cover another. Unlike `memoryPressureMode`, this mode is - * included from the start because it is reachable and testable; the default - * is what keeps it safe. - * - * `off` computes nothing and leaves children on the historical host-derived - * ceiling. - */ - childHeapMode?: 'off' | 'observe' | 'enforce'; + * `observe` (default) computes the partition and counts the spawns it would + * have refused; nothing is applied. There is no `enforce` yet — applying it + * needs a way to tell an operator in advance whether their workload fits + * the ceiling, and `refusals` cannot answer that: it counts admission + * pressure, while children still run on the far larger host-derived + * ceiling. `off` models nothing. + */ + childHeapMode?: 'off' | 'observe'; /** * Resolved at boot by `runQwenServe`. Not an operator input, and not * consumed by any spawn path — it is reported under `limits.memory` on diff --git a/packages/sdk-typescript/src/daemon/types.ts b/packages/sdk-typescript/src/daemon/types.ts index cbd827cc469..afc546b4c78 100644 --- a/packages/sdk-typescript/src/daemon/types.ts +++ b/packages/sdk-typescript/src/daemon/types.ts @@ -628,23 +628,20 @@ export interface DaemonStatusReport { */ memory?: { /** Always false: nothing in this section is applied to a process. */ + /** False, and required: nothing in this section is applied to a process. */ + enforced: false; /** - * Whether children are actually being sized by these numbers — true only - * under `--child-heap-mode enforce`. Was a required literal `false` while - * the section described a policy that had not shipped; now a boolean, and - * still `false` under `observe`, which computes everything and applies - * nothing. Read it as "this is in effect", never as "this exists". - */ - enforced: boolean; - /** - * How the derived per-child heap share is used. `null` when the daemon - * built no policy, and absent entirely on daemons predating the field. + * The per-child heap partition the daemon models but does not apply. + * `null` when no policy was built; absent on daemons predating it. */ childHeap?: { - mode: 'off' | 'observe' | 'enforce'; + mode: 'off' | 'observe'; + maxConcurrentChildren: number; + /** `null` when no child is admissible — never 0. */ + perChildCeilingMb: number | null; /** - * Spawns refused, or under `observe` that would have been refused — - * the signal for whether enabling `enforce` is safe on this host. + * Admission pressure only. 0 does not mean the partition is safe to + * apply: children still run on the host-derived ceiling. */ refusals: number; } | null;