diff --git a/packages/runtime-host/src/__tests__/execution-host.test.ts b/packages/runtime-host/src/__tests__/execution-host.test.ts index 5c42d80c46..ba172668ad 100644 --- a/packages/runtime-host/src/__tests__/execution-host.test.ts +++ b/packages/runtime-host/src/__tests__/execution-host.test.ts @@ -216,7 +216,8 @@ test('graceful Host shutdown stops and drains an active Turn before releasing ow text: FAKE_ASK_USER_QUESTION_PROMPT, }); - await fixture.stopHost(host); + const exit = await fixture.stopHost(host); + assert.deepEqual(exit, { code: 0, signal: null }); await client.closed; const successor = await fixture.startHost(); @@ -602,12 +603,19 @@ class ExecutionFixture { await owner?.close(); } - async stopHost(host: ExecutionHostHandle): Promise { + async stopHost( + host: ExecutionHostHandle, + ): Promise<{ code: number | null; signal: NodeJS.Signals | null }> { if (host.child.exitCode === null && host.child.signalCode === null) { host.child.kill('SIGTERM'); } - await withTimeout(waitForExit(host.child), PROCESS_TIMEOUT_MS, 'execution Host did not stop'); + const exit = await withTimeout( + waitForExitResult(host.child), + PROCESS_TIMEOUT_MS, + 'execution Host did not stop', + ); this.#children.delete(host.child); + return exit; } async killHost(host: ExecutionHostHandle): Promise { @@ -889,6 +897,30 @@ function waitForExit(child: ChildProcess): Promise { }); } +function waitForExitResult( + child: ChildProcess, +): Promise<{ code: number | null; signal: NodeJS.Signals | null }> { + if (child.exitCode !== null || child.signalCode !== null) { + return Promise.resolve({ code: child.exitCode, signal: child.signalCode }); + } + return new Promise((resolve, reject) => { + const cleanup = () => { + child.off('error', onError); + child.off('exit', onExit); + }; + const onError = (error: Error) => { + cleanup(); + reject(error); + }; + const onExit = (code: number | null, signal: NodeJS.Signals | null) => { + cleanup(); + resolve({ code, signal }); + }; + child.once('error', onError); + child.once('exit', onExit); + }); +} + async function acquireReader(capability: StorageRootCapability<'interactive'>) { const deadline = Date.now() + PROCESS_TIMEOUT_MS; while (true) { diff --git a/packages/runtime-host/src/__tests__/fixtures/execution-host.ts b/packages/runtime-host/src/__tests__/fixtures/execution-host.ts index 3172012604..0ce0003abd 100644 --- a/packages/runtime-host/src/__tests__/fixtures/execution-host.ts +++ b/packages/runtime-host/src/__tests__/fixtures/execution-host.ts @@ -1,4 +1,5 @@ import { startExecutionRuntimeHostCandidate } from '../../server/execution-candidate.js'; +import { runRuntimeHostProcessLifecycle } from '../../server/process-lifecycle.js'; const [rootPath, expectedRootId, idleGraceRaw] = process.argv.slice(2); if (!rootPath || !expectedRootId || !/^[a-f0-9]{64}$/.test(expectedRootId)) { @@ -22,17 +23,8 @@ process.send?.({ endpoint: result.host.endpoint, }); -let closing = false; -const close = () => { - if (closing) return; - closing = true; - void result.host.close(); -}; -process.once('SIGINT', close); -process.once('SIGTERM', close); -process.once('disconnect', close); try { - await result.host.closed; + await runRuntimeHostProcessLifecycle(result.host, { closeOnDisconnect: true }); } catch { process.exitCode = 1; } finally { diff --git a/packages/runtime-host/src/__tests__/fixtures/uncooperative-host.ts b/packages/runtime-host/src/__tests__/fixtures/uncooperative-host.ts new file mode 100644 index 0000000000..05d59a00ad --- /dev/null +++ b/packages/runtime-host/src/__tests__/fixtures/uncooperative-host.ts @@ -0,0 +1,66 @@ +import { + resolveExistingStorageRoot, + tryAcquireInteractiveRootOwner, +} from '@maka/storage/root-authority'; +import { RuntimeHostKernel, type RuntimeHostComposition } from '../../server/host-kernel.js'; +import { runRuntimeHostProcessLifecycle } from '../../server/process-lifecycle.js'; + +const [rootPath, expectedRootId, shutdownGraceRaw] = process.argv.slice(2); +if (!rootPath || !expectedRootId || !/^[a-f0-9]{64}$/.test(expectedRootId)) { + throw new Error('usage: uncooperative-host '); +} +const shutdownGraceMs = Number(shutdownGraceRaw); +if (!Number.isSafeInteger(shutdownGraceMs) || shutdownGraceMs <= 0) { + throw new Error('uncooperative-host requires a positive shutdown grace'); +} + +const capability = await resolveExistingStorageRoot({ + path: rootPath, + kind: 'interactive', + expectedRootId, +}); +const owner = await tryAcquireInteractiveRootOwner(capability); +if (!owner) throw new Error('uncooperative-host could not acquire the Interactive root'); + +const host = await RuntimeHostKernel.start({ + owner, + idleGraceMs: 60_000, + shutdownGraceMs, + compositionFactory: async (context): Promise => ({ + handlers: { + 'turn.start': async () => { + context.acquireResidency(); + process.send?.({ type: 'operation-blocked' }); + return new Promise(() => undefined); + }, + 'turn.query': async () => ({ + ok: false, + error: { code: 'operation_unavailable', message: 'Operation unavailable in test Host' }, + }), + 'turn.stop': async () => ({ + ok: false, + error: { code: 'operation_unavailable', message: 'Operation unavailable in test Host' }, + }), + }, + async recover() {}, + async close() {}, + }), +}); + +process.on('message', (message: unknown) => { + if ( + message && + typeof message === 'object' && + (message as { type?: unknown }).type === 'shutdown' + ) { + void host.close(); + process.send?.({ type: 'shutdown-requested' }); + } +}); +process.send?.({ type: 'ready', hostEpoch: host.hostEpoch, endpoint: host.endpoint }); + +try { + await runRuntimeHostProcessLifecycle(host, { closeOnDisconnect: true }); +} catch { + process.exitCode = 1; +} diff --git a/packages/runtime-host/src/__tests__/host-kernel.test.ts b/packages/runtime-host/src/__tests__/host-kernel.test.ts index df64b6dd2a..6ea10a2248 100644 --- a/packages/runtime-host/src/__tests__/host-kernel.test.ts +++ b/packages/runtime-host/src/__tests__/host-kernel.test.ts @@ -713,6 +713,112 @@ describe('non-serving Runtime Host kernel', () => { }); }); + test('forces an uncooperative command Host to exit before a successor acquires ownership', { + timeout: 10_000, + }, async () => { + await withHostPaths(async (paths) => { + const capability = await resolveStorageRoot({ path: paths.root, kind: 'interactive' }); + const child = paths.resources.trackChild( + fork( + new URL('./fixtures/uncooperative-host.js', import.meta.url), + [paths.root, capability.rootId, '2000'], + { stdio: ['ignore', 'ignore', 'inherit', 'ipc'] }, + ), + ); + let transport: FramedTransport | undefined; + try { + const ready = await waitForUncooperativeHostMessage(child, 'ready'); + transport = new FramedTransport(await openSocket(ready.endpoint)); + await transport.write({ + kind: 'hello', + clientInstanceId: 'bounded-shutdown-test', + surface: 'tui', + protocolMin: CURRENT_PROTOCOL.min, + protocolMax: CURRENT_PROTOCOL.max, + }); + const handshake = decodeHostFrame(await transport.read(2_000)); + assert.ok('kind' in handshake); + assert.equal(handshake.kind, 'accepted'); + + const blocked = waitForUncooperativeHostMessage(child, 'operation-blocked'); + await transport.write({ + requestId: 'blocked-turn-start', + operation: 'turn.start', + input: { sessionId: 'session', turnId: 'turn', text: 'block forever' }, + }); + await blocked; + const shutdownRequested = waitForUncooperativeHostMessage(child, 'shutdown-requested'); + child.send({ type: 'shutdown' }); + await shutdownRequested; + + await transport.write({ + requestId: 'post-drain-status', + operation: 'host.status', + input: {}, + }); + const rejectedOperation = decodeHostFrame(await transport.read(1_000)); + assert.ok(!('kind' in rejectedOperation)); + if (!('kind' in rejectedOperation)) { + assert.equal(rejectedOperation.requestId, 'post-drain-status'); + assert.equal(rejectedOperation.operation, 'host.status'); + assert.equal(rejectedOperation.ok, false); + if (!rejectedOperation.ok) assert.equal(rejectedOperation.error.code, 'host_draining'); + } + + const rejectedHandshakeTransport = new FramedTransport(await openSocket(ready.endpoint)); + try { + await rejectedHandshakeTransport.write({ + kind: 'hello', + clientInstanceId: 'post-drain-client', + surface: 'inspect', + protocolMin: CURRENT_PROTOCOL.min, + protocolMax: CURRENT_PROTOCOL.max, + }); + assert.deepEqual(decodeHostFrame(await rejectedHandshakeTransport.read(1_000)), { + kind: 'draining', + hostEpoch: ready.hostEpoch, + }); + } finally { + rejectedHandshakeTransport.destroy(); + } + + assert.equal(child.exitCode, null); + assert.equal(child.signalCode, null); + const contender = await tryAcquireInteractiveRootOwner(capability); + try { + assert.equal(contender, undefined); + } finally { + await contender?.close(); + } + const exit = await withTimeout( + waitForChildExitResult(child), + 5_000, + 'uncooperative Runtime Host did not exit within its shutdown bound', + ); + assert.deepEqual(exit, { code: 1, signal: null }); + + const successor = await startTestRuntimeHostCandidate(paths, { + rootPath: paths.root, + idleGraceMs: 10_000, + }); + assert.equal(successor.kind, 'winner'); + if (successor.kind !== 'winner') return; + assert.notEqual(successor.host.hostEpoch, ready.hostEpoch); + const connected = await retryConnect(paths, CURRENT_PROTOCOL); + assert.equal(connected.kind, 'connected'); + if (connected.kind !== 'connected') return; + const status = await connected.connection.status(); + assert.equal(status.hostEpoch, successor.host.hostEpoch); + await connected.connection.close(); + await successor.host.close(); + } finally { + transport?.destroy(); + if (child.exitCode === null && child.signalCode === null) child.kill('SIGKILL'); + await withTimeout(waitForExit(child), 1_000, 'uncooperative Host cleanup did not exit'); + } + }); + }); + test('startup rejects invalid lifecycle durations and releases the owner lock', async () => { await withHostPaths(async (paths) => { await assert.rejects( @@ -731,6 +837,16 @@ describe('non-serving Runtime Host kernel', () => { }), RangeError, ); + const capability = await resolveStorageRoot({ path: paths.root, kind: 'interactive' }); + const owner = paths.resources.trackCloseable( + await tryAcquireInteractiveRootOwner(capability), + ); + assert.ok(owner); + if (!owner) return; + await assert.rejects( + () => RuntimeHostKernel.start({ owner, shutdownGraceMs: 0 }), + RangeError, + ); const retry = await startTestRuntimeHostCandidate(paths, { rootPath: paths.root, idleGraceMs: 0, @@ -1199,6 +1315,80 @@ function isElectronParentLaunch( ); } +type UncooperativeHostMessage = + | { type: 'ready'; hostEpoch: string; endpoint: string } + | { type: 'operation-blocked' } + | { type: 'shutdown-requested' }; + +function waitForUncooperativeHostMessage( + child: ChildProcess, + type: T, +): Promise> { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + cleanup(); + reject(new Error(`uncooperative Host did not report ${type}`)); + }, 5_000); + const cleanup = () => { + clearTimeout(timer); + child.off('error', onError); + child.off('exit', onExit); + child.off('message', onMessage); + }; + const onError = (error: Error) => { + cleanup(); + reject(error); + }; + const onExit = (code: number | null, signal: NodeJS.Signals | null) => { + cleanup(); + reject(new Error(`uncooperative Host exited before ${type}: ${code ?? signal}`)); + }; + const onMessage = (message: unknown) => { + if (!isUncooperativeHostMessage(message) || message.type !== type) return; + cleanup(); + resolve(message as Extract); + }; + child.once('error', onError); + child.once('exit', onExit); + child.on('message', onMessage); + }); +} + +function isUncooperativeHostMessage(value: unknown): value is UncooperativeHostMessage { + if (!value || typeof value !== 'object') return false; + const message = value as Record; + if (message.type === 'operation-blocked' || message.type === 'shutdown-requested') return true; + return ( + message.type === 'ready' && + typeof message.hostEpoch === 'string' && + typeof message.endpoint === 'string' + ); +} + +function waitForChildExitResult( + child: ChildProcess, +): Promise<{ code: number | null; signal: NodeJS.Signals | null }> { + if (child.exitCode !== null || child.signalCode !== null) { + return Promise.resolve({ code: child.exitCode, signal: child.signalCode }); + } + return new Promise((resolve, reject) => { + const cleanup = () => { + child.off('error', onError); + child.off('exit', onExit); + }; + const onError = (error: Error) => { + cleanup(); + reject(error); + }; + const onExit = (code: number | null, signal: NodeJS.Signals | null) => { + cleanup(); + resolve({ code, signal }); + }; + child.once('error', onError); + child.once('exit', onExit); + }); +} + function waitForExit(child: ChildProcess): Promise { if (child.exitCode !== null || child.signalCode !== null) return Promise.resolve(); return new Promise((resolve) => child.once('exit', () => resolve())); diff --git a/packages/runtime-host/src/candidate-main.ts b/packages/runtime-host/src/candidate-main.ts index a1dda08ca2..0539facd8b 100644 --- a/packages/runtime-host/src/candidate-main.ts +++ b/packages/runtime-host/src/candidate-main.ts @@ -1,20 +1,13 @@ #!/usr/bin/env node import { startRuntimeHostCandidate, type RuntimeHostCandidateOptions } from './server/candidate.js'; +import { runRuntimeHostProcessLifecycle } from './server/process-lifecycle.js'; const options = parseArguments(process.argv.slice(2)); const result = await startRuntimeHostCandidate(options); if (result.kind === 'loser') process.exit(2); -let closing = false; -const close = () => { - if (closing) return; - closing = true; - void result.host.close(); -}; -process.once('SIGINT', close); -process.once('SIGTERM', close); try { - await result.host.closed; + await runRuntimeHostProcessLifecycle(result.host); } catch { process.exitCode = 1; } diff --git a/packages/runtime-host/src/server/host-kernel.ts b/packages/runtime-host/src/server/host-kernel.ts index 7a5cbdbe42..7fdb332c32 100644 --- a/packages/runtime-host/src/server/host-kernel.ts +++ b/packages/runtime-host/src/server/host-kernel.ts @@ -33,6 +33,7 @@ import { const DEFAULT_IDLE_GRACE_MS = 30_000; const DEFAULT_HANDSHAKE_TIMEOUT_MS = 5_000; +const DEFAULT_SHUTDOWN_GRACE_MS = 10_000; const SHUTDOWN_HANDSHAKE_GRACE_MS = 1_000; const SHUTDOWN_OPERATION_GRACE_MS = 1_000; const HOST_PROTOCOL = { @@ -42,6 +43,15 @@ const HOST_PROTOCOL = { export type RuntimeHostResidency = OperationResidency; +export class RuntimeHostProcessTerminationRequiredError extends Error { + readonly code = 'process_termination_required'; + + constructor(readonly shutdownGraceMs: number) { + super(`Runtime Host did not shut down within ${shutdownGraceMs} ms`); + this.name = 'RuntimeHostProcessTerminationRequiredError'; + } +} + export interface RuntimeHostCompositionContext { owner: InteractiveRootOwner; acquireResidency(): RuntimeHostResidency; @@ -62,6 +72,7 @@ export interface RuntimeHostKernelOptions { owner: InteractiveRootOwner; idleGraceMs?: number; handshakeTimeoutMs?: number; + shutdownGraceMs?: number; compositionFactory?: RuntimeHostCompositionFactory; } @@ -77,6 +88,7 @@ export class RuntimeHostKernel { readonly #residencyDrainWaiters = new Set<() => void>(); readonly #idleGraceMs: number; readonly #handshakeTimeoutMs: number; + readonly #shutdownGraceMs: number; #endpoint: RuntimeHostEndpoint | undefined; #state: HostLifecycleState = 'starting'; #activeOperations = 0; @@ -87,6 +99,8 @@ export class RuntimeHostKernel { #idleTimer: NodeJS.Timeout | undefined; #shutdownRequested = false; #shutdownTask: Promise | undefined; + #shutdownDeadlineTimer: NodeJS.Timeout | undefined; + #terminationRequired: RuntimeHostProcessTerminationRequiredError | undefined; #resolveClosed!: () => void; #rejectClosed!: (error: unknown) => void; @@ -97,8 +111,10 @@ export class RuntimeHostKernel { 'handshakeTimeoutMs', 1, ); + assertDuration(options.shutdownGraceMs ?? DEFAULT_SHUTDOWN_GRACE_MS, 'shutdownGraceMs', 1); this.#idleGraceMs = options.idleGraceMs ?? DEFAULT_IDLE_GRACE_MS; this.#handshakeTimeoutMs = options.handshakeTimeoutMs ?? DEFAULT_HANDSHAKE_TIMEOUT_MS; + this.#shutdownGraceMs = options.shutdownGraceMs ?? DEFAULT_SHUTDOWN_GRACE_MS; this.#options = options; this.#operationHandlers = this.#createOperationHandlers(unavailableDomainHandlers()); this.closed = new Promise((resolve, reject) => { @@ -116,6 +132,7 @@ export class RuntimeHostKernel { owner, idleGraceMs: options.idleGraceMs, handshakeTimeoutMs: options.handshakeTimeoutMs, + shutdownGraceMs: options.shutdownGraceMs, compositionFactory: options.compositionFactory, }); await host.#start(); @@ -146,8 +163,11 @@ export class RuntimeHostKernel { } #requestDrain(): void { - this.#shutdownRequested = true; - this.#cancelIdle(); + if (!this.#shutdownRequested) { + this.#shutdownRequested = true; + this.#cancelIdle(); + this.#armShutdownDeadline(); + } this.#commitRequestedShutdownIfQuiescent(); } @@ -228,10 +248,10 @@ export class RuntimeHostKernel { hello: ClientHello, transport: FramedTransport, ): Promise { - if (!(await this.#hasLiveOwnerOrDrain()) || this.#state === 'draining') { + const admittedState = await this.#readAdmissionState(); + if (!admittedState) { return { kind: 'draining', hostEpoch: this.hostEpoch }; } - const admittedState = this.#state; const selectedProtocol = negotiateProtocol( { min: hello.protocolMin, max: hello.protocolMax }, HOST_PROTOCOL, @@ -268,10 +288,7 @@ export class RuntimeHostKernel { async #beginOperation( frame: RequestFrame, ): Promise { - if (!(await this.#hasLiveOwnerOrDrain()) || this.#state === 'draining') return 'host_draining'; - if (this.#shutdownRequested && HOST_OPERATION_SPECS[frame.operation].mode === 'command') { - return 'host_draining'; - } + if (!(await this.#readAdmissionState())) return 'host_draining'; if ( HOST_OPERATION_SPECS[frame.operation].admission !== 'bootstrap' && this.#state !== 'ready' @@ -321,6 +338,13 @@ export class RuntimeHostKernel { return !this.#isDraining(); } + async #readAdmissionState(): Promise | undefined> { + if (this.#shutdownRequested || this.#isDraining()) return undefined; + if (!(await this.#hasLiveOwnerOrDrain())) return undefined; + const state = this.#state; + return this.#shutdownRequested || state === 'draining' ? undefined : state; + } + #isDraining(): boolean { return this.#state === 'draining'; } @@ -419,19 +443,53 @@ export class RuntimeHostKernel { } #commitShutdown(): Promise { + if (this.#terminationRequired) return this.closed; if (!this.#shutdownTask) { - this.#shutdownRequested = true; + if (!this.#shutdownRequested) { + this.#shutdownRequested = true; + this.#armShutdownDeadline(); + } this.#state = 'draining'; this.#cancelIdle(); this.#shutdownTask = this.#closeResources(); - void this.#shutdownTask.then(this.#resolveClosed, this.#rejectClosed); + void this.#shutdownTask.then( + () => { + this.#clearShutdownDeadline(); + if (!this.#terminationRequired) this.#resolveClosed(); + }, + (error: unknown) => { + this.#clearShutdownDeadline(); + if (!this.#terminationRequired) this.#rejectClosed(error); + }, + ); } return this.closed; } + #armShutdownDeadline(): void { + if (this.#shutdownDeadlineTimer || this.#terminationRequired) return; + this.#shutdownDeadlineTimer = setTimeout(() => { + this.#shutdownDeadlineTimer = undefined; + const error = new RuntimeHostProcessTerminationRequiredError(this.#shutdownGraceMs); + this.#terminationRequired = error; + this.#rejectClosed(error); + }, this.#shutdownGraceMs); + } + + #clearShutdownDeadline(): void { + if (!this.#shutdownDeadlineTimer) return; + clearTimeout(this.#shutdownDeadlineTimer); + this.#shutdownDeadlineTimer = undefined; + } + + #assertShutdownCanContinue(): void { + if (this.#terminationRequired) throw this.#terminationRequired; + } + async #closeResources(): Promise { const errors: unknown[] = []; await this.#publishRegistration().catch((error: unknown) => errors.push(error)); + this.#assertShutdownCanContinue(); const serverClosed = closeServer(this.#server).catch((error: unknown) => errors.push(error)); const accepted = [...this.#acceptedTransports]; const handshaking = [...this.#handshakingTransports]; @@ -440,20 +498,28 @@ export class RuntimeHostKernel { waitForBoundedCompletion(operationDrain, SHUTDOWN_OPERATION_GRACE_MS), waitForTransportClose(handshaking, SHUTDOWN_HANDSHAKE_GRACE_MS), ]); + this.#assertShutdownCanContinue(); if (!operationsDrained) { for (const transport of accepted) transport.destroy(); } for (const transport of handshaking) transport.destroy(); await operationDrain; + this.#assertShutdownCanContinue(); await this.#composition?.close().catch((error: unknown) => errors.push(error)); + this.#assertShutdownCanContinue(); await this.#waitForResidencies(); + this.#assertShutdownCanContinue(); for (const transport of accepted) transport.destroy(); await serverClosed; + this.#assertShutdownCanContinue(); await this.#endpoint?.cleanup().catch((error: unknown) => errors.push(error)); + this.#assertShutdownCanContinue(); await removeHostRegistration(this.#options.owner.controlDirectory, this.hostEpoch).catch( (error: unknown) => errors.push(error), ); + this.#assertShutdownCanContinue(); await this.#options.owner.close().catch((error: unknown) => errors.push(error)); + this.#assertShutdownCanContinue(); if (errors.length > 0) throw new AggregateError( errors, diff --git a/packages/runtime-host/src/server/index.ts b/packages/runtime-host/src/server/index.ts index d033a0861c..582255e0f4 100644 --- a/packages/runtime-host/src/server/index.ts +++ b/packages/runtime-host/src/server/index.ts @@ -1,5 +1,6 @@ export { RuntimeHostKernel, + RuntimeHostProcessTerminationRequiredError, type RuntimeHostComposition, type RuntimeHostCompositionContext, type RuntimeHostCompositionFactory, diff --git a/packages/runtime-host/src/server/process-lifecycle.ts b/packages/runtime-host/src/server/process-lifecycle.ts new file mode 100644 index 0000000000..c210111274 --- /dev/null +++ b/packages/runtime-host/src/server/process-lifecycle.ts @@ -0,0 +1,34 @@ +import { + RuntimeHostProcessTerminationRequiredError, + type RuntimeHostKernel, +} from './host-kernel.js'; + +export interface RuntimeHostProcessLifecycleOptions { + closeOnDisconnect?: boolean; +} + +export async function runRuntimeHostProcessLifecycle( + host: RuntimeHostKernel, + options: RuntimeHostProcessLifecycleOptions = {}, +): Promise { + let closing = false; + const close = () => { + if (closing) return; + closing = true; + void host.close(); + }; + + process.once('SIGINT', close); + process.once('SIGTERM', close); + if (options.closeOnDisconnect) process.once('disconnect', close); + try { + await host.closed; + } catch (error) { + if (error instanceof RuntimeHostProcessTerminationRequiredError) process.exit(1); + throw error; + } finally { + process.off('SIGINT', close); + process.off('SIGTERM', close); + if (options.closeOnDisconnect) process.off('disconnect', close); + } +}