diff --git a/src/lib/actions/sandbox/connect-hermes-portable-inference-recovery-errors.test.ts b/src/lib/actions/sandbox/connect-hermes-portable-inference-recovery-errors.test.ts index 885d3176f43..2ee77d4b801 100644 --- a/src/lib/actions/sandbox/connect-hermes-portable-inference-recovery-errors.test.ts +++ b/src/lib/actions/sandbox/connect-hermes-portable-inference-recovery-errors.test.ts @@ -55,6 +55,50 @@ describe("Hermes Portable connect recovery errors", () => { expect(output).not.toContain("Hermes Portable inference recovery for 'alpha' failed"); }); + it("verifies a compatible-endpoint route without Ollama recovery", async () => { + const entry = { + name: "alpha", + agent: "hermes", + provider: "compatible-endpoint", + model: "descriptor/model", + policies: [], + openshellDriver: "docker", + gatewayName: "nemoclaw", + lifecycleGeneration: "generation-1", + } as never; + const harness = createConnectHarness({ + agentName: "hermes", + sessionAgent: { name: "hermes" }, + registryEntry: entry, + inferenceGetOutput: + "Gateway inference:\n Provider: compatible-endpoint\n Model: descriptor/model\n", + inferenceProbeResponses: ["OK 200"], + portableReceiptDisposition: { kind: "hermes", phase: "active" }, + portableRecoveryResult: { kind: "already-running" }, + readinessDecision: { + kind: "accepted", + category: "accepted", + agent: { name: "hermes" }, + sb: entry, + }, + }); + + await expect(harness.connectSandbox("alpha", { probeOnly: true })).resolves.toBeUndefined(); + + expect(harness.registryEntries[0]?.hostLocalInferenceReceipt).toBeUndefined(); + expect(harness.recoverHermesPortableOllamaInferenceSpy).not.toHaveBeenCalled(); + expect(harness.captureResolvedOpenshellSpy).toHaveBeenCalledWith( + ["inference", "get", "-g", "nemoclaw"], + expect.objectContaining({ openshellBinary: "/usr/bin/openshell" }), + ); + expect( + harness.captureResolvedOpenshellSpy.mock.calls.some( + ([args]) => Array.isArray(args) && args[0] === "sandbox" && args[1] === "exec", + ), + ).toBe(true); + expect(harness.publishLaunchReadinessSpy).not.toHaveBeenCalled(); + }); + it.each([ [ "authority drift", diff --git a/src/lib/actions/sandbox/connect.ts b/src/lib/actions/sandbox/connect.ts index 85631d8a6c5..2a2c4d738a5 100644 --- a/src/lib/actions/sandbox/connect.ts +++ b/src/lib/actions/sandbox/connect.ts @@ -113,7 +113,8 @@ import { HermesPortableForwardRecoveryError, type HermesPortableForwardRecoveryFailure, type ManagedGatewayControlCompletion, - recoverHermesPortableLaunchForwards, + prepareHermesPortableLaunchForwards, + type PreparedHermesPortableForwardRecovery, resolveSandboxDashboardPort, resolveSandboxLaunchForwardPorts, waitForManagedGatewaySupervisor, @@ -330,40 +331,26 @@ async function runSandboxConnectProbe( const agentName = agentRuntime.getAgentDisplayName(agent); if (hermesPortable) { if (probeOnly !== true) throw new Error("Hermes inference recovery requires probe-only mode"); - measure("inference", () => - recoverHermesPortableInferenceRouteForProbeOnlyOrExit(sandboxName, agent), + const route = measure("inference", () => + verifyOrRecoverHermesPortableInferenceRouteForProbeOnlyOrExit(sandboxName, agent, undefined, { + probeTiming, + }), ); - let authority: HermesPortableActiveLifecycleAuthority; - try { - authority = requireHermesPortableActiveLifecycleAuthority( - sandboxName, - undefined, - portableAgentLifecycleAuthorityDeps(), - ); - } catch { - probeTiming?.setForwardAction("failed"); - probeTiming?.markFailureStage("forward"); - failHermesPortableForwardRecovery(sandboxName, "authority-drift"); - } - let forwardRecovery: ReturnType; - try { - forwardRecovery = measure("forward", () => - recoverHermesPortableForwardsForConnectProbe({ - intent: "connect-probe-only", + if (!route.forwardsRecovered) { + let authority: HermesPortableActiveLifecycleAuthority; + try { + authority = requireHermesPortableActiveLifecycleAuthority( sandboxName, - authority, - readRegistry: registry.getSandbox, - }), - ); - } catch (error) { - probeTiming?.setForwardAction("failed"); - probeTiming?.markFailureStage("forward"); - failHermesPortableForwardRecovery( - sandboxName, - error instanceof HermesPortableForwardRecoveryError ? error.failure : "recovery-failed", - ); + undefined, + portableAgentLifecycleAuthorityDeps(), + ); + } catch { + probeTiming?.setForwardAction("failed"); + probeTiming?.markFailureStage("forward"); + failHermesPortableForwardRecovery(sandboxName, "authority-drift"); + } + recoverHermesPortableForwardsForConnectProbeOrExit(sandboxName, authority, probeTiming); } - probeTiming?.setForwardAction(forwardRecovery.kind === "restored" ? "restored" : "verified"); console.log( ` Probe complete: ${agentName} passed receipt-owned authenticated health in '${sandboxName}'.`, ); @@ -566,7 +553,7 @@ type HermesPortableForwardConnectRecoveryInput = { }; /** Restore the launch-readiness forwards through current Hermes command authority. */ -function recoverHermesPortableForwardsForConnectProbe( +function prepareHermesPortableForwardsForConnectProbe( input: HermesPortableForwardConnectRecoveryInput, ) { const expectedEntry = structuredClone(input.authority.entry); @@ -623,7 +610,7 @@ function recoverHermesPortableForwardsForConnectProbe( }); }; - return recoverHermesPortableLaunchForwards({ + return prepareHermesPortableLaunchForwards({ intent: input.intent, sandboxName: input.sandboxName, gatewayName: input.authority.gatewayName, @@ -639,6 +626,50 @@ function recoverHermesPortableForwardsForConnectProbe( }); } +function recoverHermesPortableForwardsForConnectProbeOrExit( + sandboxName: string, + authority: HermesPortableActiveLifecycleAuthority, + probeTiming?: ProbeTimingRecorder, +): void { + try { + const prepared = prepareHermesPortableForwardsForConnectProbeMeasured( + sandboxName, + authority, + probeTiming, + ); + const forwardRecovery = prepared.release(); + probeTiming?.setForwardAction(forwardRecovery.kind === "restored" ? "restored" : "verified"); + } catch (error) { + failHermesPortableForwardRecovery( + sandboxName, + error instanceof HermesPortableForwardRecoveryError ? error.failure : "recovery-failed", + ); + } +} + +function prepareHermesPortableForwardsForConnectProbeMeasured( + sandboxName: string, + authority: HermesPortableActiveLifecycleAuthority, + probeTiming?: ProbeTimingRecorder, +) { + try { + const prepare = () => + prepareHermesPortableForwardsForConnectProbe({ + intent: "connect-probe-only", + sandboxName, + authority, + readRegistry: registry.getSandbox, + }); + return probeTiming ? probeTiming.measure("forward", prepare) : prepare(); + } catch (error) { + probeTiming?.setForwardAction("failed"); + probeTiming?.markFailureStage("forward"); + throw error instanceof HermesPortableForwardRecoveryError + ? error + : new HermesPortableForwardRecoveryError("recovery-failed"); + } +} + class HermesPortableInferenceRouteVerificationError extends Error { constructor(readonly reason: string) { super("Hermes Portable inference route verification failed"); @@ -743,12 +774,22 @@ function verifyHermesPortableInferenceRouteOrExit( } } -/** Resume published Ollama authority only for the explicit probe-only command. */ -function recoverHermesPortableInferenceRouteForProbeOnlyOrExit( +type HermesPortableProbeRouteResult = { + readonly forwardsRecovered: boolean; +}; + +type HermesPortableProbeRouteOptions = { + readonly probeTiming?: ProbeTimingRecorder; + readonly validateVerified?: (entry: SandboxEntry) => void; +}; + +/** Verify the recorded route and resume published Ollama only for probe-only recovery. */ +function verifyOrRecoverHermesPortableInferenceRouteForProbeOnlyOrExit( sandboxName: string, agent: InferenceRouteProbeAgent, expectedAuthority?: HermesPortableActiveLifecycleAuthority, -): SandboxEntry { + options: HermesPortableProbeRouteOptions = {}, +): HermesPortableProbeRouteResult { let authority: HermesPortableActiveLifecycleAuthority; try { authority = requireHermesPortableActiveLifecycleAuthority( @@ -759,7 +800,17 @@ function recoverHermesPortableInferenceRouteForProbeOnlyOrExit( } catch { failHermesPortableInferenceRoute(sandboxName, "missing or incomplete"); } + if (authority.entry.provider !== "ollama-local") { + const entry = verifyHermesPortableInferenceRouteOrExit(sandboxName, agent, authority); + try { + options.validateVerified?.(entry); + } catch { + failHermesPortableInferenceRoute(sandboxName, "changed during verification"); + } + return { forwardsRecovered: false }; + } let verified: SandboxEntry | null = null; + let preparedForwards: PreparedHermesPortableForwardRecovery | null = null; try { recoverHermesPortableInferenceForConnectProbe({ sandboxName, @@ -767,10 +818,26 @@ function recoverHermesPortableInferenceRouteForProbeOnlyOrExit( readRegistry: registry.getSandbox, verifyRoute: () => { verified = verifyHermesPortableInferenceRoute(sandboxName, agent, authority); + try { + options.validateVerified?.(verified); + } catch { + refuseHermesPortableInferenceRoute("changed during verification"); + } return verified; }, + prepareProbeDependency: () => { + preparedForwards = prepareHermesPortableForwardsForConnectProbeMeasured( + sandboxName, + authority, + options.probeTiming, + ); + return preparedForwards; + }, }); } catch (error) { + if (error instanceof HermesPortableForwardRecoveryError) { + failHermesPortableForwardRecovery(sandboxName, error.failure); + } if (error instanceof HermesPortableInferenceRouteVerificationError) { failHermesPortableInferenceRoute(sandboxName, error.reason); } @@ -779,8 +846,14 @@ function recoverHermesPortableInferenceRouteForProbeOnlyOrExit( classifyHermesPortableInferenceConnectRecoveryFailure(error), ); } - if (!verified) failHermesPortableInferenceRoute(sandboxName, "unreachable"); - return verified; + const retainedForwards = preparedForwards as PreparedHermesPortableForwardRecovery | null; + if (!verified || !retainedForwards) { + failHermesPortableInferenceRoute(sandboxName, "unreachable"); + } + options.probeTiming?.setForwardAction( + retainedForwards.result.kind === "restored" ? "restored" : "verified", + ); + return { forwardsRecovered: true }; } const GATEWAY_UNAVAILABLE_RE = @@ -2015,16 +2088,27 @@ async function prepareConnectSandboxWithinLifecycleFence( if (probeOnly !== true) { throw new Error("Hermes inference recovery requires probe-only mode"); } - const verified = probeTiming!.measure("inference", () => - recoverHermesPortableInferenceRouteForProbeOnlyOrExit( + const route = probeTiming!.measure("inference", () => + verifyOrRecoverHermesPortableInferenceRouteForProbeOnlyOrExit( sandboxName, acceptedReadiness.agent, activeAuthority, + { + probeTiming, + validateVerified: (verified) => + probeTiming!.measure("authority", () => + assertHermesPortableLifecycleForConnect(sandboxName, verified, gatewayName), + ), + }, ), ); - probeTiming!.measure("authority", () => - assertHermesPortableLifecycleForConnect(sandboxName, verified, gatewayName), - ); + if (!route.forwardsRecovered) { + recoverHermesPortableForwardsForConnectProbeOrExit( + sandboxName, + activeAuthority, + probeTiming, + ); + } } console.log(` Probe complete: launch readiness is healthy for '${sandboxName}'.`); return null; diff --git a/src/lib/actions/sandbox/forward-health.ts b/src/lib/actions/sandbox/forward-health.ts index 79d96a96b42..239287bc10c 100644 --- a/src/lib/actions/sandbox/forward-health.ts +++ b/src/lib/actions/sandbox/forward-health.ts @@ -12,11 +12,16 @@ export type SandboxForwardListEntry = { export type SandboxForwardHealth = boolean | "occupied" | null; +/** Whether OpenShell reports a forward as live in either supported CLI vocabulary. */ +export function isLiveSandboxForwardStatus(status: string): boolean { + return status === "running" || status === "active"; +} + function liveEntriesForPort( entries: SandboxForwardListEntry[], port: string, ): SandboxForwardListEntry[] { - return entries.filter((entry) => entry.port === port && entry.status === "running"); + return entries.filter((entry) => entry.port === port && isLiveSandboxForwardStatus(entry.status)); } export function classifySandboxForwardHealth( diff --git a/src/lib/actions/sandbox/forward-recovery.ts b/src/lib/actions/sandbox/forward-recovery.ts index 897ed40bbf6..baf018168ad 100644 --- a/src/lib/actions/sandbox/forward-recovery.ts +++ b/src/lib/actions/sandbox/forward-recovery.ts @@ -42,12 +42,14 @@ import { } from "./hermes-dashboard-recovery"; export { HermesPortableForwardRecoveryError, + prepareHermesPortableLaunchForwards, recoverHermesPortableLaunchForwards, } from "./probe/hermes-portable-forward-recovery"; export type { HermesPortableForwardRecoveryFailure, HermesPortableForwardRecoveryInput, HermesPortableForwardRecoveryResult, + PreparedHermesPortableForwardRecovery, } from "./probe/hermes-portable-forward-recovery"; type SandboxPortAgent = { diff --git a/src/lib/actions/sandbox/probe/hermes-portable-forward-recovery.test.ts b/src/lib/actions/sandbox/probe/hermes-portable-forward-recovery.test.ts index 339af97fbae..85721f35754 100644 --- a/src/lib/actions/sandbox/probe/hermes-portable-forward-recovery.test.ts +++ b/src/lib/actions/sandbox/probe/hermes-portable-forward-recovery.test.ts @@ -46,6 +46,31 @@ describe("Hermes Portable probe-only forward recovery", () => { expect(fixture.rollbackCalls).toEqual([]); }); + it("keeps exact active forwards verification-only", () => { + const fixture = createRecoveryFixture({ ports: [18_789, 8_642], active: [18_789, 8_642] }); + + expect(recoverHermesPortableLaunchForwards(fixture.input)).toEqual({ + kind: "verified", + restoredPorts: [], + }); + expect(fixture.currentCalls).toEqual([["forward", "list", "--gateway", "nemoclaw"]]); + expect(fixture.rollbackCalls).toEqual([]); + }); + + it.each([ + ["stopped", { stopped: [18_789] }], + ["dead after the host session ends", { dead: [18_789] }], + ])("restores an exact %s forward", (_state, options) => { + const fixture = createRecoveryFixture(options); + + expect(recoverHermesPortableLaunchForwards(fixture.input)).toEqual({ + kind: "restored", + restoredPorts: [18_789], + }); + expect(fixture.currentCalls.filter((args) => args[1] === "start")).toHaveLength(1); + expect(fixture.rollbackCalls).toEqual([]); + }); + it("accepts a returned nonzero start only after the exact owner settles healthy", () => { const fixture = createRecoveryFixture({ startStatus: 1 }); @@ -80,7 +105,8 @@ describe("Hermes Portable probe-only forward recovery", () => { }); it.each([ - ["PID", "alpha 127.0.0.1 18789 not-a-pid running"], + ["active PID", "alpha 127.0.0.1 18789 not-a-pid active"], + ["dead PID", "alpha 127.0.0.1 18789 not-a-pid dead"], ["bind", "alpha not-an-address 18789 12345 running"], ["port", "alpha 127.0.0.1 70000 12345 running"], ["status", "alpha 127.0.0.1 18789 12345 uncertain"], @@ -108,14 +134,14 @@ describe("Hermes Portable probe-only forward recovery", () => { }); it("rejects ambiguous duplicate rows before mutation", () => { - const fixture = createRecoveryFixture({ running: [18_789] }); + const fixture = createRecoveryFixture({ active: [18_789] }); Object.assign(fixture.input.deps, { captureCurrent: () => ({ status: 0, output: "SANDBOX BIND PORT PID STATUS\n" + - "alpha 127.0.0.1 18789 12345 running\n" + - "alpha 127.0.0.1 18789 12346 running", + "alpha 127.0.0.1 18789 12345 active\n" + + "alpha 127.0.0.1 18789 12346 active", }), }); @@ -125,6 +151,18 @@ describe("Hermes Portable probe-only forward recovery", () => { expect(fixture.rollbackCalls).toEqual([]); }); + it("rejects a foreign active owner before mutation", () => { + const fixture = createRecoveryFixture({ + listOutput: "SANDBOX BIND PORT PID STATUS\nbeta 127.0.0.1 18789 12345 active", + }); + + expect(() => recoverHermesPortableLaunchForwards(fixture.input)).toThrow( + expect.objectContaining({ failure: "forward-occupied" }), + ); + expect(fixture.currentCalls.some((args) => ["start", "stop"].includes(args[1]!))).toBe(false); + expect(fixture.rollbackCalls).toEqual([]); + }); + it("restores the exact missing state when authority drifts after start", () => { const fixture = createRecoveryFixture({ driftCurrentAfterStart: true }); @@ -179,6 +217,28 @@ describe("Hermes Portable probe-only forward recovery", () => { describe("Hermes Portable connect composition", () => { const originalStdoutIsTty = process.stdout.isTTY; + const acceptedHermesReadiness = () => { + const entry = { + name: "alpha", + agent: "hermes", + provider: "ollama-local", + model: "qwen3-vl:4b", + policies: [], + openshellDriver: "docker", + gatewayName: "nemoclaw", + lifecycleGeneration: "generation-1", + } as never; + return { + entry, + readinessDecision: { + kind: "accepted" as const, + category: "accepted" as const, + agent: { name: "hermes" }, + sb: entry, + }, + }; + }; + beforeEach(() => { process.env.NEMOCLAW_TEST_NO_SLEEP = "1"; Object.defineProperty(process.stdout, "isTTY", { configurable: true, value: true }); @@ -230,6 +290,228 @@ describe("Hermes Portable connect composition", () => { ); }); + it("restores an exact dead forward once before launch-readiness publication (#10423)", async () => { + const harness = createConnectHarness({ + agentName: "hermes", + sessionAgent: { name: "hermes" }, + portableReceiptDisposition: { kind: "hermes", phase: "active" }, + portableRecoveryResult: { kind: "already-running" }, + }); + configureMissingHermesForwardCapture(harness, { + initialStatus: "dead", + afterStart: () => { + expect(harness.publishLaunchReadinessSpy).not.toHaveBeenCalled(); + }, + }); + + await expect(harness.connectSandbox("alpha", { probeOnly: true })).resolves.toBeUndefined(); + + const mutations = harness.captureResolvedOpenshellSpy.mock.calls + .filter( + ([args]) => + Array.isArray(args) && + args[0] === "forward" && + ["start", "stop"].includes(String(args[1])), + ) + .map(([args]) => (args as string[])[1]); + expect(mutations).toEqual(["stop", "start"]); + expect(harness.publishLaunchReadinessSpy).toHaveBeenCalledOnce(); + expect(harness.logSpy.mock.calls.flat().join("\n")).toMatch( + /forwardAction=restored result=ready/, + ); + }); + + it.each([ + ["missing", ["stop", "start"]], + ["dead", ["stop", "start"]], + ] as const)( + "restores an accepted-readiness %s forward before reporting probe success", + async (initialStatus, expectedMutations) => { + const accepted = acceptedHermesReadiness(); + const harness = createConnectHarness({ + agentName: "hermes", + sessionAgent: { name: "hermes" }, + registryEntry: accepted.entry, + portableReceiptDisposition: { kind: "hermes", phase: "active" }, + portableRecoveryResult: { kind: "already-running" }, + readinessDecision: accepted.readinessDecision, + }); + configureMissingHermesForwardCapture(harness, { + initialStatus, + afterStart: () => { + expect(harness.logSpy.mock.calls.flat().join("\n")).not.toContain( + "Probe complete: launch readiness is healthy", + ); + }, + }); + + await expect(harness.connectSandbox("alpha", { probeOnly: true })).resolves.toBeUndefined(); + + const mutations = harness.captureResolvedOpenshellSpy.mock.calls + .filter( + ([args]) => + Array.isArray(args) && + args[0] === "forward" && + ["start", "stop"].includes(String(args[1])), + ) + .map(([args]) => (args as string[])[1]); + expect(mutations).toEqual(expectedMutations); + expect(harness.publishLaunchReadinessSpy).not.toHaveBeenCalled(); + expect(harness.logSpy.mock.calls.flat().join("\n")).toContain( + "Probe complete: launch readiness is healthy for 'alpha'.", + ); + }, + ); + + it("does not report accepted readiness when forward recovery fails", async () => { + const accepted = acceptedHermesReadiness(); + const harness = createConnectHarness({ + agentName: "hermes", + sessionAgent: { name: "hermes" }, + registryEntry: accepted.entry, + portableReceiptDisposition: { kind: "hermes", phase: "active" }, + portableRecoveryResult: { kind: "already-running" }, + readinessDecision: accepted.readinessDecision, + }); + const captureResolved = harness.captureResolvedOpenshellSpy.getMockImplementation()!; + harness.captureResolvedOpenshellSpy.mockImplementation(((args: unknown, options: unknown) => { + const argv = Array.isArray(args) ? args : []; + return argv[0] === "forward" && argv[1] === "list" + ? { status: 0, output: "malformed canary" } + : captureResolved(args, options); + }) as never); + + await expect(harness.connectSandbox("alpha", { probeOnly: true })).rejects.toThrow( + "process.exit(1)", + ); + + expect(harness.logSpy.mock.calls.flat().join("\n")).not.toContain( + "Probe complete: launch readiness is healthy", + ); + expect(harness.publishLaunchReadinessSpy).not.toHaveBeenCalled(); + expect( + harness.captureResolvedOpenshellSpy.mock.calls.some( + ([args]) => Array.isArray(args) && ["start", "stop"].includes(String(args[1])), + ), + ).toBe(false); + }); + + it("restores a recovered Ollama runtime when forward settlement fails", async () => { + const harness = createConnectHarness({ + agentName: "hermes", + sessionAgent: { name: "hermes" }, + portableReceiptDisposition: { kind: "hermes", phase: "active" }, + portableRecoveryResult: { kind: "already-running" }, + }); + let ollamaRunning = false; + harness.recoverHermesPortableOllamaInferenceSpy.mockImplementation(((input: { + verifyRoute: () => unknown; + prepareProbeDependency?: () => { release: () => void; rollback: () => void }; + }) => { + ollamaRunning = true; + try { + input.verifyRoute(); + input.prepareProbeDependency?.().release(); + return "recovered"; + } catch (error) { + ollamaRunning = false; + throw error; + } + }) as never); + let forwardStarted = false; + const forward = configureMissingHermesForwardCapture(harness, { + afterStart: () => { + forwardStarted = true; + }, + }); + const captureForward = harness.captureResolvedOpenshellSpy.getMockImplementation()!; + harness.captureResolvedOpenshellSpy.mockImplementation(((args: unknown, options: unknown) => { + const argv = Array.isArray(args) ? args : []; + return forwardStarted && forward.isRunning() && argv[0] === "forward" && argv[1] === "list" + ? { status: 0, output: "malformed canary" } + : captureForward(args, options); + }) as never); + + await expect(harness.connectSandbox("alpha", { probeOnly: true })).rejects.toThrow( + "process.exit(1)", + ); + + expect(forward.isRunning()).toBe(false); + expect(ollamaRunning).toBe(false); + expect(harness.publishLaunchReadinessSpy).not.toHaveBeenCalled(); + }); + + it("restores prepared forwards when Ollama finalization fails", async () => { + const harness = createConnectHarness({ + agentName: "hermes", + sessionAgent: { name: "hermes" }, + portableReceiptDisposition: { kind: "hermes", phase: "active" }, + portableRecoveryResult: { kind: "already-running" }, + }); + let ollamaRunning = false; + harness.recoverHermesPortableOllamaInferenceSpy.mockImplementation(((input: { + verifyRoute: () => unknown; + prepareProbeDependency?: () => { rollback: () => void }; + }) => { + ollamaRunning = true; + input.verifyRoute(); + const dependency = input.prepareProbeDependency?.(); + dependency?.rollback(); + ollamaRunning = false; + throw new Error("finalization canary"); + }) as never); + const forward = configureMissingHermesForwardCapture(harness); + + await expect(harness.connectSandbox("alpha", { probeOnly: true })).rejects.toThrow( + "process.exit(1)", + ); + + expect(forward.isRunning()).toBe(false); + expect(ollamaRunning).toBe(false); + expect(harness.publishLaunchReadinessSpy).not.toHaveBeenCalled(); + expect(harness.errorSpy.mock.calls.flat().join("\n")).not.toContain("finalization canary"); + }); + + it("reports forward restoration uncertainty after restoring Ollama", async () => { + const harness = createConnectHarness({ + agentName: "hermes", + sessionAgent: { name: "hermes" }, + portableReceiptDisposition: { kind: "hermes", phase: "active" }, + portableRecoveryResult: { kind: "already-running" }, + }); + let ollamaRunning = false; + harness.recoverHermesPortableOllamaInferenceSpy.mockImplementation(((input: { + verifyRoute: () => unknown; + prepareProbeDependency?: () => { rollback: () => void }; + }) => { + ollamaRunning = true; + input.verifyRoute(); + const dependency = input.prepareProbeDependency?.(); + harness.assertHermesPortableOperatingCommandCurrentSpy.mockImplementation(() => { + throw new Error("rollback authority canary"); + }); + try { + dependency?.rollback(); + } catch (error) { + ollamaRunning = false; + throw error; + } + throw new Error("expected rollback failure"); + }) as never); + const forward = configureMissingHermesForwardCapture(harness); + + await expect(harness.connectSandbox("alpha", { probeOnly: true })).rejects.toThrow( + "process.exit(1)", + ); + + expect(forward.isRunning()).toBe(true); + expect(ollamaRunning).toBe(false); + expect(harness.publishLaunchReadinessSpy).not.toHaveBeenCalled(); + const output = harness.errorSpy.mock.calls.flat().join("\n"); + expect(output).toContain("returned to a stopped state"); + expect(output).not.toContain("rollback authority canary"); + }); + it("stops before publication when the owning gateway forward list is malformed", async () => { const harness = createConnectHarness({ agentName: "hermes", diff --git a/src/lib/actions/sandbox/probe/hermes-portable-forward-recovery.ts b/src/lib/actions/sandbox/probe/hermes-portable-forward-recovery.ts index 45c7c87aabf..6b122d32560 100644 --- a/src/lib/actions/sandbox/probe/hermes-portable-forward-recovery.ts +++ b/src/lib/actions/sandbox/probe/hermes-portable-forward-recovery.ts @@ -4,7 +4,11 @@ import { isIP } from "node:net"; import { parseForwardList } from "../../../state/sandbox-session"; -import { classifySandboxForwardHealth, isLocalForwardReachable } from "../forward-health"; +import { + classifySandboxForwardHealth, + isLiveSandboxForwardStatus, + isLocalForwardReachable, +} from "../forward-health"; const FORWARD_SETTLEMENT_TIMEOUT_MS = 3_000; const FORWARD_SETTLEMENT_INTERVAL_MS = 100; @@ -57,6 +61,12 @@ export type HermesPortableForwardRecoveryResult = { readonly restoredPorts: readonly number[]; }; +export interface PreparedHermesPortableForwardRecovery { + readonly result: HermesPortableForwardRecoveryResult; + readonly release: () => HermesPortableForwardRecoveryResult; + readonly rollback: () => void; +} + function failure(failureClass: HermesPortableForwardRecoveryFailure): never { throw new HermesPortableForwardRecoveryError(failureClass); } @@ -92,7 +102,8 @@ function isSupportedForwardRow(parts: readonly string[]): boolean { /^\d+$/u.test(pidValue) && Number.isSafeInteger(pid) && pid > 0 && - ["running", "stopped"].includes(status.toLowerCase()) + (isLiveSandboxForwardStatus(status.toLowerCase()) || + ["dead", "stopped"].includes(status.toLowerCase())) ); } @@ -281,10 +292,31 @@ function rollbackTouchedPorts( for (const port of [...touchedPorts].reverse()) rollbackPort(input, port); } -/** Restore the exact launch-readiness forward set for one Hermes probe. */ -export function recoverHermesPortableLaunchForwards( +function retainForwardRecovery( input: HermesPortableForwardRecoveryInput, -): HermesPortableForwardRecoveryResult { + touchedPorts: readonly number[], + result: HermesPortableForwardRecoveryResult, +): PreparedHermesPortableForwardRecovery { + let state: "prepared" | "released" | "rolled-back" = "prepared"; + return Object.freeze({ + result, + release: () => { + if (state !== "prepared") failure("recovery-failed"); + state = "released"; + return result; + }, + rollback: () => { + if (state !== "prepared") failure("restoration-unproved"); + state = "rolled-back"; + if (touchedPorts.length > 0) rollbackTouchedPorts(input, touchedPorts); + }, + }); +} + +/** Prepare the exact launch-readiness forward set while retaining rollback authority. */ +export function prepareHermesPortableLaunchForwards( + input: HermesPortableForwardRecoveryInput, +): PreparedHermesPortableForwardRecovery { validatePorts(input); const touchedPorts: number[] = []; try { @@ -293,7 +325,7 @@ export function recoverHermesPortableLaunchForwards( const missing = input.ports.filter((port) => initial.get(port) !== "healthy"); if (missing.length === 0) { requireCurrent(input, false); - return { kind: "verified", restoredPorts: [] }; + return retainForwardRecovery(input, touchedPorts, { kind: "verified", restoredPorts: [] }); } const requiredHealthy = new Set(input.ports.filter((port) => initial.get(port) === "healthy")); @@ -326,7 +358,10 @@ export function recoverHermesPortableLaunchForwards( failure("recovery-failed"); } requireCurrent(input, false); - return { kind: "restored", restoredPorts: [...missing] }; + return retainForwardRecovery(input, touchedPorts, { + kind: "restored", + restoredPorts: [...missing], + }); } catch (error) { if (touchedPorts.length > 0) { try { @@ -339,6 +374,13 @@ export function recoverHermesPortableLaunchForwards( } } +/** Restore and commit the exact launch-readiness forward set for one Hermes probe. */ +export function recoverHermesPortableLaunchForwards( + input: HermesPortableForwardRecoveryInput, +): HermesPortableForwardRecoveryResult { + return prepareHermesPortableLaunchForwards(input).release(); +} + function sleepMilliseconds(milliseconds: number): void { if (milliseconds <= 0 || !Number.isFinite(milliseconds)) return; Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, milliseconds); diff --git a/src/lib/actions/sandbox/probe/hermes-portable-inference-recovery.ts b/src/lib/actions/sandbox/probe/hermes-portable-inference-recovery.ts index 63d2bcb3c13..d9cb67b4ccd 100644 --- a/src/lib/actions/sandbox/probe/hermes-portable-inference-recovery.ts +++ b/src/lib/actions/sandbox/probe/hermes-portable-inference-recovery.ts @@ -5,6 +5,7 @@ import { HermesPortableOllamaRecoveryError, HermesPortableOllamaRecoveryPhaseError, recoverHermesPortableOllamaInference, + type HermesPortableOllamaPreparedProbeDependency, type HermesPortableOllamaRecoveryFailure, type HermesPortableOllamaRecoveryPhase, } from "../../../onboard/experimental/hermes-portable-ollama-inference"; @@ -19,6 +20,7 @@ export interface HermesPortableInferenceConnectRecoveryInput { readonly authority: HermesPortableActiveLifecycleAuthority; readonly readRegistry: (sandboxName: string) => SandboxEntry | null; readonly verifyRoute: () => SandboxEntry; + readonly prepareProbeDependency?: () => HermesPortableOllamaPreparedProbeDependency; } export type HermesPortableInferenceConnectRecoveryFailure = @@ -47,5 +49,6 @@ export function recoverHermesPortableInferenceForConnectProbe( captureHermesPortableInferenceRecoveryGateway(input.sandboxName, args, options), readRegistry: input.readRegistry, verifyRoute: input.verifyRoute, + prepareProbeDependency: input.prepareProbeDependency, }); } diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts index 4b8b8f9b27b..ad054bad444 100644 --- a/src/lib/actions/sandbox/process-recovery.ts +++ b/src/lib/actions/sandbox/process-recovery.ts @@ -39,6 +39,7 @@ import { ensureSandboxPortForward, HermesPortableForwardRecoveryError, isSandboxForwardHealthy, + prepareHermesPortableLaunchForwards, recoverDeclaredAgentForwardPorts, recoverHermesPortableLaunchForwards, recoverMessagingHostForward, @@ -48,6 +49,7 @@ import { type HermesPortableForwardRecoveryFailure, type HermesPortableForwardRecoveryInput, type HermesPortableForwardRecoveryResult, + type PreparedHermesPortableForwardRecovery, } from "./forward-recovery"; import { classifyGatewayRestartFailure, @@ -85,11 +87,16 @@ export { classifySandboxForwardHealth, } from "./forward-health"; export { resolveSandboxDashboardPort, resolveSandboxLaunchForwardPorts } from "./forward-recovery"; -export { HermesPortableForwardRecoveryError, recoverHermesPortableLaunchForwards }; +export { + HermesPortableForwardRecoveryError, + prepareHermesPortableLaunchForwards, + recoverHermesPortableLaunchForwards, +}; export type { HermesPortableForwardRecoveryFailure, HermesPortableForwardRecoveryInput, HermesPortableForwardRecoveryResult, + PreparedHermesPortableForwardRecovery, }; export type { GatewayRestartDeps, diff --git a/src/lib/onboard/experimental/hermes-portable-ollama-inference.ts b/src/lib/onboard/experimental/hermes-portable-ollama-inference.ts index 51f2b69db86..0d7bab89106 100644 --- a/src/lib/onboard/experimental/hermes-portable-ollama-inference.ts +++ b/src/lib/onboard/experimental/hermes-portable-ollama-inference.ts @@ -352,6 +352,12 @@ export interface HermesPortableOllamaRecoveryInput { readonly runGatewayOpenshell: HermesPortableOllamaGatewayRunner; readonly readRegistry: (sandboxName: string) => SandboxEntry | null; readonly verifyRoute: () => SandboxEntry; + readonly prepareProbeDependency?: () => HermesPortableOllamaPreparedProbeDependency; +} + +export interface HermesPortableOllamaPreparedProbeDependency { + readonly release: () => void; + readonly rollback: () => void; } interface HermesPortableOllamaRecoveryDeps { @@ -639,20 +645,35 @@ export function recoverHermesPortableOllamaInference( return current; }); if (inspected.running) { - requireExactRecoveryReceipt( - serializedRegistryReceipt, - runtime.preserveForRebuild(receipt), - "running runtime validation changed receipt", - ); - requireCurrent(); - verifyFinalRoute(); - requireCurrent(); - registryRecovery.release(); - return "reused"; + let preparedDependency: HermesPortableOllamaPreparedProbeDependency | null = null; + try { + requireExactRecoveryReceipt( + serializedRegistryReceipt, + runtime.preserveForRebuild(receipt), + "running runtime validation changed receipt", + ); + requireCurrent(); + verifyFinalRoute(); + preparedDependency = input.prepareProbeDependency?.() ?? null; + requireCurrent(); + registryRecovery.release(); + preparedDependency?.release(); + return "reused"; + } catch (error) { + if (preparedDependency) { + try { + preparedDependency.rollback(); + } catch (rollbackError) { + throw rollbackError; + } + } + throw error; + } } let prepared: HostLocalInferencePreparedStartup; ollamaStateRestored = false; + let preparedDependency: HermesPortableOllamaPreparedProbeDependency | null = null; try { requireCurrent(); prepared = deps.prepareStartup( @@ -695,6 +716,7 @@ export function recoverHermesPortableOllamaInference( ); requireCurrent(); verifyFinalRoute(); + preparedDependency = input.prepareProbeDependency?.() ?? null; const finalizePublishedResume = prepared.finalizePublishedResume; if (!finalizePublishedResume) { failRecovery("runtime provider lacks rollback-safe published resume finalization"); @@ -706,8 +728,17 @@ export function recoverHermesPortableOllamaInference( ); ollamaStateRestored = true; registryRecovery.release(); + preparedDependency?.release(); return "recovered"; } catch (error) { + let dependencyRollbackError: unknown = null; + if (preparedDependency) { + try { + preparedDependency.rollback(); + } catch (rollbackError) { + dependencyRollbackError = rollbackError; + } + } try { restoreStoppedRuntime(prepared, serializedRegistryReceipt); ollamaStateRestored = true; @@ -717,6 +748,7 @@ export function recoverHermesPortableOllamaInference( "runtime-restoration-unproved", ); } + if (dependencyRollbackError) throw dependencyRollbackError; throw error; } } catch (error) { diff --git a/src/lib/onboard/experimental/hermes-portable-ollama-recovery.test.ts b/src/lib/onboard/experimental/hermes-portable-ollama-recovery.test.ts index 28d87ba529d..e18242167d3 100644 --- a/src/lib/onboard/experimental/hermes-portable-ollama-recovery.test.ts +++ b/src/lib/onboard/experimental/hermes-portable-ollama-recovery.test.ts @@ -228,6 +228,32 @@ function createHarness(initiallyRunning = false, registryInitiallyRunning = fals } describe("Hermes Portable Ollama inference recovery", () => { + it.each([ + ["missing", undefined, "sandbox registry host-local inference receipt is missing"], + ["malformed", "not-json\n", "serialized receipt is not valid JSON"], + ] as const)( + "rejects an ollama-local registry receipt that is %s before registry recovery", + (_label, serializedReceipt, expectedError) => { + const harness = createHarness(); + const entry = { + ...harness.input.entry, + hostLocalInferenceReceipt: serializedReceipt, + } as SandboxEntry; + + expect(() => + recoverHermesPortableOllamaInference( + { + ...harness.input, + entry, + readRegistry: vi.fn(() => entry), + }, + harness.overrides as never, + ), + ).toThrow(expectedError); + expect(harness.overrides.prepareRegistryRecovery).not.toHaveBeenCalled(); + }, + ); + it("resumes one stopped published runtime and commits only after final route proof", () => { const harness = createHarness(); @@ -250,6 +276,124 @@ describe("Hermes Portable Ollama inference recovery", () => { expect(harness.events.at(-1)).toBe("registry-release"); }); + it("releases a prepared probe dependency only after stopped-runtime finalization", () => { + const harness = createHarness(); + const dependency = { + release: vi.fn(() => harness.events.push("dependency-release")), + rollback: vi.fn(() => harness.events.push("dependency-rollback")), + }; + + expect( + recoverHermesPortableOllamaInference( + { + ...harness.input, + prepareProbeDependency: vi.fn(() => { + harness.events.push("dependency-prepare"); + return dependency; + }), + }, + harness.overrides as never, + ), + ).toBe("recovered"); + + expect(harness.events.indexOf("route")).toBeLessThan( + harness.events.indexOf("dependency-prepare"), + ); + expect(harness.events.indexOf("dependency-prepare")).toBeLessThan( + harness.events.indexOf("finalize"), + ); + expect(harness.events.indexOf("finalize")).toBeLessThan( + harness.events.indexOf("dependency-release"), + ); + expect(harness.events.indexOf("registry-release")).toBeLessThan( + harness.events.indexOf("dependency-release"), + ); + expect(dependency.rollback).not.toHaveBeenCalled(); + }); + + it("restores the stopped runtime when probe-dependency preparation fails", () => { + const harness = createHarness(); + const canary = new Error("forward preparation failed"); + + expect(() => + recoverHermesPortableOllamaInference( + { + ...harness.input, + prepareProbeDependency: vi.fn(() => { + throw canary; + }), + }, + harness.overrides as never, + ), + ).toThrow(canary); + + expect(harness.prepared.rollback).toHaveBeenCalledOnce(); + expect(harness.running()).toBe(false); + expect(harness.registryRunning()).toBe(false); + expect(harness.events.indexOf("rollback")).toBeLessThan( + harness.events.indexOf("registry-rollback"), + ); + }); + + it("rolls back a prepared probe dependency before the stopped runtime", () => { + const harness = createHarness(); + const dependency = { + release: vi.fn(() => harness.events.push("dependency-release")), + rollback: vi.fn(() => harness.events.push("dependency-rollback")), + }; + vi.mocked(harness.prepared.finalizePublishedResume!).mockImplementation(() => { + harness.events.push("finalize"); + throw new Error("finalization failed"); + }); + + expect(() => + recoverHermesPortableOllamaInference( + { ...harness.input, prepareProbeDependency: vi.fn(() => dependency) }, + harness.overrides as never, + ), + ).toThrow("finalization failed"); + + expect(dependency.release).not.toHaveBeenCalled(); + expect(harness.events.indexOf("dependency-rollback")).toBeLessThan( + harness.events.indexOf("rollback"), + ); + expect(harness.events.indexOf("rollback")).toBeLessThan( + harness.events.indexOf("registry-rollback"), + ); + expect(harness.running()).toBe(false); + expect(harness.registryRunning()).toBe(false); + }); + + it("preserves probe-dependency restoration uncertainty after restoring Ollama", () => { + const harness = createHarness(); + const restorationError = new Error("forward restoration unproved"); + const dependency = { + release: vi.fn(), + rollback: vi.fn(() => { + harness.events.push("dependency-rollback"); + throw restorationError; + }), + }; + vi.mocked(harness.prepared.finalizePublishedResume!).mockImplementation(() => { + harness.events.push("finalize"); + throw new Error("lower finalization canary"); + }); + + expect(() => + recoverHermesPortableOllamaInference( + { ...harness.input, prepareProbeDependency: vi.fn(() => dependency) }, + harness.overrides as never, + ), + ).toThrow(restorationError); + + expect(harness.prepared.rollback).toHaveBeenCalledOnce(); + expect(harness.running()).toBe(false); + expect(harness.registryRunning()).toBe(false); + expect(harness.events.indexOf("dependency-rollback")).toBeLessThan( + harness.events.indexOf("rollback"), + ); + }); + it("validates an already running runtime without invoking resume", () => { const harness = createHarness(true, true); @@ -264,6 +408,37 @@ describe("Hermes Portable Ollama inference recovery", () => { expect(harness.events.at(-1)).toBe("registry-release"); }); + it("does not invent runtime rollback for an already-running Ollama dependency failure", () => { + const harness = createHarness(true, true); + const dependency = { + release: vi.fn(), + rollback: vi.fn(() => { + harness.events.push("dependency-rollback"); + }), + }; + harness.overrides.prepareRegistryRecovery.mockReturnValue({ + started: false, + assertCurrent: vi.fn(), + rollback: vi.fn(() => { + harness.events.push("registry-rollback"); + }), + release: vi.fn(() => { + throw new Error("registry finalization failed"); + }), + }); + + expect(() => + recoverHermesPortableOllamaInference( + { ...harness.input, prepareProbeDependency: vi.fn(() => dependency) }, + harness.overrides as never, + ), + ).toThrow("registry finalization failed"); + + expect(dependency.rollback).toHaveBeenCalledOnce(); + expect(harness.prepared.rollback).not.toHaveBeenCalled(); + expect(harness.running()).toBe(true); + }); + it("reconciles a stopped registry before validating an already running runtime", () => { const harness = createHarness(true); diff --git a/test/process-recovery/process-recovery-primitives.test.ts b/test/process-recovery/process-recovery-primitives.test.ts index 96e016d0907..ef4b3f2fcbd 100644 --- a/test/process-recovery/process-recovery-primitives.test.ts +++ b/test/process-recovery/process-recovery-primitives.test.ts @@ -473,20 +473,33 @@ describe("resolveSandboxDashboardPort", () => { }); describe("classifySandboxForwardHealth", () => { - it("returns true for a running forward owned by the target sandbox", () => { + it.each(["running", "active"])( + "returns true for a %s forward owned by the target sandbox", + (status) => { + expect( + classifySandboxForwardHealth( + [{ sandboxName: "beta", port: "18790", status }], + "beta", + "18790", + ), + ).toBe(true); + }, + ); + + it("returns occupied when another sandbox owns the expected port", () => { expect( classifySandboxForwardHealth( - [{ sandboxName: "beta", port: "18790", status: "running" }], + [{ sandboxName: "alpha", port: "18790", status: "running" }], "beta", "18790", ), - ).toBe(true); + ).toBe("occupied"); }); - it("returns occupied when another sandbox owns the expected port", () => { + it("returns occupied when another sandbox owns an active forward on the expected port", () => { expect( classifySandboxForwardHealth( - [{ sandboxName: "alpha", port: "18790", status: "running" }], + [{ sandboxName: "alpha", port: "18790", status: "active" }], "beta", "18790", ), @@ -576,6 +589,22 @@ describe("classifySandboxForwardHealth", () => { }); describe("classifyForwardHealthWithReachability", () => { + it("requires an exact active owner to answer before reporting healthy", () => { + let probed = false; + const result = classifyForwardHealthWithReachability( + [{ sandboxName: "beta", port: "18790", status: "active" }], + "beta", + "18790", + () => { + probed = true; + return true; + }, + ); + + expect(result).toBe(true); + expect(probed).toBe(true); + }); + it("does not trust an arbitrary local listener for a non-running owned entry", () => { let probed = false; const result = classifyForwardHealthWithReachability( diff --git a/test/support/connect-flow-test-harness.ts b/test/support/connect-flow-test-harness.ts index 5d538089f42..9ffd5055cf8 100644 --- a/test/support/connect-flow-test-harness.ts +++ b/test/support/connect-flow-test-harness.ts @@ -299,7 +299,10 @@ export function createConnectHarness(options: ConnectHarnessOptions = {}): Conne )) as never); const recoverHermesPortableOllamaInferenceSpy = vi .spyOn(hermesInferenceRecovery, "recoverHermesPortableInferenceForConnectProbe") - .mockImplementation(((input: { verifyRoute: () => unknown }) => { + .mockImplementation(((input: { + verifyRoute: () => unknown; + prepareProbeDependency?: () => { release: () => void }; + }) => { if (options.hermesInferenceRecoveryPhase) { throw new hermesOllamaInference.HermesPortableOllamaRecoveryPhaseError( options.hermesInferenceRecoveryPhase, @@ -315,6 +318,7 @@ export function createConnectHarness(options: ConnectHarnessOptions = {}): Conne ); } input.verifyRoute(); + input.prepareProbeDependency?.().release(); return "reused"; }) as never); const requalifyPortableAgentAuthoritySpy = vi diff --git a/test/support/hermes-portable-forward-recovery-fixture.ts b/test/support/hermes-portable-forward-recovery-fixture.ts index 64f30a716b2..5453e9931fe 100644 --- a/test/support/hermes-portable-forward-recovery-fixture.ts +++ b/test/support/hermes-portable-forward-recovery-fixture.ts @@ -7,7 +7,7 @@ import type { ConnectHarness } from "./connect-flow-test-harness"; type ForwardRecord = { owner: string; reachable: boolean; - status: "dead" | "running"; + status: "active" | "dead" | "running" | "stopped"; }; function forwardList(records: ReadonlyMap): string { @@ -21,7 +21,10 @@ function forwardList(records: ReadonlyMap): string { export function createHermesPortableForwardRecoveryFixture({ ports = [18_789], + active = [], + dead = [], running = [], + stopped = [], occupied = [], malformedList = false, listStatus = 0, @@ -32,7 +35,10 @@ export function createHermesPortableForwardRecoveryFixture({ listOutput, }: { ports?: readonly number[]; + active?: readonly number[]; + dead?: readonly number[]; running?: readonly number[]; + stopped?: readonly number[]; occupied?: readonly number[]; malformedList?: boolean; listStatus?: number; @@ -43,9 +49,18 @@ export function createHermesPortableForwardRecoveryFixture({ listOutput?: string; } = {}) { const records = new Map(); + for (const port of active) { + records.set(port, { owner: "alpha", reachable: true, status: "active" }); + } + for (const port of dead) { + records.set(port, { owner: "alpha", reachable: false, status: "dead" }); + } for (const port of running) { records.set(port, { owner: "alpha", reachable: true, status: "running" }); } + for (const port of stopped) { + records.set(port, { owner: "alpha", reachable: false, status: "stopped" }); + } for (const port of occupied) { records.set(port, { owner: "beta", reachable: true, status: "running" }); } @@ -119,12 +134,15 @@ export function createHermesPortableForwardRecoveryFixture({ export function configureMissingHermesForwardCapture( harness: ConnectHarness, - options: { readonly afterStart?: () => void } = {}, + options: { + readonly afterStart?: () => void; + readonly initialStatus?: "dead" | "missing"; + } = {}, ): { readonly isRunning: () => boolean } { - let forwardRunning = false; + let forwardStatus: "dead" | "missing" | "running" = options.initialStatus ?? "missing"; const captureResolved = harness.captureResolvedOpenshellSpy.getMockImplementation()!; harness.spawnSyncSpy.mockImplementation(((command: unknown) => ({ - status: String(command) === process.execPath && !forwardRunning ? 1 : 0, + status: String(command) === process.execPath && forwardStatus !== "running" ? 1 : 0, signal: null, })) as never); harness.captureResolvedOpenshellSpy.mockImplementation((( @@ -135,21 +153,22 @@ export function configureMissingHermesForwardCapture( if (argv[0] === "forward" && argv[1] === "list") { return { status: 0, - output: forwardRunning - ? "SANDBOX BIND PORT PID STATUS\nalpha 127.0.0.1 18789 12345 running" - : "SANDBOX BIND PORT PID STATUS", + output: + forwardStatus === "missing" + ? "SANDBOX BIND PORT PID STATUS" + : `SANDBOX BIND PORT PID STATUS\nalpha 127.0.0.1 18789 12345 ${forwardStatus}`, }; } if (argv[0] === "forward" && argv[1] === "stop") { - forwardRunning = false; + forwardStatus = "missing"; return { status: 0, output: "" }; } if (argv[0] === "forward" && argv[1] === "start") { - forwardRunning = true; + forwardStatus = "running"; options.afterStart?.(); return { status: 0, output: "" }; } return captureResolved(args, captureOptions); }) as never); - return { isRunning: () => forwardRunning }; + return { isRunning: () => forwardStatus === "running" }; }