diff --git a/docs/inference/set-up-model-router.mdx b/docs/inference/set-up-model-router.mdx index 85a4b34d64a..aa7694f33c9 100644 --- a/docs/inference/set-up-model-router.mdx +++ b/docs/inference/set-up-model-router.mdx @@ -32,6 +32,9 @@ When the log contains readable output from this run, the error includes a redact The owner-only log contains unredacted router output. Review the log before you share it. NemoClaw does not provide log-only cleanup. +When `destroy` removes the last registered Model Router sandbox that uses a host port, it also stops the Model Router process and frees that port. +If the stop fails, destroy still completes and prints a warning with the manual stop command. +While another registered Model Router sandbox uses the same host port, destroying one Model Router sandbox keeps that Model Router process running. A successful [uninstall](../../manage-sandboxes/operate-sandboxes/uninstall-nemoclaw) stops the selected Model Router and removes its log with the selected gateway's operational state. Review the uninstall scope before you use it for log removal. diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 8df4f095834..e729b9d76d6 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -2260,6 +2260,7 @@ Model traffic uses the OpenShell-managed `inference.local` route configured by N Stop managed local inference resources, remove the host-side Docker image built during onboard, and delete the sandbox. This removes the sandbox from the registry. For Ollama-backed sandboxes, `destroy` also asks Ollama to unload currently loaded models and clears stale auth proxy state on a best-effort basis. +When `destroy` removes the last registered Model Router sandbox that uses a host port, it also stops the host Model Router process on a best-effort basis and frees that port. This command attempts to wipe the manifest-defined agent state while its persistent volume is mounted, then removes the sandbox. diff --git a/src/lib/actions/sandbox/destroy-flow.test.ts b/src/lib/actions/sandbox/destroy-flow.test.ts index 40eac504650..899d2ae9841 100644 --- a/src/lib/actions/sandbox/destroy-flow.test.ts +++ b/src/lib/actions/sandbox/destroy-flow.test.ts @@ -78,6 +78,41 @@ describe("destroySandbox flow", () => { ); }); + it("stops the routed sandbox proxy after registry removal under the gateway route lock (#9098)", async () => { + const harness = createDestroyHarness({ + provider: "nvidia-router", + endpointUrl: "http://host.openshell.internal:4000/v1", + }); + + await expect(harness.destroySandbox("alpha", { yes: true })).resolves.toBeUndefined(); + + expect(harness.stopModelRouterForDestroyedSandboxSpy).toHaveBeenCalledOnce(); + expect(harness.withGatewayRouteMutationLockSpy).toHaveBeenCalledWith( + "nemoclaw-19080", + expect.any(Function), + ); + expect(harness.removeSandboxSpy.mock.invocationCallOrder[0]).toBeLessThan( + harness.withGatewayRouteMutationLockSpy.mock.invocationCallOrder[0], + ); + expect(harness.withGatewayRouteMutationLockSpy.mock.invocationCallOrder[0]).toBeLessThan( + harness.stopModelRouterForDestroyedSandboxSpy.mock.invocationCallOrder[0], + ); + }); + + it("does not stop the routed sandbox proxy when registry removal does not complete (#9098)", async () => { + const harness = createDestroyHarness({ + provider: "nvidia-router", + endpointUrl: "http://host.openshell.internal:4000/v1", + removeSandboxResult: false, + }); + + await expect(harness.destroySandbox("alpha", { yes: true })).resolves.toBeUndefined(); + + expect(harness.removeSandboxSpy).toHaveBeenCalledWith("alpha"); + expect(harness.withGatewayRouteMutationLockSpy).not.toHaveBeenCalled(); + expect(harness.stopModelRouterForDestroyedSandboxSpy).not.toHaveBeenCalled(); + }); + it.each([ ["--yes", "darwin", { yes: true }, "", true], ["NEMOCLAW_NON_INTERACTIVE=1", "darwin", {}, "1", true], @@ -115,6 +150,8 @@ describe("destroySandbox flow", () => { expectFailedDeletePreservesHostState(harness, exitSpy); expect(harness.retirePortableLifecycleReceiptSpy).not.toHaveBeenCalled(); + expect(harness.withGatewayRouteMutationLockSpy).not.toHaveBeenCalled(); + expect(harness.stopModelRouterForDestroyedSandboxSpy).not.toHaveBeenCalled(); }); it("refuses before destructive work when Docker identity cannot be inspected", async () => { diff --git a/src/lib/actions/sandbox/destroy-model-router.test.ts b/src/lib/actions/sandbox/destroy-model-router.test.ts new file mode 100644 index 00000000000..a7bea07fb8b --- /dev/null +++ b/src/lib/actions/sandbox/destroy-model-router.test.ts @@ -0,0 +1,239 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import type { Session } from "../../state/onboard-session"; +import type { SandboxEntry } from "../../state/registry"; +import { + resolveDestroyedSandboxRouterPort, + stopModelRouterForDestroyedSandbox, + type StopModelRouterForDestroyedSandboxDeps, +} from "./destroy-preflight"; + +const routedSandbox = { + name: "alpha", + provider: "nvidia-router", + endpointUrl: "http://host.openshell.internal:4100/v1", +} as SandboxEntry; + +function createDeps(overrides: Partial = {}) { + const session = { + sessionId: "session-alpha", + sandboxName: "alpha", + endpointUrl: "http://host.openshell.internal:4100/v1", + routerPid: 4242, + routerCredentialHash: "hash", + } as Session; + const deps = { + findPidForPort: vi.fn(() => null), + isRoutedProvider: vi.fn((provider: string | null | undefined) => provider === "nvidia-router"), + listSandboxes: vi.fn(() => ({ sandboxes: [] as SandboxEntry[], defaultSandbox: null })), + loadSession: vi.fn(() => session), + log: vi.fn(), + ownsPort: vi.fn(() => true), + stopProcess: vi.fn(async () => undefined), + updateSession: vi.fn((mutator: (current: Session) => Session | void) => { + mutator(session); + return session; + }), + warn: vi.fn(), + ...overrides, + }; + return { deps, session }; +} + +describe("resolveDestroyedSandboxRouterPort", () => { + it("parses the router port from the sandbox endpoint URL", () => { + expect(resolveDestroyedSandboxRouterPort("http://host.openshell.internal:4100/v1")).toBe(4100); + }); + + it("falls back to port 4000 for a missing or unparseable endpoint", () => { + expect(resolveDestroyedSandboxRouterPort(null)).toBe(4000); + expect(resolveDestroyedSandboxRouterPort("not a url")).toBe(4000); + expect(resolveDestroyedSandboxRouterPort("http://host.openshell.internal/v1")).toBe(4000); + }); +}); + +describe("stopModelRouterForDestroyedSandbox", () => { + it("stops the tracked router and clears its session identity for the last routed sandbox (#9098)", async () => { + const { deps, session } = createDeps(); + + await stopModelRouterForDestroyedSandbox(routedSandbox, deps); + + expect(deps.stopProcess).toHaveBeenCalledWith(4242, 4100); + expect(session.routerPid).toBeNull(); + expect(session.routerCredentialHash).toBeNull(); + expect(deps.warn).not.toHaveBeenCalled(); + }); + + it("does nothing for a sandbox without a routed provider", async () => { + const { deps } = createDeps(); + + await stopModelRouterForDestroyedSandbox( + { name: "alpha", provider: "ollama-local" } as SandboxEntry, + deps, + ); + + expect(deps.stopProcess).not.toHaveBeenCalled(); + expect(deps.updateSession).not.toHaveBeenCalled(); + }); + + it("does nothing when the registry entry is missing", async () => { + const { deps } = createDeps(); + + await stopModelRouterForDestroyedSandbox(null, deps); + + expect(deps.stopProcess).not.toHaveBeenCalled(); + expect(deps.updateSession).not.toHaveBeenCalled(); + }); + + it("keeps the router while another registered routed sandbox remains", async () => { + const { deps } = createDeps({ + listSandboxes: vi.fn(() => ({ + sandboxes: [ + { + name: "beta", + provider: "nvidia-router", + endpointUrl: "http://host.openshell.internal:4100/v1", + } as SandboxEntry, + ], + defaultSandbox: null, + })), + }); + + await stopModelRouterForDestroyedSandbox(routedSandbox, deps); + + expect(deps.stopProcess).not.toHaveBeenCalled(); + expect(deps.updateSession).not.toHaveBeenCalled(); + }); + + it("stops the target router when a routed peer uses a different port", async () => { + const { deps } = createDeps({ + listSandboxes: vi.fn(() => ({ + sandboxes: [ + { + name: "beta", + provider: "nvidia-router", + endpointUrl: "http://host.openshell.internal:4200/v1", + } as SandboxEntry, + ], + defaultSandbox: null, + })), + }); + + await stopModelRouterForDestroyedSandbox(routedSandbox, deps); + + expect(deps.stopProcess).toHaveBeenCalledWith(4242, 4100); + }); + + it("recovers an orphaned router by port scan when the recorded PID does not own the port", async () => { + const { deps, session } = createDeps({ + ownsPort: vi.fn(() => false), + findPidForPort: vi.fn(() => 5151), + }); + + await stopModelRouterForDestroyedSandbox(routedSandbox, deps); + + expect(deps.findPidForPort).toHaveBeenCalledWith(4100); + expect(deps.stopProcess).toHaveBeenCalledWith(5151, 4100); + expect(session.routerPid).toBeNull(); + }); + + it("clears a stale recorded PID when no router process is found", async () => { + const { deps, session } = createDeps({ + ownsPort: vi.fn(() => false), + findPidForPort: vi.fn(() => null), + }); + + await stopModelRouterForDestroyedSandbox(routedSandbox, deps); + + expect(deps.stopProcess).not.toHaveBeenCalled(); + expect(session.routerPid).toBeNull(); + expect(session.routerCredentialHash).toBeNull(); + }); + + it("clears a stale credential hash when the session records no router PID (#9098)", async () => { + const session = { + sessionId: "session-alpha", + sandboxName: "alpha", + endpointUrl: "http://host.openshell.internal:4100/v1", + routerPid: null, + routerCredentialHash: "stale", + } as Session; + const { deps } = createDeps({ + loadSession: vi.fn(() => session), + ownsPort: vi.fn(() => false), + updateSession: vi.fn((mutator: (current: Session) => Session | void) => { + mutator(session); + return session; + }), + }); + + await stopModelRouterForDestroyedSandbox(routedSandbox, deps); + + expect(deps.stopProcess).not.toHaveBeenCalled(); + expect(session.routerPid).toBeNull(); + expect(session.routerCredentialHash).toBeNull(); + }); + + it("does not clear session identity for a different routed sandbox", async () => { + const session = { + sessionId: "session-beta", + sandboxName: "beta", + endpointUrl: "http://host.openshell.internal:4200/v1", + routerPid: 5252, + routerCredentialHash: "beta-hash", + } as Session; + const { deps } = createDeps({ + loadSession: vi.fn(() => session), + ownsPort: vi.fn(() => false), + findPidForPort: vi.fn(() => 5151), + updateSession: vi.fn((mutator: (current: Session) => Session | void) => { + mutator(session); + return session; + }), + }); + + await stopModelRouterForDestroyedSandbox(routedSandbox, deps); + + expect(deps.stopProcess).toHaveBeenCalledWith(5151, 4100); + expect(deps.updateSession).not.toHaveBeenCalled(); + expect(session).toMatchObject({ routerPid: 5252, routerCredentialHash: "beta-hash" }); + }); + + it("leaves the session untouched when it records no router PID and no orphan exists", async () => { + const { deps } = createDeps({ + loadSession: vi.fn( + () => + ({ + sessionId: "session-beta", + sandboxName: "beta", + endpointUrl: "http://host.openshell.internal:4200/v1", + routerPid: null, + }) as Session, + ), + ownsPort: vi.fn(() => false), + }); + + await stopModelRouterForDestroyedSandbox(routedSandbox, deps); + + expect(deps.stopProcess).not.toHaveBeenCalled(); + expect(deps.updateSession).not.toHaveBeenCalled(); + }); + + it("warns and keeps the recorded PID when the stop fails, so uninstall can still find it", async () => { + const { deps, session } = createDeps({ + stopProcess: vi.fn(async () => { + throw new Error("shutdown did not converge"); + }), + }); + + await expect(stopModelRouterForDestroyedSandbox(routedSandbox, deps)).resolves.toBeUndefined(); + + expect(deps.warn).toHaveBeenCalledWith(expect.stringContaining("shutdown did not converge")); + expect(deps.warn).toHaveBeenCalledWith(expect.stringContaining("kill 4242")); + expect(deps.updateSession).not.toHaveBeenCalled(); + expect(session.routerPid).toBe(4242); + }); +}); diff --git a/src/lib/actions/sandbox/destroy-preflight.ts b/src/lib/actions/sandbox/destroy-preflight.ts index dc2b688d726..a27b64c62b1 100644 --- a/src/lib/actions/sandbox/destroy-preflight.ts +++ b/src/lib/actions/sandbox/destroy-preflight.ts @@ -2,6 +2,13 @@ // SPDX-License-Identifier: Apache-2.0 import { OPENSHELL_PROBE_TIMEOUT_MS } from "../../adapters/openshell/timeouts"; +import { isRoutedInferenceProvider } from "../../onboard/model-router"; +import { + doesModelRouterProcessOwnPort, + findModelRouterPidForPort, + stopModelRouterProcess, +} from "../../onboard/model-router-process"; +import type { Session } from "../../state/onboard-session"; import type { SandboxEntry } from "../../state/registry"; import * as registry from "../../state/registry"; import { type DestroyRunOpenshell, selectGatewayForSandboxDestroy } from "./destroy-gateway"; @@ -42,6 +49,116 @@ export function stopSandboxInferenceResources( } } +// Routed onboard profiles use blueprint port 4000 by default; matches the +// uninstall teardown default in src/lib/actions/uninstall/run-plan.ts. +const DEFAULT_MODEL_ROUTER_PORT = 4000; + +export type StopModelRouterForDestroyedSandboxDeps = { + loadSession: () => Session | null; + updateSession: (mutator: (session: Session) => Session | void) => Session; + findPidForPort?: typeof findModelRouterPidForPort; + isRoutedProvider?: typeof isRoutedInferenceProvider; + listSandboxes?: typeof registry.listSandboxes; + log?: (message: string) => void; + ownsPort?: typeof doesModelRouterProcessOwnPort; + stopProcess?: (pid: number, port: number) => Promise; + warn?: (message: string) => void; +}; + +export function resolveDestroyedSandboxRouterPort(endpointUrl: string | null | undefined): number { + try { + const port = Number(new URL(endpointUrl ?? "").port); + return Number.isInteger(port) && port > 0 ? port : DEFAULT_MODEL_ROUTER_PORT; + } catch { + return DEFAULT_MODEL_ROUTER_PORT; + } +} + +/** + * Stop the host Model Router proxy after the last routed sandbox is destroyed. + * + * The router is a detached host process whose PID is recorded only in the + * onboarding session (routerPid). Destroy never stopped it, so the orphan kept + * its port and the next routed onboard failed with "Port 4000 already has a + * healthy router endpoint" (#9098). This mirrors the uninstall teardown + * (#5169) but stays scoped: it acts only when the destroyed sandbox was routed + * and no registered routed sandbox remains, so a routed peer keeps its router. + * + * The recorded PID is preferred; when a fresh session no longer records it, + * the /proc scan recovers the orphan by verified command line, exactly like + * reconcileModelRouter's recovery path. A stop failure is a warning, not an + * error: the sandbox delete already succeeded, and a stuck session-global host + * proxy must not fail the destroy. The session keeps routerPid on failure so + * uninstall and reconcile can still find the process. + */ +export async function stopModelRouterForDestroyedSandbox( + sandbox: SandboxEntry | null, + deps: StopModelRouterForDestroyedSandboxDeps, +): Promise { + const isRoutedProvider = deps.isRoutedProvider ?? isRoutedInferenceProvider; + if (!isRoutedProvider(sandbox?.provider)) return; + const port = resolveDestroyedSandboxRouterPort(sandbox?.endpointUrl); + const listSandboxes = deps.listSandboxes ?? registry.listSandboxes; + // Called after registry removal, so every remaining entry is a peer. + const routedPeerRemains = listSandboxes().sandboxes.some( + (entry) => + isRoutedProvider(entry.provider) && + resolveDestroyedSandboxRouterPort(entry.endpointUrl) === port, + ); + if (routedPeerRemains) return; + + const ownsPort = deps.ownsPort ?? doesModelRouterProcessOwnPort; + const findPidForPort = deps.findPidForPort ?? findModelRouterPidForPort; + const session = deps.loadSession(); + const recordedPid = session?.routerPid ?? null; + const recordedCredentialHash = session?.routerCredentialHash ?? null; + const recordedPidOwnsPort = ownsPort(recordedPid, port); + const sessionMatchesDestroyedSandbox = + session !== null && + session.sandboxName === sandbox?.name && + resolveDestroyedSandboxRouterPort(session.endpointUrl) === port; + const sessionOwnsTargetRouter = recordedPidOwnsPort || sessionMatchesDestroyedSandbox; + const pid = recordedPidOwnsPort ? (recordedPid as number) : findPidForPort(port); + + if (pid !== null) { + const log = deps.log ?? console.log; + const warn = deps.warn ?? console.warn; + log(` Stopping Model Router (PID ${pid})...`); + try { + await (deps.stopProcess ?? stopModelRouterProcess)(pid, port); + } catch (error) { + warn( + `Failed to stop the Model Router (PID ${pid}) on port ${port}: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + warn( + `Stop it manually (kill ${pid}) before the next Model Router onboarding, or onboarding fails with "Port ${port} already has a healthy router endpoint".`, + ); + return; + } + } + + // Clear when either field is set: a session with only a credential hash + // still carries stale router identity after the last routed sandbox is gone. + if (sessionOwnsTargetRouter && (recordedPid !== null || recordedCredentialHash !== null)) { + deps.updateSession((current: Session) => { + if ( + current.sessionId !== session?.sessionId || + current.sandboxName !== session?.sandboxName || + current.endpointUrl !== session?.endpointUrl || + current.routerPid !== recordedPid || + current.routerCredentialHash !== recordedCredentialHash + ) { + return current; + } + current.routerPid = null; + current.routerCredentialHash = null; + return current; + }); + } +} + export function prepareSandboxDestroy(sandboxName: string): SandboxDestroyPreflight { const sandbox = registry.getSandbox(sandboxName); console.log(` Deleting sandbox '${sandboxName}'...`); diff --git a/src/lib/actions/sandbox/destroy.ts b/src/lib/actions/sandbox/destroy.ts index 710dc66988b..dc912ddb1f1 100644 --- a/src/lib/actions/sandbox/destroy.ts +++ b/src/lib/actions/sandbox/destroy.ts @@ -53,7 +53,11 @@ import { classifyDestroySandboxPresence, isSameDestroyContainerIdentityProof, } from "./destroy-presence"; -import { prepareSandboxDestroy, stopSandboxInferenceResources } from "./destroy-preflight"; +import { + prepareSandboxDestroy, + stopModelRouterForDestroyedSandbox, + stopSandboxInferenceResources, +} from "./destroy-preflight"; import { type WipeSandboxStateDeps, wipeSandboxState } from "./wipe-state"; export { assertUnambiguousDestroyContainerIdentity, classifyDestroySandboxPresence }; @@ -661,6 +665,27 @@ async function destroySandboxUnlocked( if (deleteSucceededOrAlreadyGone && removed && priorHttpsPinRouteId) { await revokeDestroyedSandboxHttpsPinRoute(cleanupGatewayName, priorHttpsPinRouteId); } + if (deleteSucceededOrAlreadyGone && removed) { + try { + // The routed-peer scan and router stop are one critical section with + // routed onboarding's route registration, which runs under the same + // gateway route lock. Otherwise concurrent onboarding can register a + // routed sandbox after the scan and then lose its shared router. + await withGatewayRouteMutationLock(cleanupGatewayName, () => + stopModelRouterForDestroyedSandbox(sandbox, { + loadSession: onboardSession.loadSession, + updateSession: onboardSession.updateSession, + warn: defaultDestroyWarn, + }), + ); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + defaultDestroyWarn( + `Sandbox deletion succeeded, but the Model Router teardown did not complete: ${detail}. ` + + `Stop the Model Router process manually before the next Model Router onboarding.`, + ); + } + } const session = onboardSession.loadSession(); if (session && session.sandboxName === sandboxName) { onboardSession.updateSession((s: Session) => { diff --git a/test/helpers/destroy-flow-test-harness.ts b/test/helpers/destroy-flow-test-harness.ts index 7ca7a023302..fb09a0647e0 100644 --- a/test/helpers/destroy-flow-test-harness.ts +++ b/test/helpers/destroy-flow-test-harness.ts @@ -39,10 +39,12 @@ export type DestroyHarness = { setSandboxPresent: (present: boolean) => void; shieldsDownSpy: MockInstance; stopAllSpy: MockInstance; + stopModelRouterForDestroyedSandboxSpy: MockInstance; stopNimByNameSpy: MockInstance; unloadOllamaModelsSpy: MockInstance; updateSessionSpy: MockInstance; warnSpy: MockInstance; + withGatewayRouteMutationLockSpy: MockInstance; }; type DestroyHarnessOptions = { @@ -69,9 +71,12 @@ type DestroyHarnessOptions = { openshellDriver?: string; prepareMcpBridgeError?: string; promptResponses?: string[]; + provider?: string; registeredSandboxCount?: number; + removeSandboxResult?: boolean; restoreMcpError?: string; sandboxPresent?: boolean; + sessionRouterPid?: number; shieldsDown?: boolean; shieldsUpError?: Error; stopInferenceError?: string; @@ -154,11 +159,13 @@ export function createDestroyHarness(options: DestroyHarnessOptions = {}): Destr const sandboxProviderCleanup = requireDist("../../onboard/sandbox-provider-cleanup.js"); const nim = requireDist("../../inference/nim.js"); const ollamaProxy = requireDist("../../inference/ollama/proxy.js"); + const gatewayRouteMutationLock = requireDist("../../inference/gateway-route-mutation-lock.js"); const httpsPinRuntimeAdapter = requireDist("../../inference/https-pin-runtime-adapter.js"); const tunnelServices = requireDist("../../tunnel/services.js"); const onboardSession = requireDist("../../state/onboard-session.js"); const registry = requireDist("../../state/registry.js"); const destroyExecution = requireDist("./destroy-execution.js"); + const destroyPreflight = requireDist("./destroy-preflight.js"); const sandboxSession = requireDist("../../state/sandbox-session.js"); const shields = requireDist("../../shields/index.js"); const timerControl = requireDist("../../shields/timer-control.js"); @@ -178,6 +185,7 @@ export function createDestroyHarness(options: DestroyHarnessOptions = {}): Destr ...sandboxEntry, imageTag: options.imageTag === undefined ? sandboxEntry.imageTag : options.imageTag, agent: options.agent ?? sandboxEntry.agent, + ...(options.provider ? { provider: options.provider } : {}), ...(options.openshellDriver ? { openshellDriver: options.openshellDriver } : {}), ...(options.endpointUrl ? { endpointUrl: options.endpointUrl } : {}), ...(options.workload ? { workload: options.workload } : {}), @@ -204,17 +212,29 @@ export function createDestroyHarness(options: DestroyHarnessOptions = {}): Destr })), })); const removeSandboxSpy = vi.spyOn(registry, "removeSandbox").mockImplementation(() => { + if (options.removeSandboxResult === false) return false; registeredSandboxCount = Math.max(0, registeredSandboxCount - 1); return true; }); + const stopModelRouterForDestroyedSandboxSpy = vi + .spyOn(destroyPreflight, "stopModelRouterForDestroyedSandbox") + .mockResolvedValue(undefined); const retirePortableLifecycleReceiptSpy = vi .spyOn(destroyExecution, "retirePortableLifecycleAuthority") .mockImplementation(() => undefined); const revokeHttpsPinRuntimeAdapterRouteSpy = vi .spyOn(httpsPinRuntimeAdapter, "revokeHttpsPinRuntimeAdapterRoute") .mockResolvedValue(true); + // Pass-through: run the critical section without the cross-process lease so + // flow tests stay hermetic while asserting lock scope. + const withGatewayRouteMutationLockSpy = vi + .spyOn(gatewayRouteMutationLock, "withGatewayRouteMutationLock") + .mockImplementation(async (_gatewayName: unknown, operation: unknown) => + (operation as () => Promise)(), + ); vi.spyOn(onboardSession, "loadSession").mockReturnValue({ sandboxName: "alpha", + ...(options.sessionRouterPid ? { routerPid: options.sessionRouterPid } : {}), }); const updateSessionSpy = vi .spyOn(onboardSession, "updateSession") @@ -419,9 +439,11 @@ export function createDestroyHarness(options: DestroyHarnessOptions = {}): Destr }, shieldsDownSpy, stopAllSpy, + stopModelRouterForDestroyedSandboxSpy, stopNimByNameSpy, unloadOllamaModelsSpy, updateSessionSpy, warnSpy, + withGatewayRouteMutationLockSpy, }; } diff --git a/test/package-contract/destroy-model-router-flow.test.ts b/test/package-contract/destroy-model-router-flow.test.ts new file mode 100644 index 00000000000..db3a913d503 --- /dev/null +++ b/test/package-contract/destroy-model-router-flow.test.ts @@ -0,0 +1,121 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { type ChildProcess, spawn } from "node:child_process"; +import fs from "node:fs"; +import net from "node:net"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { stopModelRouterForDestroyedSandbox } from "../../dist/lib/actions/sandbox/destroy-preflight"; +import type { Session } from "../../dist/lib/state/onboard-session"; + +// A real detached HTTP server whose command line matches the model-router +// proxy shape (venv-style interposition: args[0]=node, args[1]=.../model-router). +const STUB_SOURCE = [ + 'const http = require("node:http");', + 'const port = Number(process.argv[process.argv.indexOf("--port") + 1]);', + "http", + ' .createServer((_req, res) => { res.statusCode = 200; res.end("{}"); })', + ' .listen(port, "127.0.0.1");', +].join("\n"); + +async function reserveLoopbackPort(): Promise { + return new Promise((resolve, reject) => { + const server = net.createServer(); + server.once("error", reject); + server.listen(0, "127.0.0.1", () => { + const address = server.address() as net.AddressInfo; + server.close(() => resolve(address.port)); + }); + }); +} + +async function probeHealthy(port: number): Promise { + try { + const response = await fetch(`http://127.0.0.1:${port}/health`, { + signal: AbortSignal.timeout(1000), + }); + return response.ok; + } catch { + return false; + } +} + +describe("destroySandbox model-router teardown (#9098)", () => { + let stubDir: string; + let stub: ChildProcess | null = null; + + beforeEach(() => { + stubDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-router-stub-")); + }); + + afterEach(() => { + try { + stub?.kill("SIGKILL"); + } catch { + // Already exited. + } + stub = null; + fs.rmSync(stubDir, { recursive: true, force: true }); + vi.restoreAllMocks(); + }); + + it( + "destroying the last routed sandbox stops the tracked router proxy and frees its port (#9098)", + { timeout: 30_000 }, + async () => { + const port = await reserveLoopbackPort(); + const stubPath = path.join(stubDir, "model-router"); + fs.writeFileSync(stubPath, STUB_SOURCE); + stub = spawn(process.execPath, [stubPath, "proxy", "--port", String(port)], { + stdio: "ignore", + }); + let stubExited = false; + stub.on("exit", () => { + stubExited = true; + }); + await vi.waitFor(async () => expect(await probeHealthy(port)).toBe(true), { + timeout: 10_000, + interval: 100, + }); + + const session = { + sessionId: "router-session", + sandboxName: "alpha", + provider: "nvidia-router", + endpointUrl: `http://host.openshell.internal:${port}/v1`, + routerPid: stub.pid, + routerCredentialHash: "router-credential-hash", + } as Session; + const updateSession = vi.fn((mutator: (current: Session) => Session | void) => { + mutator(session); + return session; + }); + + await expect( + stopModelRouterForDestroyedSandbox( + { + name: "alpha", + provider: "nvidia-router", + endpointUrl: session.endpointUrl, + }, + { + listSandboxes: () => ({ sandboxes: [], defaultSandbox: null }), + loadSession: () => session, + updateSession, + }, + ), + ).resolves.toBeUndefined(); + + await vi.waitFor(() => expect(stubExited).toBe(true), { timeout: 8_000, interval: 100 }); + expect(await probeHealthy(port)).toBe(false); + expect(updateSession).toHaveBeenCalledOnce(); + expect(session).toEqual( + expect.objectContaining({ routerPid: null, routerCredentialHash: null }), + ); + }, + ); +});