diff --git a/src/lib/actions/uninstall/all-gateway-ports.test.ts b/src/lib/actions/uninstall/all-gateway-ports.test.ts index c6cb52f7921..bb46542bad4 100644 --- a/src/lib/actions/uninstall/all-gateway-ports.test.ts +++ b/src/lib/actions/uninstall/all-gateway-ports.test.ts @@ -29,13 +29,16 @@ afterEach(() => { function sweepDeps(overrides: AllGatewayPortsDeps = {}) { const error = vi.fn(); - const runPortPass = vi.fn((_port: number) => 0); + const runPortPass = vi.fn( + (_port: number, _options: UninstallRunOptions, _env: NodeJS.ProcessEnv) => 0, + ); const runSelectedPass = vi.fn(async (_options: UninstallRunOptions, _deps: UninstallRunDeps) => ({ exitCode: 0, })); const deps: AllGatewayPortsDeps = { env: { HOME: "/home/tester" } as NodeJS.ProcessEnv, error, + gatewayStateDirForPort: () => null, home: "/home/tester", listGatewayPorts: () => [8080, 18080, 9000], log: vi.fn(), @@ -103,6 +106,71 @@ describe("uninstall across every gateway port (#7791)", () => { expect(error).toHaveBeenCalledWith(expect.stringContaining("Refusing to uninstall gateway")); }); + it("restores each recorded custom state directory only for its gateway child (#10665)", async () => { + const { deps, runPortPass } = sweepDeps({ + gatewayStateDirForPort: (_home, port) => + port === 9000 ? "/home/tester/custom-gateway-9000" : null, + }); + + await runUninstallAllGatewayPorts(OPTIONS, deps); + + const envByPort = new Map( + runPortPass.mock.calls.map(([port, _options, env]) => [ + port, + env.NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR, + ]), + ); + expect(envByPort.get(9000)).toBe("/home/tester/custom-gateway-9000"); + expect(envByPort.get(18080)).toBeUndefined(); + }); + + it("restores a recorded custom state directory for the selected pass (#10665)", async () => { + const { deps, runSelectedPass } = sweepDeps({ + gatewayStateDirForPort: (_home, port) => + port === 8080 ? "/home/tester/custom-gateway-8080" : null, + listGatewayPorts: () => [8080], + }); + + await runUninstallAllGatewayPorts(OPTIONS, deps); + + expect(runSelectedPass.mock.calls[0]?.[1].env).toMatchObject({ + NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR: "/home/tester/custom-gateway-8080", + }); + }); + + it("keeps an explicit selected-port override ahead of recorded state (#10665)", async () => { + const { deps, runSelectedPass } = sweepDeps({ + env: { + HOME: "/home/tester", + NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR: "/home/tester/explicit-gateway-8080", + }, + gatewayStateDirForPort: (_home, port) => + port === 8080 ? "/home/tester/recorded-gateway-8080" : null, + listGatewayPorts: () => [8080], + }); + + await runUninstallAllGatewayPorts(OPTIONS, deps); + + expect(runSelectedPass.mock.calls[0]?.[1].env).toMatchObject({ + NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR: "/home/tester/explicit-gateway-8080", + }); + }); + + it("fails before any pass when recorded state directories conflict (#10665)", async () => { + const { deps, error, runPortPass, runSelectedPass } = sweepDeps({ + gatewayStateDirForPort: () => { + throw new Error("conflicting OpenShell state directories"); + }, + }); + + const result = await runUninstallAllGatewayPorts(OPTIONS, deps); + + expect(result).toEqual({ exitCode: 1, ports: [9000, 18080, 8080] }); + expect(runPortPass).not.toHaveBeenCalled(); + expect(runSelectedPass).not.toHaveBeenCalled(); + expect(error).toHaveBeenCalledWith(expect.stringContaining("conflicting OpenShell")); + }); + it("reports a failed port pass and still finishes the remaining ports", async () => { const failingPortPass = vi.fn((port: number) => (port === 9000 ? 1 : 0)); const { deps, error, runSelectedPass } = sweepDeps({ @@ -344,6 +412,23 @@ describe("uninstall across every gateway port (#7791)", () => { expect(env.NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR).toBe("/srv/nemoclaw/selected-gateway"); }); + it("binds a recorded custom state directory to the matching child only (#10665)", () => { + const env = { + HOME: "/home/tester", + NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR: "/srv/nemoclaw/selected-gateway", + [ALL_GATEWAY_PORTS_ENV]: "1", + } as NodeJS.ProcessEnv; + + const childEnv = uninstallChildEnv(env, 9000, "/srv/nemoclaw/recorded-gateway-9000"); + + expect(childEnv).toMatchObject({ + NEMOCLAW_GATEWAY_PORT: "9000", + NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR: "/srv/nemoclaw/recorded-gateway-9000", + }); + expect(childEnv[ALL_GATEWAY_PORTS_ENV]).toBeUndefined(); + expect(env.NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR).toBe("/srv/nemoclaw/selected-gateway"); + }); + it.each([ [ "no extra flags", diff --git a/src/lib/actions/uninstall/all-gateway-ports.ts b/src/lib/actions/uninstall/all-gateway-ports.ts index 62e4bd96ea2..926b3a57722 100644 --- a/src/lib/actions/uninstall/all-gateway-ports.ts +++ b/src/lib/actions/uninstall/all-gateway-ports.ts @@ -27,7 +27,7 @@ import { GATEWAY_PORT } from "../../core/ports"; import { spawnExitCode } from "../../core/process-exit"; import { readLineFromStdin } from "../../core/stdin"; import { resolveGatewayName } from "../../onboard/gateway-binding"; -import { listGatewayStateRoots } from "../../state/gateway-registry"; +import { listGatewayStateRoots, readGatewayOpenShellStateDir } from "../../state/gateway-registry"; import { runUninstallPlanProduction, type UninstallRunDeps, @@ -40,6 +40,7 @@ export const ALL_GATEWAY_PORTS_ENV = "NEMOCLAW_UNINSTALL_ALL_GATEWAY_PORTS"; export interface AllGatewayPortsDeps extends UninstallRunDeps { home?: string; listGatewayPorts?: (home: string) => readonly number[]; + gatewayStateDirForPort?: (home: string, port: number) => string | null; runPortPass?: (port: number, options: UninstallRunOptions, env: NodeJS.ProcessEnv) => number; runSelectedPass?: ( options: UninstallRunOptions, @@ -76,15 +77,25 @@ export function uninstallChildArgs(options: UninstallRunOptions): string[] { * The child must never re-enter the sweep: dropping the request variable keeps * an inherited `NEMOCLAW_UNINSTALL_ALL_GATEWAY_PORTS=1` from recursing. */ -export function uninstallChildEnv(env: NodeJS.ProcessEnv, port: number): NodeJS.ProcessEnv { +export function uninstallChildEnv( + env: NodeJS.ProcessEnv, + port: number, + recordedGatewayStateDir?: string | null, +): NodeJS.ProcessEnv { const next: NodeJS.ProcessEnv = { ...env, NEMOCLAW_GATEWAY_PORT: String(port) }; delete next[ALL_GATEWAY_PORTS_ENV]; - if (port !== GATEWAY_PORT) delete next.NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR; + if (port !== GATEWAY_PORT) { + if (recordedGatewayStateDir) { + next.NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR = recordedGatewayStateDir; + } else { + delete next.NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR; + } + } return next; } function defaultRunPortPass( - port: number, + _port: number, options: UninstallRunOptions, env: NodeJS.ProcessEnv, ): number { @@ -92,7 +103,7 @@ function defaultRunPortPass( if (!entry) return 1; return spawnExitCode( spawnSync(process.execPath, [entry, ...uninstallChildArgs(options)], { - env: uninstallChildEnv(env, port), + env, stdio: "inherit", }), ); @@ -134,9 +145,9 @@ export async function runUninstallAllGatewayPorts( const error = deps.error ?? ((message: string) => console.error(message)); const readLine = deps.readLine ?? (() => readLineFromStdin()); const listPorts = deps.listGatewayPorts ?? defaultListGatewayPorts; + const gatewayStateDirForPort = deps.gatewayStateDirForPort ?? readGatewayOpenShellStateDir; const runPortPass = deps.runPortPass ?? defaultRunPortPass; const runSelectedPass = deps.runSelectedPass ?? runUninstallPlanProduction; - const runDeps = { ...deps, env }; const expectedGatewayName = resolveGatewayName(GATEWAY_PORT); if (options.gatewayName && options.gatewayName !== expectedGatewayName) { @@ -161,6 +172,28 @@ export async function runUninstallAllGatewayPorts( const otherPorts = [...new Set(discovered)] .filter((port) => port !== GATEWAY_PORT) .sort((left, right) => left - right); + const ordered = [...otherPorts, GATEWAY_PORT]; + const recordedStateDirs = new Map(); + try { + for (const port of ordered) { + const recorded = gatewayStateDirForPort(home, port); + if (recorded) recordedStateDirs.set(port, recorded); + } + } catch (failure) { + error( + `Cannot recover per-gateway OpenShell state directories: ${ + failure instanceof Error ? failure.message : String(failure) + }`, + ); + return { exitCode: 1, ports: ordered }; + } + const selectedRecordedStateDir = recordedStateDirs.get(GATEWAY_PORT); + const selectedEnv = + selectedRecordedStateDir && !env.NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR?.trim() + ? { ...env, NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR: selectedRecordedStateDir } + : env; + const runDeps = { ...deps, env: selectedEnv }; + if (otherPorts.length === 0) { let selected: Pick; try { @@ -182,7 +215,6 @@ export async function runUninstallAllGatewayPorts( return { exitCode: selected.exitCode, ports: [GATEWAY_PORT] }; } - const ordered = [...otherPorts, GATEWAY_PORT]; if (!confirmSweep(options, ordered, branding, log, readLine)) { return { exitCode: 0, ports: ordered }; } @@ -190,11 +222,21 @@ export async function runUninstallAllGatewayPorts( let exitCode = 0; const retainedGatewayPorts: number[] = []; for (const port of otherPorts) { + const recordedStateDir = recordedStateDirs.get(port); log(`Uninstalling gateway '${resolveGatewayName(port)}' on port ${String(port)}.`); - if (runPortPass(port, options, env) !== 0) { + if (recordedStateDir) { + log( + `Using recorded OpenShell gateway state directory ${JSON.stringify(recordedStateDir)} for port ${String(port)}.`, + ); + } + if (runPortPass(port, options, uninstallChildEnv(env, port, recordedStateDir)) !== 0) { exitCode = 1; retainedGatewayPorts.push(port); - error(`Uninstall failed for gateway port ${String(port)}; its resources may remain on disk.`); + error( + recordedStateDir + ? `Uninstall failed for gateway port ${String(port)} using recorded OpenShell state directory ${JSON.stringify(recordedStateDir)}; its resources may remain on disk.` + : `Uninstall failed for gateway port ${String(port)}; its resources may remain on disk.`, + ); } } log( diff --git a/src/lib/actions/uninstall/run-plan.ts b/src/lib/actions/uninstall/run-plan.ts index 375b2166be1..42c515a2a5f 100644 --- a/src/lib/actions/uninstall/run-plan.ts +++ b/src/lib/actions/uninstall/run-plan.ts @@ -1780,11 +1780,12 @@ function canRemoveScopedOpenShellResources( ? "Refusing scoped gateway cleanup because its sandbox namespace cannot be proven." : "Refusing gateway cleanup because the configured state directory's sandbox namespace cannot be proven.", ); - if (!runtime.env.NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR?.trim()) { - runtime.warn( - "If onboarding used a gateway state override, rerun with NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR= set to its original resolved directory.", - ); - } + const configuredStateDir = runtime.env.NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR?.trim(); + runtime.warn( + configuredStateDir + ? `Gateway port ${String(GATEWAY_PORT)} is using OpenShell state directory ${JSON.stringify(configuredStateDir)}. Verify that it is the original resolved onboarding directory, then rerun uninstall.` + : `If onboarding for gateway port ${String(GATEWAY_PORT)} used a gateway state override, rerun with NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR= set to its original resolved directory.`, + ); return false; } if (runtime.env.NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR?.trim()) { diff --git a/src/lib/onboard/created-sandbox-finalization.ts b/src/lib/onboard/created-sandbox-finalization.ts index 91e7964873a..55786d0cd05 100644 --- a/src/lib/onboard/created-sandbox-finalization.ts +++ b/src/lib/onboard/created-sandbox-finalization.ts @@ -615,6 +615,7 @@ type OnboardSandboxRegistrationOptions = { type OnboardGatewayBinding = { readonly gatewayName: string; readonly gatewayPort: number; + readonly openshellGatewayStateDir?: string | null; }; type OnboardPreparedPolicy = Pick< managedWorkloadOnboard.PreparedOnboardSandboxWorkloadLaunch, diff --git a/src/lib/onboard/sandbox-create/orchestration.ts b/src/lib/onboard/sandbox-create/orchestration.ts index 81f51c2fae0..eadedb373da 100644 --- a/src/lib/onboard/sandbox-create/orchestration.ts +++ b/src/lib/onboard/sandbox-create/orchestration.ts @@ -86,6 +86,13 @@ import { readValidatedRebuildPolicySource, } from "./rebuild-policy-handoff"; +function recordedOpenShellGatewayStateDir( + resolveStateDir: () => string, + env: NodeJS.ProcessEnv = process.env, +): string | null { + return env.NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR?.trim() ? resolveStateDir() : null; +} + function cancelRecoveryIdentity( liveExists: boolean, requireVerifiedCreateBoundary: () => VerifiedSandboxCreateBoundary, @@ -3055,7 +3062,11 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche { webSearchConfig, hermesAuthMethod: normalizeHermesAuthMethod(hermesAuthMethod) }, { plannedMessagingState, hermesToolGateways }, hermesApiPortReservationScope.effectivePort, - { gatewayName: GATEWAY_NAME, gatewayPort: GATEWAY_PORT }, + { + gatewayName: GATEWAY_NAME, + gatewayPort: GATEWAY_PORT, + openshellGatewayStateDir: recordedOpenShellGatewayStateDir(getDockerDriverGatewayStateDir), + }, { initialSandboxPolicy, compatibilityPolicyPath, diff --git a/src/lib/onboard/sandbox-registration.test.ts b/src/lib/onboard/sandbox-registration.test.ts index 4f5fe1a4232..075753a7617 100644 --- a/src/lib/onboard/sandbox-registration.test.ts +++ b/src/lib/onboard/sandbox-registration.test.ts @@ -93,6 +93,18 @@ function createdRegistryEntryInput( } describe("buildCreatedSandboxRegistryEntry", () => { + it("records the resolved custom OpenShell gateway state directory (#10665)", () => { + const entry = buildCreatedSandboxRegistryEntry( + createdRegistryEntryInput({ + gatewayName: "nemoclaw-19080", + gatewayPort: 19080, + openshellGatewayStateDir: "/home/tester/gateways/custom-19080", + }), + ); + + expect(entry.openshellGatewayStateDir).toBe("/home/tester/gateways/custom-19080"); + }); + it("records explicit OpenClaw identity for a managed workload receipt (#9356)", () => { const workload = managedWorkloadReceipt("openclaw"); const entry = buildCreatedSandboxRegistryEntry( diff --git a/src/lib/onboard/sandbox-registration.ts b/src/lib/onboard/sandbox-registration.ts index 4c214222cd8..95d8b220621 100644 --- a/src/lib/onboard/sandbox-registration.ts +++ b/src/lib/onboard/sandbox-registration.ts @@ -91,6 +91,7 @@ export interface CreatedSandboxRegistryEntryInput { lifecycleLiveIdentityFingerprint?: string; gatewayName: string; gatewayPort: number; + openshellGatewayStateDir?: string | null; hostMounts?: readonly import("../state/registry/types").SandboxHostMount[]; } @@ -289,6 +290,7 @@ export function buildCreatedSandboxRegistryEntry( lifecycleLiveIdentityFingerprint: input.lifecycleLiveIdentityFingerprint, gatewayName: input.gatewayName, gatewayPort: input.gatewayPort, + openshellGatewayStateDir: input.openshellGatewayStateDir ?? undefined, ...(input.hostMounts && input.hostMounts.length > 0 ? { hostMounts: cloneSandboxHostMounts(input.hostMounts) } : {}), diff --git a/src/lib/state/gateway-registry.test.ts b/src/lib/state/gateway-registry.test.ts index 2dde76a4959..9a5fc5f1044 100644 --- a/src/lib/state/gateway-registry.test.ts +++ b/src/lib/state/gateway-registry.test.ts @@ -7,9 +7,55 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; -import { listHostGatewayRegistryEntries } from "./gateway-registry"; +import { + listHostGatewayRegistryEntries, + registryOpenShellGatewayStateDir, +} from "./gateway-registry"; describe("host gateway registry index", () => { + it("recovers one custom OpenShell state directory across current and legacy rows (#10665)", () => { + expect( + registryOpenShellGatewayStateDir( + { + defaultSandbox: "current", + sandboxes: { + current: { + name: "current", + gatewayName: "nemoclaw-9123", + gatewayPort: 9123, + openshellGatewayStateDir: "/home/tester/custom-gateway-9123", + }, + legacy: { name: "legacy", gatewayName: "nemoclaw-9123", gatewayPort: 9123 }, + }, + }, + 9123, + ), + ).toBe("/home/tester/custom-gateway-9123"); + }); + + it("rejects conflicting custom OpenShell state directories for one port (#10665)", () => { + expect(() => + registryOpenShellGatewayStateDir( + { + defaultSandbox: "first", + sandboxes: { + first: { + name: "first", + gatewayPort: 9123, + openshellGatewayStateDir: "/home/tester/custom-a", + }, + second: { + name: "second", + gatewayPort: 9123, + openshellGatewayStateDir: "/home/tester/custom-b", + }, + }, + }, + 9123, + ), + ).toThrow(/conflicting OpenShell state directories/); + }); + it.runIf(process.platform !== "win32")( "rejects a symlinked numeric gateway root instead of omitting its allocations", () => { @@ -43,6 +89,34 @@ describe("host gateway registry index", () => { } }); + it("rejects a relative persisted OpenShell gateway state directory (#10665)", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-gateway-index-state-dir-")); + try { + const root = path.join(home, ".nemoclaw", "gateways", "9123"); + fs.mkdirSync(root, { recursive: true }); + fs.writeFileSync( + path.join(root, "sandboxes.json"), + JSON.stringify({ + defaultSandbox: "instance-a", + sandboxes: { + "instance-a": { + name: "instance-a", + gatewayName: "nemoclaw-9123", + gatewayPort: 9123, + openshellGatewayStateDir: "relative/gateway-state", + }, + }, + }), + ); + + expect(() => listHostGatewayRegistryEntries(home)).toThrow( + /invalid openshellGatewayStateDir/, + ); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + it("rejects malformed persisted dashboard ports instead of treating them as free", () => { const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-gateway-index-port-")); try { diff --git a/src/lib/state/gateway-registry.ts b/src/lib/state/gateway-registry.ts index 656fd5e045d..1a0bd5cea58 100644 --- a/src/lib/state/gateway-registry.ts +++ b/src/lib/state/gateway-registry.ts @@ -31,6 +31,7 @@ export interface GatewayRegistryEntry extends Record { hermesApiPort?: number | null; gatewayName?: string | null; gatewayPort?: number | null; + openshellGatewayStateDir?: string | null; } export interface GatewayRegistryDocument extends Record { @@ -114,6 +115,19 @@ function parseRegistry(filePath: string, raw: string): GatewayRegistryDocument { throw stateError(`${filePath} has an invalid ${field} for sandbox ${JSON.stringify(name)}`); } } + const gatewayStateDir = value.openshellGatewayStateDir; + if ( + gatewayStateDir !== undefined && + gatewayStateDir !== null && + (typeof gatewayStateDir !== "string" || + gatewayStateDir.length === 0 || + !path.isAbsolute(gatewayStateDir) || + path.resolve(gatewayStateDir) !== gatewayStateDir) + ) { + throw stateError( + `${filePath} has an invalid openshellGatewayStateDir for sandbox ${JSON.stringify(name)}`, + ); + } sandboxes[name] = value.dashboardPort === 0 ? { ...(value as GatewayRegistryEntry), dashboardPort: null } @@ -182,6 +196,35 @@ export function registryEntryGatewayPort(entry: GatewayRegistryEntry): number { return DEFAULT_GATEWAY_PORT; } +/** Recover one unambiguous onboard-time custom OpenShell state directory for a gateway port. */ +export function registryOpenShellGatewayStateDir( + registry: GatewayRegistryDocument, + gatewayPort: number, +): string | null { + const recorded = new Set(); + for (const entry of Object.values(registry.sandboxes)) { + if (registryEntryGatewayPort(entry) !== gatewayPort) continue; + if (typeof entry.openshellGatewayStateDir === "string") { + recorded.add(entry.openshellGatewayStateDir); + } + } + if (recorded.size > 1) { + throw stateError( + `gateway port ${String(gatewayPort)} has conflicting OpenShell state directories`, + ); + } + return recorded.values().next().value ?? null; +} + +/** Read one port's recorded custom OpenShell state directory from its canonical registry. */ +export function readGatewayOpenShellStateDir(home: string, gatewayPort: number): string | null { + const registry = readGatewayRegistryFile( + home, + path.join(nemoclawStateRoot(home, gatewayPort), "sandboxes.json"), + ); + return registry ? registryOpenShellGatewayStateDir(registry, gatewayPort) : null; +} + /** Enumerate the default root plus bounded, real, numeric non-default gateway roots. */ export function listGatewayStateRoots(home: string): GatewayStateRoot[] { const sharedRoot = nemoclawStateRoot(home, DEFAULT_GATEWAY_PORT); diff --git a/src/lib/state/registry.ts b/src/lib/state/registry.ts index 523977e00c8..412877d39d7 100644 --- a/src/lib/state/registry.ts +++ b/src/lib/state/registry.ts @@ -533,6 +533,7 @@ export function registerSandbox( dashboardRemoteBindPrepared: entry.dashboardRemoteBindPrepared === true ? true : undefined, gatewayName: entry.gatewayName ?? undefined, gatewayPort: entry.gatewayPort ?? undefined, + openshellGatewayStateDir: entry.openshellGatewayStateDir ?? undefined, pendingRouteReservation: options.pending === true ? true : undefined, reservationSessionId: options.pending === true ? options.reservationSessionId : undefined, }; diff --git a/src/lib/state/registry/types.ts b/src/lib/state/registry/types.ts index 0b7308283f5..325561e7cf5 100644 --- a/src/lib/state/registry/types.ts +++ b/src/lib/state/registry/types.ts @@ -159,6 +159,8 @@ export interface SandboxEntry extends Partial { // different NEMOCLAW_GATEWAY_PORT no longer recreates/kills the first (#4422). gatewayName?: string | null; gatewayPort?: number | null; + /** Resolved custom OpenShell gateway state directory used when this sandbox was onboarded. */ + openshellGatewayStateDir?: string | null; /** Whether the sandbox was intentionally stopped via the stop command (#11025). */ stopped?: boolean; }