From 6a86b428c9a79f895f4e9cee7d238679a8ad8414 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Mon, 20 Jul 2026 12:33:00 -0400 Subject: [PATCH 01/20] feat(onboard): bind external gateway authority Add a versioned gateway lifecycle declaration and durable resume binding. Register the exact endpoint and fail closed across preflight and recovery. Co-authored-by: Souvik Ghosh <138186578+souvikDevloper@users.noreply.github.com> Co-authored-by: Apurv Kumaria Signed-off-by: Julie Yaunches --- .../gateway-lifecycle-authority.mdx | 160 ++++++ docs/index.yml | 3 + src/lib/onboard.ts | 157 +++--- src/lib/onboard/checkpoint-replay.test.ts | 1 + .../onboard/checkpoint-resume-guard.test.ts | 1 + .../onboard/dashboard-preflight-ports.test.ts | 46 +- ...ocker-driver-gateway-port-listener.test.ts | 48 ++ .../docker-driver-gateway-port-listener.ts | 34 +- .../onboard/docker-driver-gateway-runtime.ts | 7 + .../gateway-authority-checkpoint.test.ts | 96 ++++ .../onboard/gateway-authority-checkpoint.ts | 73 +++ .../gateway-host-runtime-https.test.ts | 105 ++++ src/lib/onboard/gateway-host-runtime.test.ts | 311 ++++++++++++ src/lib/onboard/gateway-host-runtime.ts | 378 ++++++++++++++ src/lib/onboard/gateway-management.test.ts | 276 ++++++++++ src/lib/onboard/gateway-management.ts | 378 ++++++++++++++ src/lib/onboard/gateway-ownership.test.ts | 322 ++++++++++++ src/lib/onboard/gateway-ownership.ts | 470 ++++++++++++++++++ src/lib/onboard/gateway-port-conflict.test.ts | 19 + src/lib/onboard/gateway-port-conflict.ts | 6 + src/lib/onboard/gateway-recovery.test.ts | 30 ++ src/lib/onboard/gateway-recovery.ts | 10 + .../onboard/gateway-stale-port-reuse.test.ts | 66 ++- src/lib/onboard/gateway-stale-port-reuse.ts | 18 +- .../onboard/machine/handlers/gateway.test.ts | 169 ++++++- src/lib/onboard/machine/handlers/gateway.ts | 68 ++- .../sandbox-checkpoint-crash-recovery.test.ts | 1 + .../handlers/sandbox-messaging.test.ts | 1 + .../onboard/machine/handlers/sandbox.test.ts | 5 + .../machine/initial-flow-phases.test.ts | 51 +- ...preflight-gateway-cleanup-decision.test.ts | 29 ++ .../preflight-gateway-cleanup-decision.ts | 8 + .../onboard/preflight-gateway-reuse.test.ts | 25 +- src/lib/onboard/preflight-gateway-reuse.ts | 7 +- .../preflight-gateway-sequence.test.ts | 178 +++++++ src/lib/onboard/preflight-gateway-sequence.ts | 110 ++++ .../preflight-orphan-gateway-cleanup.test.ts | 96 ++++ .../preflight-orphan-gateway-cleanup.ts | 65 +++ src/lib/onboard/preflight-ports.ts | 11 +- src/lib/onboard/session-bootstrap.test.ts | 2 + .../state/onboard-checkpoint-migrate.test.ts | 1 + src/lib/state/onboard-checkpoint-migrate.ts | 3 +- src/lib/state/onboard-checkpoint-types.ts | 21 +- src/lib/state/onboard-checkpoint.test.ts | 63 ++- src/lib/state/onboard-checkpoint.ts | 121 ++++- 45 files changed, 3939 insertions(+), 111 deletions(-) create mode 100644 docs/deployment/gateway-lifecycle-authority.mdx create mode 100644 src/lib/onboard/gateway-authority-checkpoint.test.ts create mode 100644 src/lib/onboard/gateway-authority-checkpoint.ts create mode 100644 src/lib/onboard/gateway-host-runtime-https.test.ts create mode 100644 src/lib/onboard/gateway-host-runtime.test.ts create mode 100644 src/lib/onboard/gateway-host-runtime.ts create mode 100644 src/lib/onboard/gateway-management.test.ts create mode 100644 src/lib/onboard/gateway-management.ts create mode 100644 src/lib/onboard/gateway-ownership.test.ts create mode 100644 src/lib/onboard/gateway-ownership.ts create mode 100644 src/lib/onboard/preflight-gateway-sequence.test.ts create mode 100644 src/lib/onboard/preflight-gateway-sequence.ts create mode 100644 src/lib/onboard/preflight-orphan-gateway-cleanup.test.ts create mode 100644 src/lib/onboard/preflight-orphan-gateway-cleanup.ts diff --git a/docs/deployment/gateway-lifecycle-authority.mdx b/docs/deployment/gateway-lifecycle-authority.mdx new file mode 100644 index 00000000000..0c1d729ae7f --- /dev/null +++ b/docs/deployment/gateway-lifecycle-authority.mdx @@ -0,0 +1,160 @@ +--- +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +title: "Declare the OpenShell Gateway Lifecycle Authority" +sidebar-title: "Gateway Lifecycle Authority" +description: "Declare whether NemoClaw or an external platform supervisor owns the OpenShell gateway lifecycle, so one component controls each gateway port." +description-agent: "Explains the versioned gateway-management declaration, external supervisor validation, gateway registration, and resume safeguards. Use when a platform supervises the OpenShell gateway or onboarding reports an ownership, listener, or checkpoint conflict." +keywords: ["nemoclaw gateway lifecycle", "externally supervised gateway", "openshell gateway service", "gateway port conflict", "gateway ownership"] +content: + type: "reference" +--- +At every point, exactly one component owns the OpenShell gateway lifecycle. + +Platform images sometimes supervise the gateway with their own service. +Without an explicit authority, both the platform and NemoClaw can try to own the same process and port. +The gateway-management contract prevents that conflict by making NemoClaw manage the gateway or attach to a gateway that an external supervisor owns. + +## Management modes + +Choose the mode that matches the component responsible for process lifecycle operations. + +| Mode | Lifecycle owner | NemoClaw behavior | +|---|---|---| +| `nemoclaw-managed` | NemoClaw | Starts, stops, replaces, and recovers the gateway. | +| `externally-supervised` | A declared platform supervisor | Validates and attaches to the running gateway without managing its process. | + +When nothing is declared, NemoClaw keeps its existing managed behavior. +It uses the packaged gateway user service when that service is installed and otherwise manages a standalone gateway. + +## Declare the mode + +Set `NEMOCLAW_GATEWAY_MANAGEMENT` to the path of a JSON declaration. + +```bash +export NEMOCLAW_GATEWAY_MANAGEMENT=/etc/nemoclaw/gateway-management.json +``` + +An externally supervised declaration identifies the exact local endpoint, state directory, systemd unit, and executable. + +```json +{ + "version": 1, + "mode": "externally-supervised", + "endpoint": "https://127.0.0.1:8080", + "stateDir": "/var/lib/openshell/gateway", + "supervisor": { + "kind": "systemd-system", + "serviceName": "openshell-gateway.service", + "execPath": "/usr/local/bin/openshell-gateway" + }, + "requiredCapabilities": ["gateway.health", "sandbox.create"] +} +``` + +A managed declaration omits `endpoint`, `stateDir`, and `supervisor`. + +```json +{ + "version": 1, + "mode": "nemoclaw-managed", + "requiredCapabilities": [] +} +``` + +The declaration uses the following fields. + +| Field | Meaning | +|---|---| +| `version` | Contract version. Only `1` is supported. | +| `mode` | `nemoclaw-managed` or `externally-supervised`. | +| `endpoint` | Exact bare origin for an externally supervised gateway. Use `http` or `https`, a numeric IPv4 or IPv6 loopback address, and the gateway port that this NemoClaw process uses. Credentials, DNS names such as `localhost`, paths, query strings, and fragments are rejected. | +| `stateDir` | Absolute external gateway state directory. For HTTPS, it contains the client trust bundle. | +| `supervisor` | Required for `externally-supervised` and rejected for `nemoclaw-managed`. `kind` is `systemd-system` or `systemd-user`. `serviceName` names one `.service` unit, and `execPath` is the absolute gateway executable path. | +| `requiredCapabilities` | Capabilities onboarding needs. Supported values are `gateway.health`, `sandbox.create`, `sandbox.exec`, and `gpu.passthrough`. | + +The endpoint port must match `NEMOCLAW_GATEWAY_PORT`. +When the endpoint omits a port, `http` resolves to port `80` and `https` resolves to port `443` for this comparison. +Onboarding rejects a mismatch because validating one endpoint and operating a different gateway would break the authority boundary. + + +The declaration is versioned and secret-free. +NemoClaw rejects unknown fields and unsupported capabilities instead of ignoring them. +The authority may be persisted in checkpoints, diagnostics, and machine events, so do not place credentials or other secrets in it. + + + +Only systemd supervisors are supported in version 1. +NemoClaw verifies both the listener executable and its membership in the declared unit's cgroup. +A matching executable alone is not sufficient because the same binary can run outside the declared unit. + + +## Configure HTTPS health checks + +An HTTPS endpoint requires the external gateway's mTLS client files under the declared `stateDir`. + +| File | Purpose | +|---|---| +| `stateDir/tls/ca.crt` | Certificate authority used to verify the gateway. | +| `stateDir/tls/client/tls.crt` | Client certificate presented to the gateway. | +| `stateDir/tls/client/tls.key` | Client private key used for mTLS. | + +All three paths must be readable regular files. +NemoClaw uses this bundle for the mTLS gRPC health request at the exact declared endpoint path `/openshell.v1.OpenShell/Health`. +For an HTTP endpoint, NemoClaw sends the readiness request to the exact declared origin. + +## Validate and attach the gateway + +NemoClaw validates external authority before provider, policy, sandbox, or sandbox-registry mutations. +It rejects a systemd unit that reports inactive and checks that exactly one recognizable process holds the configured port, the process belongs to the unit's cgroup, the executable matches `execPath`, and the exact endpoint answers its health check. + +After validation, NemoClaw registers the exact endpoint under the canonical gateway name and selects it as the active OpenShell gateway. +Port `8080` uses the name `nemoclaw`, and another port uses `nemoclaw-`. +NemoClaw verifies the named and active registrations before advancing to provider selection. +If registration, selection, or verification fails, onboarding stops instead of continuing with a different gateway. + +Onboarding fails when any of the following conditions applies: + +- The endpoint port differs from the gateway port for this process. +- A required capability is unsupported. +- The declared supervisor is inactive. +- Nothing is listening on the declared endpoint. +- The listener set cannot be enumerated completely. +- No recognizable gateway process holds the port. +- More than one process holds the port. +- The listener does not belong to the declared systemd unit. +- The listener executable does not match `execPath`. +- The endpoint does not answer the required health check. +- The authority differs from the authority already bound to this run or checkpoint. + +## Preserve authority across resume + +NemoClaw records the resolved, secret-free authority in onboarding checkpoint schema version `2` before gateway preflight. +The record includes the canonical gateway name and port, mode, source, endpoint, state directory, supervisor identity, and required capabilities. + +Resume resolves the current authority again and compares the complete record with the checkpoint. +If any value changes, including the per-port gateway binding, resume fails before gateway effects and directs you to start a fresh onboarding run. +A completed gateway step does not bypass listener, supervisor, identity, health, registration, or checkpoint validation. + +## Keep external lifecycle paths inert + +External supervision prevents every NemoClaw gateway lifecycle path from affecting the supervised process. +Preflight skips stale gateway cleanup, orphaned container cleanup, volume removal, and registry cleanup associated with a NemoClaw-owned runtime. +The shared gateway start guard blocks normal starts. +The external attachment path has no standalone fallback. +Recovery and rebuild guard every start branch, including non-default port recovery. +The gateway state handler validates and attaches instead of starting, stopping, restarting, destroying, or replacing the gateway. + +Registering and selecting the validated endpoint changes only the local OpenShell gateway registration used by downstream commands. +It does not transfer process ownership to NemoClaw. + +## Fix a failure + +Resolve an external gateway failure through the declared supervisor. + +```bash +systemctl status openshell-gateway.service +``` + +Bring the supervised gateway up, leave exactly one verified process holding the configured port, and rerun `nemoclaw onboard`. +To hand the lifecycle back to NemoClaw, remove the platform gateway service and declare `nemoclaw-managed`, or remove the declaration. diff --git a/docs/index.yml b/docs/index.yml index 497e0652bae..9fb463a0cbf 100644 --- a/docs/index.yml +++ b/docs/index.yml @@ -289,6 +289,9 @@ navigation: - page: "Brev Web UI" path: deployment/brev-web-ui.mdx slug: brev-web-ui + - page: "Gateway Lifecycle Authority" + path: deployment/gateway-lifecycle-authority.mdx + slug: gateway-lifecycle-authority - section: "Monitoring" slug: monitoring collapsed: open-by-default diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 3c117bbf2d7..acba8249608 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -507,8 +507,8 @@ const { tryCleanupOrphanedDashboardForward } = require("./onboard/orphaned-dashboard-forward") as typeof import("./onboard/orphaned-dashboard-forward"); const { destroyGatewayForReuse } = require("./onboard/gateway-cleanup") as typeof import("./onboard/gateway-cleanup"); -const { applyPreflightGatewayCleanup } = - require("./onboard/preflight-gateway-cleanup-decision") as typeof import("./onboard/preflight-gateway-cleanup-decision"); +const { runPreflightGatewaySequence } = + require("./onboard/preflight-gateway-sequence") as typeof import("./onboard/preflight-gateway-sequence"); const { verifyGatewayContainerRunning } = require("./onboard/gateway-container-running") as typeof import("./onboard/gateway-container-running"); const { applyHealthyPortReuse } = @@ -517,8 +517,6 @@ const { destroyGatewayWithVolumeCleanup } = require("./onboard/gateway-destroy") as typeof import("./onboard/gateway-destroy"); const { gatewayCliSupportsLifecycleCommands } = require("./onboard/gateway-lifecycle") as typeof import("./onboard/gateway-lifecycle"); -const { reconcilePreflightGatewayReuseState } = - require("./onboard/preflight-gateway-reuse") as typeof import("./onboard/preflight-gateway-reuse"); const { getGatewayReuseHealthWaitConfig, isDockerDriverGatewayHttpReady: probeDockerDriverGatewayHttpReady, @@ -580,6 +578,8 @@ import { } from "./messaging-channel-config"; import { finalizationHandlerDeps } from "./onboard/finalization-deps"; import { streamGatewayStart } from "./onboard/gateway"; +import { bindGatewayAuthorityToCheckpoint } from "./onboard/gateway-authority-checkpoint"; +import { createGatewayHostRuntime } from "./onboard/gateway-host-runtime"; import { mergeRequiredHermesToolGatewayPolicyPresets, normalizeHermesToolGatewaySelections, @@ -650,6 +650,7 @@ const { getDockerDriverGatewayRuntimeDrift, getDockerDriverGatewayRuntimeDriftFromSnapshot, getDockerDriverGatewayStateDir, + getGatewayPortListenerRawScan, isDockerDriverGatewayPortListener, isDockerDriverGatewayProcess, isDockerDriverGatewayProcessAlive, @@ -1381,10 +1382,6 @@ function checkGatewayPortAvailable() { return checkPortAvailable(GATEWAY_PORT, dockerDriverGatewayEnv.getGatewayPortCheckOptions()); } -function getGatewayLocalEndpoint(): string { - return dockerDriverGatewayEnv.getGatewayHttpsEndpoint(GATEWAY_PORT); -} - const { gatewayClusterHealthcheckPassed, repairGatewayBootstrapSecrets } = createGatewayBootstrapRepairHelpers({ buildGatewayClusterExecArgv, @@ -1491,19 +1488,21 @@ async function preflight( }); ensureOpenshellForOnboard(); + // Bind the one lifecycle authority before applying any legacy listener-name + // heuristic. An external supervisor may intentionally use an arbitrary + // executable name; its listener is validated exactly by the downstream FSM. + const gatewayExternallySupervised = isGatewayExternallySupervised(); await failFastOnForeignGatewayPortConflict({ gatewayPort: GATEWAY_PORT, + externallySupervised: gatewayExternallySupervised, checkPortAvailable, getGatewayPortCheckOptions: dockerDriverGatewayEnv.getGatewayPortCheckOptions, isDockerDriverGatewayPortListener, exitProcess: (code) => process.exit(code), }); - // Classify gateway state before port checks. Legacy non-Docker-driver - // path destroys stale/unnamed gateways here so the port frees up for - // checks below; Docker-driver path defers the destructive recreate to - // step [2/8] (see applyPreflightGatewayCleanup). If another gateway is - // active but the named one exists, select it to avoid false conflicts. + // Classify gateway state before port checks; the legacy path destroys + // stale/unnamed gateways here while the Docker-driver path defers (#2020). const gatewaySnapshot = selectNamedGatewayForReuseIfNeeded(getGatewayReuseSnapshot()); let gatewayReuseState = gatewaySnapshot.gatewayReuseState; gatewayReuseState = await refreshDockerDriverGatewayReuseState(gatewayReuseState); @@ -1512,10 +1511,16 @@ async function preflight( // metadata can be stale after a manual `docker rm`. See #2020. Newer // package-managed OpenShell gateways do not have an openshell-cluster-* // Docker container, so the live CLI health check is the source of truth. - gatewayReuseState = await reconcilePreflightGatewayReuseState({ + // The reuse/cleanup/orphan stages run as one composed sequence so external + // supervision is enforced across the whole path, not per stage (#6576). + gatewayReuseState = await runPreflightGatewaySequence({ gatewayReuseState, + externallySupervised: gatewayExternallySupervised, supportsLifecycleCommands: gatewayCliSupportsLifecycleCommands(runCaptureOpenshell), + isDockerDriverGatewayEnabled: isLinuxDockerDriverGatewayEnabled(), gatewayName: GATEWAY_NAME, + cliDisplayName: cliDisplayName(), + dashboardPort: getOnboardDashboardPort(), verifyGatewayContainerRunning, recoverGatewayRuntime, waitForGatewayHttpReady, @@ -1525,60 +1530,20 @@ async function preflight( ignoreError: true, }), stopAllDashboardForwards, - destroyGateway, - destroyGatewayForReuse, getGatewayClusterImageDrift, exitProcess: (code) => process.exit(code), - }); - - gatewayReuseState = applyPreflightGatewayCleanup({ - gatewayReuseState, - isDockerDriverGatewayEnabled: isLinuxDockerDriverGatewayEnabled(), - cliDisplayName: cliDisplayName(), - dashboardPort: getOnboardDashboardPort(), - log: console.log, - warn: console.warn, - runOpenshell, destroyGateway, destroyGatewayForReuse, + runOpenshell, + dockerInspect, + dockerStop, + dockerRm, + dockerRemoveVolumesByPrefix, + clearRegistry: registry.clearAll, + log: console.log, + warn: console.warn, }); - // Clean up orphaned Docker containers from interrupted onboard (e.g. Ctrl+C - // during gateway start). The container may still be running even though - // OpenShell has no metadata for it (gatewayReuseState === "missing"). - if (gatewayReuseState === "missing" && !isLinuxDockerDriverGatewayEnabled()) { - const containerName = `openshell-cluster-${GATEWAY_NAME}`; - const inspectResult = dockerInspect( - ["--type", "container", "--format", "{{.State.Status}}", containerName], - { ignoreError: true, suppressOutput: true }, - ); - if (inspectResult.status === 0) { - console.log(" Cleaning up orphaned gateway container..."); - dockerStop(containerName, { - ignoreError: true, - suppressOutput: true, - }); - dockerRm(containerName, { - ignoreError: true, - suppressOutput: true, - }); - const postInspectResult = dockerInspect(["--type", "container", containerName], { - ignoreError: true, - suppressOutput: true, - }); - if (postInspectResult.status !== 0) { - dockerRemoveVolumesByPrefix(`openshell-cluster-${GATEWAY_NAME}`, { - ignoreError: true, - suppressOutput: true, - }); - registry.clearAll(); - console.log(" ✓ Orphaned gateway container removed"); - } else { - console.warn(" ! Found an orphaned gateway container, but automatic cleanup failed."); - } - } - } - // Required ports — gateway, plus the dashboard port when an explicit one // is requested. envVar is the override env var documented in // src/lib/core/ports.ts; surfacing it in the preflight error gives users a clear @@ -1596,19 +1561,20 @@ async function preflight( dashboardPort: dashboardPortToCheck, dashboardLabel: `${cliDisplayName()} dashboard`, }); - for (const { port, label, envVar } of requiredPorts) { + for (const { kind, port, label, envVar } of requiredPorts) { const portCheckOptions = - port === GATEWAY_PORT ? dockerDriverGatewayEnv.getGatewayPortCheckOptions() : undefined; + kind === "gateway" ? dockerDriverGatewayEnv.getGatewayPortCheckOptions() : undefined; let portCheck = await checkPortAvailable(port, portCheckOptions); if (!portCheck.ok) { const reuse = await applyHealthyPortReuse({ + kind, port, - gatewayPort: GATEWAY_PORT, dashboardPort: getOnboardDashboardPort(), label, runtimeDisplayName: cliDisplayName(), gatewayName: GATEWAY_NAME, gatewayReuseState, + externallySupervised: gatewayExternallySupervised, portCheckOptions, supportsLifecycleCommands: gatewayCliSupportsLifecycleCommands(runCaptureOpenshell), destroyGateway, @@ -1621,7 +1587,7 @@ async function preflight( ({ gatewayReuseState, portCheck } = reuse); if (portCheck.ok) continue; } - if (port === GATEWAY_PORT) { + if (kind === "gateway") { const dockerGatewayPid = getDockerDriverGatewayPortListenerPid(portCheck); if (dockerGatewayPid !== null) { rememberDockerDriverGatewayPid(dockerGatewayPid); @@ -1635,7 +1601,7 @@ async function preflight( // (e.g. dashboard forward left behind after destroy). Only kill the process // if its command line contains "openshell" to avoid killing unrelated SSH // tunnels the user may have set up on the same port. (#1950) - if (port === getOnboardDashboardPort() && portCheck.process === "ssh" && portCheck.pid) { + if (kind === "dashboard" && portCheck.process === "ssh" && portCheck.pid) { const outcome = await tryCleanupOrphanedDashboardForward({ port, pid: portCheck.pid, @@ -1746,6 +1712,7 @@ async function startGatewayWithOptions( gpuPassthrough = false, }: { exitOnFailure?: boolean; gpuPassthrough?: boolean } = {}, ) { + assertGatewayStartAllowed(exitOnFailure); step(2, 8, "Starting OpenShell gateway"); if (isLinuxDockerDriverGatewayEnabled()) { @@ -2144,6 +2111,7 @@ async function startGateway( async function startGatewayForRecovery(options = {}): Promise { return require("./onboard/gateway-recovery").startGatewayForRecovery(options, { + assertGatewayStartAllowed, getGatewayStartEnv, runCaptureOpenshell, runOpenshell, @@ -2152,29 +2120,35 @@ async function startGatewayForRecovery(options = {}): Promise { }); } -function getGatewayStartEnv(): Record { - const gatewayEnv = dockerDriverGatewayEnv.getGatewayStartNetworkEnv(GATEWAY_PORT); - const openshellVersion = getInstalledOpenshellVersion(); - const stableGatewayImage = openshellVersion - ? `ghcr.io/nvidia/openshell/cluster:${openshellVersion}` - : null; - if (stableGatewayImage && openshellVersion) { - gatewayEnv.OPENSHELL_CLUSTER_IMAGE = stableGatewayImage; - gatewayEnv.IMAGE_TAG = openshellVersion; - const overlayOverride = applyOverlayfsAutoFix(stableGatewayImage); - if (overlayOverride) { - gatewayEnv.OPENSHELL_CLUSTER_IMAGE = overlayOverride; - } - } - return gatewayEnv; -} - const applyOverlayfsAutoFix = overlayfsAutoFix.createOverlayfsAutoFix({ assessHost: preflightUtils.assessHost, ensurePatchedClusterImage: clusterImagePatch.ensurePatchedClusterImage, }); +const { + assertGatewayStartAllowed, + bindGatewayOwner, + getGatewayLocalEndpoint, + getGatewayOwner, + getGatewayStartEnv, + isGatewayExternallySupervised, + machineGatewayOwnerDeps, + resetGatewayOwnerBinding, +} = createGatewayHostRuntime({ + applyOverlayfsAutoFix, + checkGatewayPortAvailable, + gatewayName: () => GATEWAY_NAME, + gatewayPort: () => GATEWAY_PORT, + getGatewayPortListenerRawScan, + getInstalledOpenshellVersion, + runCaptureOpenshell, + runOpenshell, + resolveOpenShellGatewayBinary, + waitForGatewayHttpReady, +}); + async function recoverGatewayRuntime() { + assertGatewayStartAllowed(false); if (isLinuxDockerDriverGatewayEnabled()) { try { await startDockerDriverGateway({ exitOnFailure: false }); @@ -3986,12 +3960,14 @@ async function preflightAuthoritativeRebuildTarget( // ── Main ───────────────────────────────────────────────────────── const onboard = onboardEntryOptions.wrapOnboard(runOnboard, onboardSession); async function runOnboard(opts: OnboardOptions = {}): Promise { + resetGatewayOwnerBinding(); setupInferenceFactory.assertNoOpenShellGatewayEndpointOverride(); const runtimeControlRequests = runtimeControlFlow.applyOnboardRuntimeControlRequests(opts); const authoritativeGateway = authoritativeRebuildTarget.resolveAuthoritativeOnboardGatewayBinding(opts); const previousGatewayBinding = { name: GATEWAY_NAME, port: GATEWAY_PORT }; const previousOpenshellGateway = process.env.OPENSHELL_GATEWAY; + const previousOpenshellLocalTlsDir = process.env.OPENSHELL_LOCAL_TLS_DIR; const preparedDcodeRuntime = preparedDcodeRebuild.createPreparedDcodeRebuildRuntime( opts, authoritativeGateway?.name ?? GATEWAY_NAME, @@ -4203,6 +4179,15 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. ({ name: GATEWAY_NAME, port: GATEWAY_PORT } = onboardGateway); process.env.OPENSHELL_GATEWAY = GATEWAY_NAME; + const resolvedGatewayOwner = getGatewayOwner(); + let checkpointedGatewayOwner = resolvedGatewayOwner; + session = onboardSession.updateSession((currentSession) => { + checkpointedGatewayOwner = bindGatewayAuthorityToCheckpoint( + currentSession, + resolvedGatewayOwner, + ); + }); + bindGatewayOwner(checkpointedGatewayOwner); console.log(""); console.log(` ${cliDisplayName()} Onboarding`); if (isNonInteractive()) note(" (non-interactive mode)"); @@ -4279,6 +4264,7 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { gatewayName: GATEWAY_NAME, recreateSandbox: isRecreateSandbox, gatewayDeps: { + ...machineGatewayOwnerDeps, refreshDockerDriverGatewayReuseState, gatewayCliSupportsLifecycleCommands: () => gatewayCliSupportsLifecycleCommands(runCaptureOpenshell), @@ -4671,6 +4657,9 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { GATEWAY_PORT = previousGatewayBinding.port; if (previousOpenshellGateway === undefined) delete process.env.OPENSHELL_GATEWAY; else process.env.OPENSHELL_GATEWAY = previousOpenshellGateway; + if (previousOpenshellLocalTlsDir === undefined) delete process.env.OPENSHELL_LOCAL_TLS_DIR; + else process.env.OPENSHELL_LOCAL_TLS_DIR = previousOpenshellLocalTlsDir; + resetGatewayOwnerBinding(); } } diff --git a/src/lib/onboard/checkpoint-replay.test.ts b/src/lib/onboard/checkpoint-replay.test.ts index f741a2f92a9..639e9eb293e 100644 --- a/src/lib/onboard/checkpoint-replay.test.ts +++ b/src/lib/onboard/checkpoint-replay.test.ts @@ -23,6 +23,7 @@ function checkpoint(overrides: Partial = {}): OnboardCheckpoi webSearch: decisionUnset(), messaging: decisionUnset(), resourceProfile: decisionUnset(), + gatewayAuthority: decisionUnset(), effectGroups: {}, bindings: { credentialEnvs: [], registeredProviders: [] }, ...overrides, diff --git a/src/lib/onboard/checkpoint-resume-guard.test.ts b/src/lib/onboard/checkpoint-resume-guard.test.ts index 695fd9986fc..1ea17ce3006 100644 --- a/src/lib/onboard/checkpoint-resume-guard.test.ts +++ b/src/lib/onboard/checkpoint-resume-guard.test.ts @@ -36,6 +36,7 @@ const loadedCheckpoint: OnboardCheckpoint = { webSearch: decisionUnset(), messaging: decisionUnset(), resourceProfile: decisionUnset(), + gatewayAuthority: decisionUnset(), effectGroups: {}, bindings: { credentialEnvs: [], registeredProviders: [] }, }; diff --git a/src/lib/onboard/dashboard-preflight-ports.test.ts b/src/lib/onboard/dashboard-preflight-ports.test.ts index 502ce911e45..85b0ff80d5b 100644 --- a/src/lib/onboard/dashboard-preflight-ports.test.ts +++ b/src/lib/onboard/dashboard-preflight-ports.test.ts @@ -13,7 +13,14 @@ describe("buildRequiredPreflightPorts", () => { dashboardPort: null, dashboardLabel: "NemoClaw dashboard", }), - ).toEqual([{ port: 8080, label: "OpenShell gateway", envVar: "NEMOCLAW_GATEWAY_PORT" }]); + ).toEqual([ + { + kind: "gateway", + port: 8080, + label: "OpenShell gateway", + envVar: "NEMOCLAW_GATEWAY_PORT", + }, + ]); }); it("includes the dashboard port when one is explicitly requested", () => { @@ -24,8 +31,41 @@ describe("buildRequiredPreflightPorts", () => { dashboardLabel: "NemoClaw dashboard", }), ).toEqual([ - { port: 8080, label: "OpenShell gateway", envVar: "NEMOCLAW_GATEWAY_PORT" }, - { port: 18789, label: "NemoClaw dashboard", envVar: "NEMOCLAW_DASHBOARD_PORT" }, + { + kind: "gateway", + port: 8080, + label: "OpenShell gateway", + envVar: "NEMOCLAW_GATEWAY_PORT", + }, + { + kind: "dashboard", + port: 18789, + label: "NemoClaw dashboard", + envVar: "NEMOCLAW_DASHBOARD_PORT", + }, + ]); + }); + + it("keeps equal-number gateway and dashboard entries role-distinct (#6576)", () => { + expect( + buildRequiredPreflightPorts({ + gatewayPort: 8080, + dashboardPort: 8080, + dashboardLabel: "NemoClaw dashboard", + }), + ).toEqual([ + { + kind: "gateway", + port: 8080, + label: "OpenShell gateway", + envVar: "NEMOCLAW_GATEWAY_PORT", + }, + { + kind: "dashboard", + port: 8080, + label: "NemoClaw dashboard", + envVar: "NEMOCLAW_DASHBOARD_PORT", + }, ]); }); }); diff --git a/src/lib/onboard/docker-driver-gateway-port-listener.test.ts b/src/lib/onboard/docker-driver-gateway-port-listener.test.ts index 1444bb04fe1..60144951dc4 100644 --- a/src/lib/onboard/docker-driver-gateway-port-listener.test.ts +++ b/src/lib/onboard/docker-driver-gateway-port-listener.test.ts @@ -127,3 +127,51 @@ describe("Docker-driver gateway port listener discovery", () => { expect(runCaptureEx).toHaveBeenNthCalledWith(2, ["lsof", "-ti", ":18081", "-sTCP:LISTEN"]); }); }); + +describe("raw gateway port listener enumeration (#6576)", () => { + it("returns a live listener the Docker-driver filter would discard", () => { + // An externally supervised gateway is an ordinary systemd process with no + // Docker-driver markers, so isDockerDriverGatewayProcess returns false for + // it. The raw scan must still report it; the filtered scan must not. + const runCaptureEx = vi.fn(() => ({ stdout: "4242\n", exitCode: 0, timedOut: false })); + const { helpers } = makeHelpers({ + runCaptureEx, + isPidAlive: () => true, + isDockerDriverGatewayProcess: () => false, + }); + const portCheck = { ok: false, process: "openshell-gateway", pid: 4242 } as const; + + expect(helpers.getGatewayPortListenerRawScan(portCheck)).toEqual({ + pids: [4242], + complete: true, + }); + expect(helpers.getDockerDriverGatewayPortListenerScan(portCheck)).toEqual({ + pids: [], + complete: true, + }); + }); + + it("drops a dead PID from the raw enumeration", () => { + const runCaptureEx = vi.fn(() => ({ stdout: "4242\n5353\n", exitCode: 0, timedOut: false })); + const { helpers } = makeHelpers({ + runCaptureEx, + isPidAlive: (pid: number) => pid === 4242, + }); + + expect( + helpers.getGatewayPortListenerRawScan({ ok: false, process: "x", pid: 4242 }).pids, + ).toEqual([4242]); + }); + + it("reports an incomplete scan when lsof cannot enumerate against a held port", () => { + const runCaptureEx = vi.fn(() => ({ stdout: "", exitCode: 1, timedOut: false })); + const { helpers } = makeHelpers({ runCaptureEx }); + + // Port held (ok:false) but lsof saw nothing: a visibility contradiction, so + // the single-owner claim cannot be proven. + expect(helpers.getGatewayPortListenerRawScan({ ok: false, process: "x", pid: 0 })).toEqual({ + pids: [], + complete: false, + }); + }); +}); diff --git a/src/lib/onboard/docker-driver-gateway-port-listener.ts b/src/lib/onboard/docker-driver-gateway-port-listener.ts index c53076261c5..12188cfe26f 100644 --- a/src/lib/onboard/docker-driver-gateway-port-listener.ts +++ b/src/lib/onboard/docker-driver-gateway-port-listener.ts @@ -54,6 +54,10 @@ export function createDockerDriverGatewayPortListenerHelpers( portCheck: PortProbeResult, opts?: DockerDriverGatewayPortListenerOptions, ): DockerDriverGatewayPortListenerScan; + getGatewayPortListenerRawScan( + portCheck: PortProbeResult, + opts?: DockerDriverGatewayPortListenerOptions, + ): DockerDriverGatewayPortListenerScan; isDockerDriverGatewayPortListener( portCheck: PortProbeResult, opts?: DockerDriverGatewayPortListenerOptions, @@ -86,13 +90,21 @@ export function createDockerDriverGatewayPortListenerHelpers( return isGateway(pid, opts.gatewayBin) ? pid : null; } - function getDockerDriverGatewayPortListenerScan( + /** + * Every live process holding the gateway port, with no assumption about what + * kind of gateway it is. + * + * Docker-driver identity is deliberately not applied here: an externally + * supervised gateway (#6576) is an ordinary systemd-run executable with none + * of the Docker-driver env markers, so filtering by them would discard the + * very listener the caller is trying to recognize. Callers that need + * Docker-driver identity use getDockerDriverGatewayPortListenerScan instead. + */ + function getGatewayPortListenerRawScan( portCheck: PortProbeResult, opts: DockerDriverGatewayPortListenerOptions = {}, ): DockerDriverGatewayPortListenerScan { const candidates = new Set(); - const primaryPid = getDockerDriverGatewayPortListenerPid(portCheck, opts); - if (primaryPid !== null) candidates.add(primaryPid); let result: ListenerCaptureResult; try { @@ -108,6 +120,19 @@ export function createDockerDriverGatewayPortListenerHelpers( for (const pid of parseListenerPids(result.stdout)) candidates.add(pid); } + const alive = opts.isPidAliveFn ?? deps.isPidAlive; + return { pids: Array.from(candidates).filter((pid) => alive(pid)), complete }; + } + + function getDockerDriverGatewayPortListenerScan( + portCheck: PortProbeResult, + opts: DockerDriverGatewayPortListenerOptions = {}, + ): DockerDriverGatewayPortListenerScan { + const raw = getGatewayPortListenerRawScan(portCheck, opts); + const candidates = new Set(raw.pids); + const primaryPid = getDockerDriverGatewayPortListenerPid(portCheck, opts); + if (primaryPid !== null) candidates.add(primaryPid); + const platform = opts.platform ?? process.platform; const alive = opts.isPidAliveFn ?? deps.isPidAlive; const isGateway = @@ -116,13 +141,14 @@ export function createDockerDriverGatewayPortListenerHelpers( deps.isDockerDriverGatewayProcess(pid, gatewayBin, platform)); return { pids: Array.from(candidates).filter((pid) => alive(pid) && isGateway(pid, opts.gatewayBin)), - complete, + complete: raw.complete, }; } return { getDockerDriverGatewayPortListenerPid, getDockerDriverGatewayPortListenerScan, + getGatewayPortListenerRawScan, isDockerDriverGatewayPortListener: (portCheck, opts) => getDockerDriverGatewayPortListenerPid(portCheck, opts) !== null, }; diff --git a/src/lib/onboard/docker-driver-gateway-runtime.ts b/src/lib/onboard/docker-driver-gateway-runtime.ts index d475b2dac12..ff7c80a4e86 100644 --- a/src/lib/onboard/docker-driver-gateway-runtime.ts +++ b/src/lib/onboard/docker-driver-gateway-runtime.ts @@ -78,6 +78,11 @@ export function createDockerDriverGatewayRuntimeHelpers(deps: DockerDriverGatewa portCheck: PortProbeResult, opts?: DockerDriverGatewayPortListenerOptions, ): DockerDriverGatewayPortListenerScan; + /** Unfiltered listener enumeration; see getGatewayPortListenerRawScan (#6576). */ + getGatewayPortListenerRawScan( + portCheck: PortProbeResult, + opts?: DockerDriverGatewayPortListenerOptions, + ): DockerDriverGatewayPortListenerScan; /** Compatibility view for callers that only need the verified PID list. */ getDockerDriverGatewayPortListenerPids( portCheck: PortProbeResult, @@ -436,6 +441,7 @@ export function createDockerDriverGatewayRuntimeHelpers(deps: DockerDriverGatewa const { getDockerDriverGatewayPortListenerPid, getDockerDriverGatewayPortListenerScan, + getGatewayPortListenerRawScan, isDockerDriverGatewayPortListener, } = createDockerDriverGatewayPortListenerHelpers({ gatewayPort: currentGatewayPort, @@ -465,6 +471,7 @@ export function createDockerDriverGatewayRuntimeHelpers(deps: DockerDriverGatewa getDockerDriverGatewayPid, getDockerDriverGatewayPidFile, getDockerDriverGatewayPortListenerScan, + getGatewayPortListenerRawScan, getDockerDriverGatewayPortListenerPids, getDockerDriverGatewayPortListenerPid, getDockerDriverGatewayRuntimeDrift, diff --git a/src/lib/onboard/gateway-authority-checkpoint.test.ts b/src/lib/onboard/gateway-authority-checkpoint.test.ts new file mode 100644 index 00000000000..d0ed11c4870 --- /dev/null +++ b/src/lib/onboard/gateway-authority-checkpoint.test.ts @@ -0,0 +1,96 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { isDecisionSelected } from "../state/onboard-checkpoint-decision"; +import { createSession, normalizeSession } from "../state/onboard-session"; +import { + bindGatewayAuthorityToCheckpoint, + checkpointGatewayAuthority, +} from "./gateway-authority-checkpoint"; +import type { GatewayManagementDeclaration } from "./gateway-management"; +import { resolveGatewayOwner } from "./gateway-ownership"; + +const externalDeclaration: GatewayManagementDeclaration = { + version: 1, + mode: "externally-supervised", + endpoint: "http://127.0.0.1:8080", + stateDir: "/var/lib/openshell/gateway", + supervisor: { + kind: "systemd-system", + serviceName: "openshell-gateway.service", + execPath: "/usr/local/bin/openshell-gateway", + }, + requiredCapabilities: ["gateway.health", "sandbox.create"], +}; + +function externalOwner() { + return resolveGatewayOwner({ + gatewayName: "nemoclaw", + gatewayPort: 8080, + declaration: externalDeclaration, + hasPackagedService: false, + }); +} + +function managedOwner() { + return resolveGatewayOwner({ + gatewayName: "nemoclaw", + gatewayPort: 8080, + declaration: null, + hasPackagedService: false, + }); +} + +describe("durable gateway lifecycle authority", () => { + it("records the resolved authority before gateway effects (#6576)", () => { + const session = createSession({ sessionId: "authority-session" }); + const owner = externalOwner(); + + expect(bindGatewayAuthorityToCheckpoint(session, owner)).toEqual(owner); + const authority = session.checkpoint?.gatewayAuthority; + expect(authority).toBeDefined(); + if (!authority) throw new Error("expected gateway authority checkpoint"); + expect(isDecisionSelected(authority)).toBe(true); + expect(authority.kind === "selected" && authority.value).toEqual( + checkpointGatewayAuthority(owner), + ); + }); + + it("accepts the same authority after a process resume round-trip (#6576)", () => { + const firstProcess = createSession({ sessionId: "authority-session" }); + const owner = externalOwner(); + bindGatewayAuthorityToCheckpoint(firstProcess, owner); + const resumed = normalizeSession(JSON.parse(JSON.stringify(firstProcess)) as never); + + expect(resumed).not.toBeNull(); + expect(bindGatewayAuthorityToCheckpoint(resumed!, owner)).toEqual(owner); + }); + + it("rejects external-to-managed drift after a process resume before effects (#6576)", () => { + const firstProcess = createSession({ sessionId: "authority-session" }); + bindGatewayAuthorityToCheckpoint(firstProcess, externalOwner()); + const resumed = normalizeSession(JSON.parse(JSON.stringify(firstProcess)) as never); + + expect(() => bindGatewayAuthorityToCheckpoint(resumed!, managedOwner())).toThrow( + /changed since this onboarding attempt was checkpointed.*fresh onboarding run/s, + ); + }); + + it("rejects a different per-port gateway binding after resume (#6576)", () => { + const firstProcess = createSession({ sessionId: "authority-session" }); + bindGatewayAuthorityToCheckpoint(firstProcess, managedOwner()); + const resumed = normalizeSession(JSON.parse(JSON.stringify(firstProcess)) as never); + const otherPort = resolveGatewayOwner({ + gatewayName: "nemoclaw-9443", + gatewayPort: 9443, + declaration: null, + hasPackagedService: false, + }); + + expect(() => bindGatewayAuthorityToCheckpoint(resumed!, otherPort)).toThrow( + /authority changed since this onboarding attempt was checkpointed/, + ); + }); +}); diff --git a/src/lib/onboard/gateway-authority-checkpoint.ts b/src/lib/onboard/gateway-authority-checkpoint.ts new file mode 100644 index 00000000000..e0611b7612e --- /dev/null +++ b/src/lib/onboard/gateway-authority-checkpoint.ts @@ -0,0 +1,73 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + isDecisionDeclined, + isDecisionSelected, + decisionSelected, +} from "../state/onboard-checkpoint-decision"; +import type { CheckpointGatewayAuthority } from "../state/onboard-checkpoint-types"; +import type { Session } from "../state/onboard-session"; +import { deriveCheckpointFromSession } from "../state/onboard-checkpoint-migrate"; +import type { GatewayCapability } from "./gateway-management"; +import { + describeGatewayOwnerForError, + type GatewayOwner, + sameGatewayOwner, +} from "./gateway-ownership"; + +export function checkpointGatewayAuthority(owner: GatewayOwner): CheckpointGatewayAuthority { + return { + gatewayName: owner.gatewayName, + gatewayPort: owner.gatewayPort, + mode: owner.mode, + source: owner.source, + endpoint: owner.endpoint, + stateDir: owner.stateDir, + supervisor: owner.supervisor ? { ...owner.supervisor } : null, + requiredCapabilities: [...owner.requiredCapabilities], + }; +} + +export function gatewayOwnerFromCheckpoint(authority: CheckpointGatewayAuthority): GatewayOwner { + return { + ...authority, + supervisor: authority.supervisor ? { ...authority.supervisor } : null, + requiredCapabilities: [...authority.requiredCapabilities] as GatewayCapability[], + }; +} + +/** + * Bind the resolved owner to durable resume state before gateway preflight. + * A new process may re-resolve the declaration, but it may not silently adopt + * a different authority from the one recorded by the interrupted run. + */ +export function bindGatewayAuthorityToCheckpoint( + session: Session, + resolvedOwner: GatewayOwner, +): GatewayOwner { + const checkpoint = session.checkpoint ?? deriveCheckpointFromSession(session); + const decision = checkpoint.gatewayAuthority; + if (isDecisionDeclined(decision)) { + throw new Error("Onboarding checkpoint contains an invalid declined gateway authority."); + } + if (isDecisionSelected(decision)) { + const recordedOwner = gatewayOwnerFromCheckpoint(decision.value); + if (!sameGatewayOwner(recordedOwner, resolvedOwner)) { + throw new Error( + "Gateway lifecycle authority changed since this onboarding attempt was checkpointed " + + `(${describeGatewayOwnerForError(recordedOwner)} -> ${describeGatewayOwnerForError(resolvedOwner)}). ` + + "Changing authority requires a fresh onboarding run; resume will not perform gateway effects.", + ); + } + session.checkpoint = checkpoint; + return recordedOwner; + } + + session.checkpoint = { + ...checkpoint, + updatedAt: new Date().toISOString(), + gatewayAuthority: decisionSelected(checkpointGatewayAuthority(resolvedOwner)), + }; + return resolvedOwner; +} diff --git a/src/lib/onboard/gateway-host-runtime-https.test.ts b/src/lib/onboard/gateway-host-runtime-https.test.ts new file mode 100644 index 00000000000..a2974ebe4f9 --- /dev/null +++ b/src/lib/onboard/gateway-host-runtime-https.test.ts @@ -0,0 +1,105 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; + +import { afterEach, describe, expect, it, vi } from "vitest"; + +const readiness = vi.hoisted(() => ({ + http: vi.fn(async () => false), + https: vi.fn(async () => true), +})); + +vi.mock("./gateway-http-readiness", () => ({ + isGatewayHttpReady: readiness.http, + isDockerDriverGatewayHttpReady: readiness.https, + waitForGatewayHttpReady: async (options: { probe?: () => Promise } = {}) => + options.probe?.() ?? false, +})); + +import { createGatewayHostRuntime, type GatewayHostRuntimeDeps } from "./gateway-host-runtime"; +import { GATEWAY_MANAGEMENT_ENV_VAR } from "./gateway-management"; +import type { PortProbeResult } from "./preflight"; + +const ORIGINAL_ENV = { ...process.env }; +const STATE_DIR = "/var/lib/openshell/gateway"; +const SERVICE_NAME = "openshell-gateway.service"; +const EXEC_PATH = "/opt/platform/gatewayd"; + +afterEach(() => { + process.env = { ...ORIGINAL_ENV }; + vi.restoreAllMocks(); + readiness.http.mockClear(); + readiness.https.mockClear(); +}); + +function createDeps(): GatewayHostRuntimeDeps { + return { + applyOverlayfsAutoFix: () => null, + checkGatewayPortAvailable: async () => ({ ok: false }) as PortProbeResult, + gatewayName: () => "nemoclaw", + gatewayPort: () => 8080, + getGatewayPortListenerRawScan: () => ({ pids: [4242], complete: true }), + getInstalledOpenshellVersion: () => "0.0.90", + isGatewayHealthy: () => true, + readProcCgroup: () => `0::/system.slice/${SERVICE_NAME}\n`, + readProcExe: () => EXEC_PATH, + resolveOpenShellGatewayBinary: () => EXEC_PATH, + runCaptureOpenshell: () => "healthy", + runOpenshell: () => ({ status: 0 }), + spawnSyncImpl: (() => ({ status: 0, stdout: "active\n", stderr: "" })) as never, + waitForGatewayHttpReady: async () => false, + }; +} + +function declareHttpsExternalSupervision() { + process.env[GATEWAY_MANAGEMENT_ENV_VAR] = "/etc/nemoclaw/gateway-management.json"; + const declaration = JSON.stringify({ + version: 1, + mode: "externally-supervised", + endpoint: "https://127.0.0.1:8080", + stateDir: STATE_DIR, + supervisor: { + kind: "systemd-system", + serviceName: SERVICE_NAME, + execPath: EXEC_PATH, + }, + requiredCapabilities: ["gateway.health"], + }); + vi.spyOn(fs, "readFileSync").mockReturnValue(declaration as never); + vi.spyOn(fs, "statSync").mockReturnValue({ isFile: () => true } as never); + vi.spyOn(fs, "accessSync").mockImplementation(() => undefined); +} + +describe("externally supervised HTTPS gateway readiness", () => { + it("uses the production mTLS gRPC probe and the declared TLS bundle (#6576)", async () => { + declareHttpsExternalSupervision(); + const runtime = createGatewayHostRuntime(createDeps()); + const owner = runtime.getGatewayOwner(); + + const probe = await runtime.probeGatewayAttachment(owner); + + expect(probe.httpReady).toBe(true); + expect(readiness.http).not.toHaveBeenCalled(); + expect(readiness.https).toHaveBeenCalledWith( + undefined, + "https://127.0.0.1:8080/openshell.v1.OpenShell/Health", + ); + expect(process.env.OPENSHELL_LOCAL_TLS_DIR).toBe(`${STATE_DIR}/tls`); + expect(fs.statSync).toHaveBeenCalledTimes(3); + expect(fs.accessSync).toHaveBeenCalledTimes(3); + }); + + it("fails before probing when the declared client TLS bundle is unreadable (#6576)", async () => { + declareHttpsExternalSupervision(); + vi.mocked(fs.accessSync).mockImplementation(() => { + throw new Error("EACCES"); + }); + const runtime = createGatewayHostRuntime(createDeps()); + + await expect(runtime.probeGatewayAttachment(runtime.getGatewayOwner())).rejects.toThrow( + /TLS file is missing or unreadable/, + ); + expect(readiness.https).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/onboard/gateway-host-runtime.test.ts b/src/lib/onboard/gateway-host-runtime.test.ts new file mode 100644 index 00000000000..31c116cd15e --- /dev/null +++ b/src/lib/onboard/gateway-host-runtime.test.ts @@ -0,0 +1,311 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { createGatewayHostRuntime, type GatewayHostRuntimeDeps } from "./gateway-host-runtime"; +import { GATEWAY_MANAGEMENT_ENV_VAR } from "./gateway-management"; +import { evaluateGatewayAttachment } from "./gateway-ownership"; +import type { PortProbeResult } from "./preflight"; + +// Intentionally does not match the legacy OpenShell process-name allowlist. +// A declared external owner may use any absolute executable path; exact +// downstream attachment validation, not its basename, establishes identity. +const SYSTEMD_GATEWAY_EXEC = "/opt/platform/gatewayd"; +const SYSTEMD_GATEWAY_PID = 4242; + +const DECLARATION = { + version: 1, + mode: "externally-supervised", + endpoint: "http://127.0.0.1:8080", + stateDir: "/var/lib/openshell/gateway", + supervisor: { + kind: "systemd-system", + serviceName: "openshell-gateway.service", + execPath: SYSTEMD_GATEWAY_EXEC, + }, + requiredCapabilities: ["gateway.health"], +}; + +const ORIGINAL_ENV = { ...process.env }; + +afterEach(() => { + process.env = { ...ORIGINAL_ENV }; + vi.restoreAllMocks(); +}); + +/** The port is held, so the independent port probe reports it as unavailable. */ +const OCCUPIED_PORT: PortProbeResult = { ok: false } as PortProbeResult; + +function createDeps(overrides: Partial = {}): GatewayHostRuntimeDeps { + return { + applyOverlayfsAutoFix: () => null, + checkGatewayPortAvailable: async () => OCCUPIED_PORT, + gatewayName: () => "nemoclaw", + gatewayPort: () => 8080, + // A systemd-supervised gateway is an ordinary executable: the Docker-driver + // filtered scan would return no pids at all, which is why the probe must use + // the raw enumeration. + getGatewayPortListenerRawScan: () => ({ pids: [SYSTEMD_GATEWAY_PID], complete: true }), + getInstalledOpenshellVersion: () => "0.0.72", + isGatewayHealthy: () => true, + runCaptureOpenshell: () => "healthy", + runOpenshell: () => ({ status: 0 }), + resolveOpenShellGatewayBinary: () => SYSTEMD_GATEWAY_EXEC, + spawnSyncImpl: (() => ({ status: 0, stdout: "active\n", stderr: "" })) as never, + probeGatewayHttpReady: async () => true, + // Realistic /proc content for the declared systemd listener, so the probe + // produces listenerExecPath and listenerSupervisorMatch itself rather than a + // test overwriting them. + readProcExe: () => SYSTEMD_GATEWAY_EXEC, + readProcCgroup: () => `0::/system.slice/${DECLARATION.supervisor.serviceName}\n`, + waitForGatewayHttpReady: async () => true, + ...overrides, + }; +} + +function declareExternalSupervision(declaration: unknown = DECLARATION) { + process.env[GATEWAY_MANAGEMENT_ENV_VAR] = "/etc/nemoclaw/gateway-management.json"; + vi.spyOn(require("node:fs") as typeof import("node:fs"), "readFileSync").mockReturnValue( + JSON.stringify(declaration) as never, + ); +} + +describe("gateway host runtime ownership", () => { + it("resolves the declared external supervisor as the lifecycle owner (#6576)", () => { + declareExternalSupervision(); + + expect(createGatewayHostRuntime(createDeps()).getGatewayOwner()).toMatchObject({ + gatewayName: "nemoclaw", + gatewayPort: 8080, + mode: "externally-supervised", + source: "declared", + supervisor: { serviceName: "openshell-gateway.service" }, + }); + }); + + it("refuses to start a gateway the declared supervisor owns (#6576)", () => { + declareExternalSupervision(); + + expect(() => createGatewayHostRuntime(createDeps()).assertGatewayStartAllowed(false)).toThrow( + /openshell-gateway\.service/, + ); + }); + + it("fails closed on a malformed declaration rather than self-managing (#6576)", () => { + declareExternalSupervision({ ...DECLARATION, version: 99 }); + + expect(() => createGatewayHostRuntime(createDeps()).getGatewayOwner()).toThrow( + /Invalid gateway management declaration/, + ); + }); + + it("fails closed on unsupported capabilities before probing an external listener (#6576)", () => { + declareExternalSupervision({ + ...DECLARATION, + requiredCapabilities: ["gateway.teleport"], + }); + const checkGatewayPortAvailable = vi.fn().mockResolvedValue(OCCUPIED_PORT); + const runtime = createGatewayHostRuntime(createDeps({ checkGatewayPortAvailable })); + + expect(() => runtime.getGatewayOwner()).toThrow(/unsupported capability/); + expect(checkGatewayPortAvailable).not.toHaveBeenCalled(); + }); + + it("binds one authority for the run and returns it on later calls (#6576)", () => { + declareExternalSupervision(); + const runtime = createGatewayHostRuntime(createDeps()); + + const first = runtime.getGatewayOwner(); + const second = runtime.getGatewayOwner(); + + expect(first).toMatchObject({ mode: "externally-supervised" }); + expect(second).toEqual(first); + }); + + it("allows capability reordering but fails closed when required capabilities drift (#6576)", () => { + process.env[GATEWAY_MANAGEMENT_ENV_VAR] = "/etc/nemoclaw/gateway-management.json"; + let requiredCapabilities = ["sandbox.create", "gateway.health"]; + vi.spyOn(require("node:fs") as typeof import("node:fs"), "readFileSync").mockImplementation( + () => JSON.stringify({ ...DECLARATION, requiredCapabilities }) as never, + ); + const runtime = createGatewayHostRuntime(createDeps()); + + const first = runtime.getGatewayOwner(); + requiredCapabilities = ["gateway.health", "sandbox.create", "sandbox.create"]; + + expect(runtime.getGatewayOwner()).toEqual(first); + + requiredCapabilities = ["gpu.passthrough"]; + expect(() => runtime.getGatewayOwner()).toThrow(/authority changed during this run/); + }); + + it("fails closed when the authority would change mid-run instead of switching (#6576)", () => { + const runtime = createGatewayHostRuntime(createDeps()); + expect(runtime.getGatewayOwner()).toMatchObject({ mode: "nemoclaw-managed" }); + + // The declaration appears after the owner was already bound: silently + // adopting it would open a check/use gap between preflight and the FSM. + declareExternalSupervision(); + + expect(() => runtime.getGatewayOwner()).toThrow(/authority changed during this run/); + }); +}); + +describe("gateway host runtime attachment probe", () => { + it("attaches to a real systemd-supervised gateway listener (#6576)", async () => { + declareExternalSupervision(); + const runtime = createGatewayHostRuntime(createDeps()); + const owner = runtime.getGatewayOwner(); + + const probe = await runtime.probeGatewayAttachment(owner); + + // The declared systemd process carries no Docker-driver markers, so it must + // still be enumerated, resolved to its executable, and bound to the unit's + // cgroup — all produced by the probe, then evaluated without overwriting. + expect(probe).toMatchObject({ + gatewayPort: 8080, + httpReady: true, + portOccupied: true, + listenerPids: [SYSTEMD_GATEWAY_PID], + listenerScanComplete: true, + supervisorActive: true, + listenerExecPath: SYSTEMD_GATEWAY_EXEC, + listenerSupervisorMatch: true, + }); + expect(evaluateGatewayAttachment(owner, probe)).toMatchObject({ ok: true }); + }); + + it("rejects a same-binary listener outside the declared unit's cgroup (#6576)", async () => { + declareExternalSupervision(); + // Same executable, answering health, supervisor active — but the PID lives + // in a login session scope, not the unit. Exec-path match alone would accept + // it; cgroup binding must not. + const runtime = createGatewayHostRuntime( + createDeps({ + readProcCgroup: () => "0::/user.slice/user-1000.slice/session-7.scope\n", + }), + ); + const owner = runtime.getGatewayOwner(); + + const probe = await runtime.probeGatewayAttachment(owner); + + expect(probe.listenerExecPath).toBe(SYSTEMD_GATEWAY_EXEC); + expect(probe.listenerSupervisorMatch).toBe(false); + expect(evaluateGatewayAttachment(owner, probe)).toMatchObject({ + ok: false, + code: "identity_mismatch", + }); + }); + + it("rejects a listener whose executable differs from the arbitrary declared path (#6576)", async () => { + declareExternalSupervision(); + const runtime = createGatewayHostRuntime( + createDeps({ readProcExe: () => "/opt/platform/impostor-gateway" }), + ); + const owner = runtime.getGatewayOwner(); + + expect( + evaluateGatewayAttachment(owner, await runtime.probeGatewayAttachment(owner)), + ).toMatchObject({ ok: false, code: "identity_mismatch" }); + }); + + it("rejects an exact external listener that fails the declared health probe (#6576)", async () => { + declareExternalSupervision(); + const runtime = createGatewayHostRuntime( + createDeps({ probeGatewayHttpReady: async () => false }), + ); + const owner = runtime.getGatewayOwner(); + + expect( + evaluateGatewayAttachment(owner, await runtime.probeGatewayAttachment(owner)), + ).toMatchObject({ ok: false, code: "gateway_unreachable" }); + }); + + it("fails closed when the listener cgroup cannot be read (#6576)", async () => { + declareExternalSupervision(); + const runtime = createGatewayHostRuntime(createDeps({ readProcCgroup: () => null })); + const owner = runtime.getGatewayOwner(); + + const probe = await runtime.probeGatewayAttachment(owner); + + expect(probe.listenerSupervisorMatch).toBeNull(); + expect(evaluateGatewayAttachment(owner, probe)).toMatchObject({ + ok: false, + code: "unknown_listener", + }); + }); + + it("reports an unprobeable supervisor rather than guessing (#6576)", async () => { + declareExternalSupervision(); + const runtime = createGatewayHostRuntime( + createDeps({ + spawnSyncImpl: (() => ({ error: new Error("spawn ETIMEDOUT"), status: null })) as never, + }), + ); + + const probe = await runtime.probeGatewayAttachment(runtime.getGatewayOwner()); + + expect(probe.supervisorActive).toBeNull(); + }); + + it("reads the authoritative gateway port lazily, not at construction (#6576)", () => { + let port = 8080; + const runtime = createGatewayHostRuntime(createDeps({ gatewayPort: () => port })); + + port = 9443; + + expect(runtime.getGatewayStartEnv()).toMatchObject({ OPENSHELL_SERVER_PORT: "9443" }); + }); + + it("registers and selects the exact declared endpoint without prior gateway metadata (#6576)", () => { + declareExternalSupervision(); + process.env.OPENSHELL_GATEWAY = "ambient-sibling"; + const runOpenshell = vi.fn((_args: string[]) => ({ status: 0 })); + const runtime = createGatewayHostRuntime(createDeps({ runOpenshell })); + + runtime.attachGateway(runtime.getGatewayOwner()); + + expect(runOpenshell.mock.calls).toEqual([ + [ + ["gateway", "add", "http://127.0.0.1:8080", "--local", "--name", "nemoclaw"], + { ignoreError: true, suppressOutput: true }, + ], + [["gateway", "select", "nemoclaw"], { ignoreError: true, suppressOutput: true }], + ]); + expect(process.env.OPENSHELL_GATEWAY).toBe("nemoclaw"); + }); + + it("replaces stale registration before selecting the declared endpoint (#6576)", () => { + declareExternalSupervision(); + const statuses = [1, 0, 0]; + const runOpenshell = vi.fn((_args: string[]) => ({ status: statuses.shift() ?? 0 })); + const runtime = createGatewayHostRuntime(createDeps({ runOpenshell })); + + runtime.attachGateway(runtime.getGatewayOwner()); + + expect(runOpenshell.mock.calls.map(([args]) => args)).toEqual([ + ["gateway", "add", "http://127.0.0.1:8080", "--local", "--name", "nemoclaw"], + ["gateway", "remove", "nemoclaw"], + ["gateway", "add", "http://127.0.0.1:8080", "--local", "--name", "nemoclaw"], + ["gateway", "select", "nemoclaw"], + ]); + }); + + it("removes the attempted registration when exact gateway selection is unhealthy (#6576)", () => { + declareExternalSupervision(); + const runOpenshell = vi.fn((_args: string[]) => ({ status: 0 })); + const runtime = createGatewayHostRuntime( + createDeps({ isGatewayHealthy: () => false, runOpenshell }), + ); + + expect(() => runtime.attachGateway(runtime.getGatewayOwner())).toThrow( + /Failed to register and select/, + ); + expect(runOpenshell).toHaveBeenLastCalledWith(["gateway", "remove", "nemoclaw"], { + ignoreError: true, + suppressOutput: true, + }); + expect(process.env.OPENSHELL_GATEWAY).toBeUndefined(); + }); +}); diff --git a/src/lib/onboard/gateway-host-runtime.ts b/src/lib/onboard/gateway-host-runtime.ts new file mode 100644 index 00000000000..5bb80c003f3 --- /dev/null +++ b/src/lib/onboard/gateway-host-runtime.ts @@ -0,0 +1,378 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Host-side gateway runtime wiring for onboarding: the environment a gateway is + * started with, and the lifecycle authority that decides whether NemoClaw may + * start one at all (#6576). + * + * The ownership decision lives here rather than in the onboard entrypoint so it + * can be constructed with explicit dependencies and tested directly. The pure + * contract and decision logic live in `gateway-management` and + * `gateway-ownership`; this module only binds them to real host probes. + */ + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; +import { isGatewayHealthy } from "../state/gateway"; +import type { DockerDriverGatewayPortListenerScan } from "./docker-driver-gateway-port-listener"; +import { hasOpenShellGatewayUserService } from "./docker-driver-gateway-service"; +import { + isDockerDriverGatewayHttpReady, + isGatewayHttpReady, + waitForGatewayHttpReady, +} from "./gateway-http-readiness"; +import { loadGatewayManagementDeclaration } from "./gateway-management"; +import { + assertGatewayEffectAllowed, + cgroupBelongsToUnit, + describeGatewayOwnerForError, + type GatewayAttachmentProbe, + type GatewayOwner, + GatewayOwnershipError, + isExternallySupervised, + resolveGatewayOwner, + sameGatewayOwner, +} from "./gateway-ownership"; +import type { PortProbeResult } from "./preflight"; + +/** `systemctl is-active` is a local query; anything slower than this is wedged. */ +const SUPERVISOR_PROBE_TIMEOUT_MS = 5_000; + +export interface GatewayHostRuntimeDeps { + applyOverlayfsAutoFix(clusterImage: string): string | null; + checkGatewayPortAvailable(): Promise; + /** + * Read lazily: the onboarding entrypoint rebinds its gateway port at runtime + * when an authoritative gateway is selected, so a captured value goes stale. + */ + gatewayPort(): number; + gatewayName(): string; + /** + * Unfiltered listener enumeration. An externally supervised gateway is an + * ordinary systemd-run executable with no Docker-driver env markers, so the + * Docker-driver-filtered scan would discard it and report an unknown listener. + */ + getGatewayPortListenerRawScan( + portCheck: PortProbeResult, + opts?: { gatewayBin?: string | null }, + ): DockerDriverGatewayPortListenerScan; + getInstalledOpenshellVersion(): string | null; + isGatewayHealthy?: typeof isGatewayHealthy; + runCaptureOpenshell(args: string[], opts?: { ignoreError?: boolean }): string; + runOpenshell( + args: string[], + opts?: { ignoreError?: boolean; suppressOutput?: boolean }, + ): { status: number | null }; + resolveOpenShellGatewayBinary(): string | null; + spawnSyncImpl?: typeof import("node:child_process").spawnSync; + /** Overrides the readiness request; defaults to a real probe of `endpoint`. */ + probeGatewayHttpReady?(endpoint: string | null): Promise; + /** Overrides `/proc//exe` resolution; defaults to the real symlink. */ + readProcExe?(pid: number): string | null; + /** Overrides `/proc//cgroup` reads; defaults to the real file. */ + readProcCgroup?(pid: number): string | null; + waitForGatewayHttpReady(): Promise; +} + +export interface GatewayHostRuntime { + /** + * Fail before the caller can start a gateway that an external supervisor + * owns. Applies to onboarding, rebuild, and recovery alike. + */ + assertGatewayStartAllowed(exitOnFailure: boolean): void; + attachGateway(owner: GatewayOwner): void; + bindGatewayOwner(owner: GatewayOwner): void; + /** HTTPS endpoint of the gateway this process operates. */ + getGatewayLocalEndpoint(): string; + getGatewayOwner(): GatewayOwner; + resetGatewayOwnerBinding(): void; + /** Whether an external supervisor owns the gateway lifecycle this run (#6576). */ + isGatewayExternallySupervised(): boolean; + getGatewayStartEnv(): Record; + /** Gateway-ownership dependencies consumed by the onboarding FSM handler. */ + machineGatewayOwnerDeps: { + probeGatewayAttachment(owner: GatewayOwner): Promise; + resolveGatewayOwner(): GatewayOwner; + attachGateway(owner: GatewayOwner): void; + }; + probeGatewayAttachment(owner: GatewayOwner): Promise; +} + +export function createGatewayHostRuntime(deps: GatewayHostRuntimeDeps): GatewayHostRuntime { + let boundOwner: GatewayOwner | null = null; + + /** + * Resolve the one gateway lifecycle authority for this run. A malformed + * declaration throws instead of degrading to self-management: a host that + * meant to hand the gateway to an external supervisor must never silently get + * a second NemoClaw-owned gateway on the same port. + * + * The authority is bound on first resolution and stays fixed for the run. + * Later calls re-read the declaration and packaged-service state and compare: + * if they now describe a *different* authority, that is a check/use gap + * between preflight and the FSM handler, so it fails closed rather than + * silently switching owners mid-run. Changing authority is an explicit + * migration, not something a mutating file can do underneath a running + * onboard (#6576). + */ + function getGatewayOwner(): GatewayOwner { + const loaded = loadGatewayManagementDeclaration(); + if (!loaded.ok) { + throw new Error(`Invalid gateway management declaration: ${loaded.reason}`); + } + const resolved = resolveGatewayOwner({ + gatewayName: deps.gatewayName(), + gatewayPort: deps.gatewayPort(), + declaration: loaded.declaration, + hasPackagedService: hasOpenShellGatewayUserService(), + }); + if (boundOwner) { + if (!sameGatewayOwner(boundOwner, resolved)) { + throw new Error( + "Gateway lifecycle authority changed during this run " + + `(${describeGatewayOwnerForError(boundOwner)} -> ${describeGatewayOwnerForError(resolved)}). ` + + "Exactly one component owns the gateway per run; re-run onboarding to adopt the new authority.", + ); + } + return boundOwner; + } + boundOwner = resolved; + return boundOwner; + } + + function isSupervisorUnitActive(owner: GatewayOwner): boolean | null { + const supervisor = owner.supervisor; + if (!supervisor) return null; + const spawnSyncImpl = deps.spawnSyncImpl ?? spawnSync; + const scope = supervisor.kind === "systemd-user" ? ["--user"] : []; + const result = spawnSyncImpl("systemctl", [...scope, "is-active", supervisor.serviceName], { + encoding: "utf-8", + // spawnSync blocks the event loop, so a wedged systemd/D-Bus session would + // otherwise stall onboarding indefinitely. A timeout surfaces as an error, + // which the unknown-supervisor path below already treats as "cannot tell". + timeout: SUPERVISOR_PROBE_TIMEOUT_MS, + }); + if (result.error || result.status === null) return null; + return String(result.stdout ?? "").trim() === "active"; + } + + function readListenerExecPath(pid: number): string | null { + if (deps.readProcExe) return deps.readProcExe(pid); + try { + return fs.realpathSync.native(`/proc/${pid}/exe`); + } catch { + return null; + } + } + + function readProcCgroup(pid: number): string | null { + if (deps.readProcCgroup) return deps.readProcCgroup(pid); + try { + return fs.readFileSync(`/proc/${pid}/cgroup`, "utf-8"); + } catch { + return null; + } + } + + /** + * Bind a listening PID to the declared supervisor unit via cgroup membership + * — the authoritative evidence that the process is the unit's, not merely a + * same-binary impostor holding the port. Returns null when the relationship + * cannot be established (no PID or an unreadable cgroup), and the caller then + * fails closed. + */ + function readListenerSupervisorMatch( + owner: GatewayOwner, + pid: number | undefined, + ): boolean | null { + const supervisor = owner.supervisor; + if (!supervisor || typeof pid !== "number") return null; + const cgroupText = readProcCgroup(pid); + if (cgroupText === null) return null; + return cgroupBelongsToUnit(cgroupText, supervisor.serviceName); + } + + /** + * Probe the declared endpoint rather than the process default, so a + * declaration is never assessed against a different local listener. A + * declared endpoint on a port this process does not operate is rejected by + * `evaluateGatewayAttachment`, which sees both values. + */ + function waitForDeclaredGatewayHttpReady(owner: GatewayOwner): Promise { + if (deps.probeGatewayHttpReady) return deps.probeGatewayHttpReady(owner.endpoint); + if (!owner.endpoint) return deps.waitForGatewayHttpReady(); + // The endpoint is constrained to a supported loopback origin at parse time, + // so this cannot be pointed at an arbitrary host. + prepareExternalGatewayClient(owner); + const endpoint = owner.endpoint; + if (new URL(endpoint).protocol === "https:") { + return waitForGatewayHttpReady({ + probe: () => + isDockerDriverGatewayHttpReady(undefined, `${endpoint}/openshell.v1.OpenShell/Health`), + }); + } + return waitForGatewayHttpReady({ probe: () => isGatewayHttpReady(undefined, `${endpoint}/`) }); + } + + /** + * Gather the evidence needed to decide whether NemoClaw may attach to a + * gateway it does not own. Read-only: this runs before any effect. + */ + async function probeGatewayAttachment(owner: GatewayOwner): Promise { + const portCheck = await deps.checkGatewayPortAvailable(); + const scan = deps.getGatewayPortListenerRawScan(portCheck, { + gatewayBin: deps.resolveOpenShellGatewayBinary(), + }); + const [firstPid] = scan.pids; + return { + gatewayPort: deps.gatewayPort(), + httpReady: await waitForDeclaredGatewayHttpReady(owner), + // `ok` means the port is free; anything else means something holds it. + portOccupied: !portCheck.ok, + listenerPids: scan.pids, + listenerScanComplete: scan.complete, + supervisorActive: isSupervisorUnitActive(owner), + listenerExecPath: typeof firstPid === "number" ? readListenerExecPath(firstPid) : null, + listenerSupervisorMatch: readListenerSupervisorMatch(owner, firstPid), + }; + } + + function assertGatewayStartAllowed(exitOnFailure: boolean): void { + try { + assertGatewayEffectAllowed(getGatewayOwner(), "start"); + } catch (error) { + console.error(` ${error instanceof Error ? error.message : String(error)}`); + if (exitOnFailure) process.exit(1); + throw error; + } + } + + function prepareExternalGatewayClient(owner: GatewayOwner): void { + if (!isExternallySupervised(owner) || !owner.endpoint) return; + if (new URL(owner.endpoint).protocol !== "https:") return; + if (!owner.stateDir) { + throw new Error("Externally supervised HTTPS gateway requires a declared stateDir."); + } + const localTlsDir = path.join(owner.stateDir, "tls"); + for (const relativePath of ["ca.crt", "client/tls.crt", "client/tls.key"]) { + const filePath = path.join(localTlsDir, relativePath); + try { + if (!fs.statSync(filePath).isFile()) throw new Error("not a file"); + fs.accessSync(filePath, fs.constants.R_OK); + } catch { + throw new Error( + `Externally supervised gateway TLS file is missing or unreadable: ${filePath}`, + ); + } + } + process.env.OPENSHELL_LOCAL_TLS_DIR = localTlsDir; + } + + /** Register and select the exact endpoint whose listener identity was validated. */ + function attachGateway(owner: GatewayOwner): void { + if (!isExternallySupervised(owner) || !owner.endpoint) return; + prepareExternalGatewayClient(owner); + const add = () => + deps.runOpenshell( + ["gateway", "add", owner.endpoint as string, "--local", "--name", owner.gatewayName], + { ignoreError: true, suppressOutput: true }, + ); + let addResult = add(); + if (addResult.status !== 0) { + deps.runOpenshell(["gateway", "remove", owner.gatewayName], { + ignoreError: true, + suppressOutput: true, + }); + addResult = add(); + } + const selectResult = deps.runOpenshell(["gateway", "select", owner.gatewayName], { + ignoreError: true, + suppressOutput: true, + }); + const status = deps.runCaptureOpenshell(["status"], { ignoreError: true }); + const namedInfo = deps.runCaptureOpenshell(["gateway", "info", "-g", owner.gatewayName], { + ignoreError: true, + }); + const activeInfo = deps.runCaptureOpenshell(["gateway", "info"], { ignoreError: true }); + if ( + addResult.status !== 0 || + selectResult.status !== 0 || + !(deps.isGatewayHealthy ?? isGatewayHealthy)(status, namedInfo, activeInfo, owner.gatewayName) + ) { + deps.runOpenshell(["gateway", "remove", owner.gatewayName], { + ignoreError: true, + suppressOutput: true, + }); + if (process.env.OPENSHELL_GATEWAY === owner.gatewayName) { + delete process.env.OPENSHELL_GATEWAY; + } + throw new GatewayOwnershipError( + "gateway_registration_failed", + `Failed to register and select externally supervised gateway '${owner.gatewayName}' at ${owner.endpoint}.`, + owner, + ); + } + process.env.OPENSHELL_GATEWAY = owner.gatewayName; + } + + function bindGatewayOwner(owner: GatewayOwner): void { + const current = getGatewayOwner(); + if (!sameGatewayOwner(owner, current)) { + throw new Error( + "Gateway lifecycle authority changed before it could be bound to this run " + + `(${describeGatewayOwnerForError(owner)} -> ${describeGatewayOwnerForError(current)}).`, + ); + } + boundOwner = owner; + } + + function getGatewayLocalEndpoint(): string { + const owner = getGatewayOwner(); + if (isExternallySupervised(owner) && owner.endpoint) return owner.endpoint; + const { getGatewayHttpsEndpoint } = + require("./docker-driver-gateway-env") as typeof import("./docker-driver-gateway-env"); + return getGatewayHttpsEndpoint(deps.gatewayPort()); + } + + function getGatewayStartEnv(): Record { + // Resolved lazily, not through a module-scope import: the gateway env module + // reads NEMOCLAW_GATEWAY_BIND_ADDRESS at load, and callers that reload + // onboarding with a different environment drop it from the require cache. + // A hoisted binding would pin this module to the stale first instance. + const { getGatewayStartNetworkEnv } = + require("./docker-driver-gateway-env") as typeof import("./docker-driver-gateway-env"); + const gatewayEnv = getGatewayStartNetworkEnv(deps.gatewayPort()); + const openshellVersion = deps.getInstalledOpenshellVersion(); + if (openshellVersion) { + const stableGatewayImage = `ghcr.io/nvidia/openshell/cluster:${openshellVersion}`; + gatewayEnv.OPENSHELL_CLUSTER_IMAGE = stableGatewayImage; + gatewayEnv.IMAGE_TAG = openshellVersion; + const overlayOverride = deps.applyOverlayfsAutoFix(stableGatewayImage); + if (overlayOverride) { + gatewayEnv.OPENSHELL_CLUSTER_IMAGE = overlayOverride; + } + } + return gatewayEnv; + } + + return { + assertGatewayStartAllowed, + attachGateway, + bindGatewayOwner, + getGatewayLocalEndpoint, + getGatewayOwner, + getGatewayStartEnv, + isGatewayExternallySupervised: () => isExternallySupervised(getGatewayOwner()), + machineGatewayOwnerDeps: { + attachGateway, + probeGatewayAttachment, + resolveGatewayOwner: getGatewayOwner, + }, + probeGatewayAttachment, + resetGatewayOwnerBinding: () => { + boundOwner = null; + }, + }; +} diff --git a/src/lib/onboard/gateway-management.test.ts b/src/lib/onboard/gateway-management.test.ts new file mode 100644 index 00000000000..7f28a578512 --- /dev/null +++ b/src/lib/onboard/gateway-management.test.ts @@ -0,0 +1,276 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + GATEWAY_MANAGEMENT_ENV_VAR, + loadGatewayManagementDeclaration, + parseGatewayManagementDeclaration, +} from "./gateway-management"; + +function externalDeclaration(overrides: Record = {}) { + return { + version: 1, + mode: "externally-supervised", + endpoint: "http://127.0.0.1:8080", + stateDir: "/var/lib/openshell/gateway", + supervisor: { + kind: "systemd-system", + serviceName: "openshell-gateway.service", + execPath: "/usr/local/bin/openshell-gateway", + }, + ...overrides, + }; +} + +describe("gateway management declaration", () => { + it("accepts an externally supervised declaration (#6576)", () => { + const result = parseGatewayManagementDeclaration( + externalDeclaration({ requiredCapabilities: ["sandbox.create", "gateway.health"] }), + ); + + expect(result).toEqual({ + ok: true, + declaration: { + version: 1, + mode: "externally-supervised", + endpoint: "http://127.0.0.1:8080", + stateDir: "/var/lib/openshell/gateway", + supervisor: { + kind: "systemd-system", + serviceName: "openshell-gateway.service", + execPath: "/usr/local/bin/openshell-gateway", + }, + requiredCapabilities: ["sandbox.create", "gateway.health"], + }, + }); + }); + + it("accepts a nemoclaw-managed declaration without a supervisor (#6576)", () => { + const result = parseGatewayManagementDeclaration({ + version: 1, + mode: "nemoclaw-managed", + }); + + expect(result).toEqual({ + ok: true, + declaration: { + version: 1, + mode: "nemoclaw-managed", + endpoint: null, + stateDir: null, + supervisor: null, + requiredCapabilities: [], + }, + }); + }); + + it("rejects external-only fields on a nemoclaw-managed declaration (#6576)", () => { + const result = parseGatewayManagementDeclaration({ + version: 1, + mode: "nemoclaw-managed", + endpoint: "http://127.0.0.1:31818", + stateDir: "/home/user/.local/state/nemoclaw/gateway", + }); + + expect(result.ok === false && result.reason).toMatch(/must not be declared/); + }); + + it("rejects an unknown contract version instead of treating it as absent (#6576)", () => { + const result = parseGatewayManagementDeclaration(externalDeclaration({ version: 2 })); + + expect(result).toMatchObject({ ok: false }); + expect(result.ok === false && result.reason).toMatch(/unsupported gateway-management contract/); + }); + + it("rejects a supervisor on a nemoclaw-managed declaration (#6576)", () => { + const result = parseGatewayManagementDeclaration({ + version: 1, + mode: "nemoclaw-managed", + supervisor: externalDeclaration().supervisor, + }); + + expect(result.ok === false && result.reason).toMatch(/supervisor must not be declared/); + }); + + it("requires a supervisor when external supervision is declared (#6576)", () => { + const result = parseGatewayManagementDeclaration( + externalDeclaration({ supervisor: undefined }), + ); + + expect(result.ok === false && result.reason).toMatch(/supervisor is required/); + }); + + it("requires supervisor.execPath so a listener can be checked against it (#6576)", () => { + const result = parseGatewayManagementDeclaration( + externalDeclaration({ + supervisor: { kind: "systemd-system", serviceName: "openshell-gateway.service" }, + }), + ); + + expect(result.ok === false && result.reason).toMatch( + /supervisor\.execPath must be a non-empty/, + ); + }); + + it("rejects a supervisor unit that could be parsed as a systemctl option (#6576)", () => { + const result = parseGatewayManagementDeclaration( + externalDeclaration({ + supervisor: { + kind: "systemd-system", + serviceName: "--no-pager.service", + execPath: "/usr/local/bin/openshell-gateway", + }, + }), + ); + + expect(result.ok === false && result.reason).toMatch(/must name one systemd/); + }); + + it("rejects unknown fields so credentials cannot ride the contract (#6576)", () => { + const result = parseGatewayManagementDeclaration( + externalDeclaration({ apiToken: "sk-live-not-a-real-token" }), + ); + + expect(result.ok === false && result.reason).toMatch( + /unknown declaration field\(s\): apiToken/, + ); + }); + + it("rejects an endpoint that embeds credentials (#6576)", () => { + const result = parseGatewayManagementDeclaration( + externalDeclaration({ endpoint: "http://user:secret@127.0.0.1:8080" }), + ); + + expect(result.ok === false && result.reason).toMatch(/must not embed credentials/); + }); + + it("rejects an endpoint carrying a query string (#6576)", () => { + const result = parseGatewayManagementDeclaration( + externalDeclaration({ endpoint: "http://127.0.0.1:8080?token=abc" }), + ); + + expect(result.ok === false && result.reason).toMatch(/query string/); + }); + + it.each([ + ["a remote host", "http://gateway.example.com:8080"], + ["a DNS-resolved localhost name", "http://localhost:8080"], + ["a cloud metadata address", "http://169.254.169.254:8080"], + ["a link-local address", "http://169.254.1.1:8080"], + ["a non-loopback private address", "http://10.0.0.5:8080"], + ])("rejects an endpoint pointing at %s, which onboarding would otherwise request (#6576)", (_label, endpoint) => { + const result = parseGatewayManagementDeclaration(externalDeclaration({ endpoint })); + + expect(result.ok === false && result.reason).toMatch(/not a supported local gateway origin/); + }); + + it("accepts only numeric loopback endpoint hosts (#6576)", () => { + for (const endpoint of ["http://127.0.0.1:8080", "http://[::1]:8080"]) { + expect(parseGatewayManagementDeclaration(externalDeclaration({ endpoint }))).toMatchObject({ + ok: true, + }); + } + }); + + it("rejects a capability this build does not provide (#6576)", () => { + const result = parseGatewayManagementDeclaration( + externalDeclaration({ requiredCapabilities: ["gateway.teleport"] }), + ); + + expect(result.ok === false && result.reason).toMatch(/unsupported capability/); + }); + + it("rejects a relative state directory (#6576)", () => { + const result = parseGatewayManagementDeclaration( + externalDeclaration({ stateDir: "relative/state" }), + ); + + expect(result.ok === false && result.reason).toMatch(/absolute path/); + }); +}); + +describe("gateway management declaration loading", () => { + it("returns no declaration when nothing is configured (#6576)", () => { + expect(loadGatewayManagementDeclaration({ env: {} })).toEqual({ + ok: true, + declaration: null, + source: null, + }); + }); + + it("loads a declaration from the environment-configured file (#6576)", () => { + const result = loadGatewayManagementDeclaration({ + env: { [GATEWAY_MANAGEMENT_ENV_VAR]: "/etc/nemoclaw/gateway.json" }, + readFile: () => JSON.stringify(externalDeclaration()), + }); + + expect(result).toMatchObject({ ok: true, source: "file" }); + expect(result.ok === true && result.declaration?.mode).toBe("externally-supervised"); + }); + + it("fails closed when the configured file is unreadable (#6576)", () => { + const result = loadGatewayManagementDeclaration({ + env: { [GATEWAY_MANAGEMENT_ENV_VAR]: "/etc/nemoclaw/gateway.json" }, + readFile: () => { + throw new Error("ENOENT: no such file"); + }, + }); + + expect(result.ok === false && result.reason).toMatch(/could not be read/); + }); + + it("fails closed on malformed JSON rather than self-managing the gateway (#6576)", () => { + const result = loadGatewayManagementDeclaration({ + env: { [GATEWAY_MANAGEMENT_ENV_VAR]: "/etc/nemoclaw/gateway.json" }, + readFile: () => "{ not json", + }); + + expect(result.ok === false && result.reason).toMatch(/not valid JSON/); + }); + + it("prefers an in-process profile declaration over the environment file (#6576)", () => { + const result = loadGatewayManagementDeclaration({ + declaration: externalDeclaration(), + env: { [GATEWAY_MANAGEMENT_ENV_VAR]: "/etc/nemoclaw/gateway.json" }, + readFile: () => { + throw new Error("must not read the file when a profile declares the mode"); + }, + }); + + expect(result).toMatchObject({ ok: true, source: "profile" }); + }); +}); + +describe("supported supervisor kinds (#6576)", () => { + it("accepts the systemd kinds it can bind a listener to", () => { + for (const kind of ["systemd-system", "systemd-user"]) { + expect( + parseGatewayManagementDeclaration( + externalDeclaration({ + supervisor: { + kind, + serviceName: "openshell-gateway.service", + execPath: "/usr/local/bin/openshell-gateway", + }, + }), + ), + ).toMatchObject({ ok: true }); + } + }); + + it("rejects the opaque 'external' kind, which could never attach", () => { + const result = parseGatewayManagementDeclaration( + externalDeclaration({ + supervisor: { + kind: "external", + serviceName: "some-supervisor", + execPath: "/usr/local/bin/openshell-gateway", + }, + }), + ); + + expect(result.ok === false && result.reason).toMatch(/supervisor\.kind must be one of/); + }); +}); diff --git a/src/lib/onboard/gateway-management.ts b/src/lib/onboard/gateway-management.ts new file mode 100644 index 00000000000..45f8942ba09 --- /dev/null +++ b/src/lib/onboard/gateway-management.ts @@ -0,0 +1,378 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Public, versioned contract describing who owns the OpenShell gateway + * lifecycle on this host (#6576). + * + * Platform profiles (#6404) and downstream automation declare a gateway + * management mode; core NemoClaw consumes only what is declared here. The + * declaration is deliberately platform-neutral: it carries an endpoint, a + * state location, an expected service/runtime identity, and the capabilities + * the gateway must provide. It carries no Brev/GCP control-plane concepts and, + * by construction, no credentials — unknown keys are rejected rather than + * ignored, so a secret cannot be smuggled through this surface into persisted + * ownership metadata, diagnostics, or machine events. + * + * Parsing is fail-closed. An unrecognized version, an unknown capability, or a + * malformed field is an error, never a silent downgrade to "no declaration" — + * treating a bad declaration as absent is exactly how a host ends up with two + * gateway authorities. + */ + +import fs from "node:fs"; +import path from "node:path"; + +/** Bump only for a breaking change to the declaration shape. */ +export const GATEWAY_MANAGEMENT_CONTRACT_VERSION = 1; + +/** Environment variable naming a JSON file holding the declaration. */ +export const GATEWAY_MANAGEMENT_ENV_VAR = "NEMOCLAW_GATEWAY_MANAGEMENT"; + +export type GatewayManagementMode = "nemoclaw-managed" | "externally-supervised"; + +/** + * Capabilities a gateway must expose for canonical onboarding to attach to it. + * A declaration requiring a capability NemoClaw does not implement fails closed + * rather than attaching and discovering the gap after provider or sandbox + * effects have already run. + */ +export const SUPPORTED_GATEWAY_CAPABILITIES = [ + "gateway.health", + "sandbox.create", + "sandbox.exec", + "gpu.passthrough", +] as const; + +export type GatewayCapability = (typeof SUPPORTED_GATEWAY_CAPABILITIES)[number]; + +/** How the external supervisor runs the gateway, and how to recognize it. */ +/** + * Supervisor kinds NemoClaw can authoritatively bind a listening PID to. Only + * systemd is supported in v1: an opaque "external" supervisor offers no way to + * prove the listener belongs to it, so declaring it could never attach. The + * contract is versioned, so a proven external-identity mechanism can be added + * later without a breaking change (#6576). + */ +export const SUPPORTED_GATEWAY_SUPERVISOR_KINDS = ["systemd-system", "systemd-user"] as const; + +export type GatewaySupervisorKind = (typeof SUPPORTED_GATEWAY_SUPERVISOR_KINDS)[number]; + +export interface GatewaySupervisorDeclaration { + kind: GatewaySupervisorKind; + /** Unit (or equivalent) name the platform supervisor manages. */ + serviceName: string; + /** Absolute path of the gateway executable the supervisor runs. Required. */ + execPath: string; +} + +export interface GatewayManagementDeclaration { + version: typeof GATEWAY_MANAGEMENT_CONTRACT_VERSION; + mode: GatewayManagementMode; + /** Required only when an external supervisor owns the gateway. */ + endpoint: string | null; + /** External gateway state root containing the client TLS bundle. */ + stateDir: string | null; + /** Required for `externally-supervised`; must be absent for `nemoclaw-managed`. */ + supervisor: GatewaySupervisorDeclaration | null; + requiredCapabilities: readonly GatewayCapability[]; +} + +export type GatewayManagementParseResult = + | { ok: true; declaration: GatewayManagementDeclaration } + | { ok: false; reason: string }; + +const DECLARATION_KEYS = new Set([ + "version", + "mode", + "endpoint", + "stateDir", + "supervisor", + "requiredCapabilities", +]); + +const SUPERVISOR_KEYS = new Set(["kind", "serviceName", "execPath"]); + +const SUPERVISOR_KINDS = new Set(SUPPORTED_GATEWAY_SUPERVISOR_KINDS); + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function rejectUnknownKeys(value: Record, allowed: Set, where: string) { + const unknown = Object.keys(value).filter((key) => !allowed.has(key)); + if (unknown.length > 0) { + // Fail closed rather than ignore: an unknown key is either a newer contract + // this build cannot honor, or an attempt to route data (a credential, a + // control-plane detail) through a surface that gets persisted and emitted. + return `unknown ${where} field(s): ${unknown.sort().join(", ")}`; + } + return null; +} + +function requireNonEmptyString(value: unknown, field: string): string | { error: string } { + if (typeof value !== "string" || value.trim().length === 0) { + return { error: `${field} must be a non-empty string` }; + } + return value.trim(); +} + +/** + * Numeric hosts a declared gateway endpoint may name. The gateway is supervised + * on this machine, so the endpoint is always loopback. DNS names are excluded: + * even `localhost` can be redirected by resolver configuration between parsing + * and the readiness request. + */ +const SUPPORTED_GATEWAY_ENDPOINT_HOSTS = new Set(["127.0.0.1", "[::1]", "::1"]); + +/** + * The endpoint is persisted, emitted in diagnostics, and — for an externally + * supervised gateway — used as the target of a readiness request. So it must + * neither carry a secret nor be able to point NemoClaw at an arbitrary host. + * + * Reject embedded credentials, queries, and fragments; keep it to a bare origin; + * and constrain it to the supported local gateway origins, so a declaration + * cannot direct a request at a remote, link-local, or cloud-metadata address. + */ +function parseEndpoint(value: unknown): string | { error: string } { + const raw = requireNonEmptyString(value, "endpoint"); + if (typeof raw !== "string") return raw; + let url: URL; + try { + url = new URL(raw); + } catch { + return { error: `endpoint is not a valid URL: ${raw}` }; + } + if (url.protocol !== "http:" && url.protocol !== "https:") { + return { error: `endpoint must use http or https, got ${url.protocol.replace(":", "")}` }; + } + if (url.username || url.password) { + return { error: "endpoint must not embed credentials" }; + } + if (url.search || url.hash) { + return { error: "endpoint must not carry a query string or fragment" }; + } + if ((url.pathname && url.pathname !== "/") || raw.includes("@")) { + return { error: "endpoint must be a bare origin (no path)" }; + } + if (!SUPPORTED_GATEWAY_ENDPOINT_HOSTS.has(url.hostname)) { + return { + error: + `endpoint host ${url.hostname} is not a supported local gateway origin; ` + + `the declared gateway is supervised on this machine, so the endpoint must be loopback ` + + `(one of: ${[...SUPPORTED_GATEWAY_ENDPOINT_HOSTS].join(", ")})`, + }; + } + return url.origin; +} + +// Declared paths are kept verbatim rather than normalized: they name locations +// on the host the declaration describes, and normalizing them through the +// running platform's path rules would rewrite a POSIX path on a Windows host. +function parseStateDir(value: unknown): string | { error: string } { + const raw = requireNonEmptyString(value, "stateDir"); + if (typeof raw !== "string") return raw; + if (!path.isAbsolute(raw)) { + return { error: `stateDir must be an absolute path, got ${raw}` }; + } + return raw; +} + +function parseCapabilities(value: unknown): readonly GatewayCapability[] | { error: string } { + if (value === undefined) return []; + if (!Array.isArray(value)) { + return { error: "requiredCapabilities must be an array" }; + } + const supported = new Set(SUPPORTED_GATEWAY_CAPABILITIES); + const capabilities: GatewayCapability[] = []; + for (const entry of value) { + if (typeof entry !== "string" || !supported.has(entry)) { + return { + error: + `unsupported capability ${JSON.stringify(entry)}; ` + + `this NemoClaw build provides: ${SUPPORTED_GATEWAY_CAPABILITIES.join(", ")}`, + }; + } + if (!capabilities.includes(entry as GatewayCapability)) { + capabilities.push(entry as GatewayCapability); + } + } + return capabilities; +} + +function parseSupervisor( + value: unknown, + mode: GatewayManagementMode, +): GatewaySupervisorDeclaration | null | { error: string } { + if (mode === "nemoclaw-managed") { + if (value !== undefined && value !== null) { + return { + error: "supervisor must not be declared for mode nemoclaw-managed", + }; + } + return null; + } + if (!isRecord(value)) { + return { error: "supervisor is required for mode externally-supervised" }; + } + const unknown = rejectUnknownKeys(value, SUPERVISOR_KEYS, "supervisor"); + if (unknown) return { error: unknown }; + + const kind = value.kind; + if (typeof kind !== "string" || !SUPERVISOR_KINDS.has(kind as GatewaySupervisorKind)) { + return { + error: `supervisor.kind must be one of: ${[...SUPERVISOR_KINDS].join(", ")}`, + }; + } + const serviceName = requireNonEmptyString(value.serviceName, "supervisor.serviceName"); + if (typeof serviceName !== "string") return serviceName; + if (!/^[A-Za-z0-9][A-Za-z0-9:_.@-]*\.service$/.test(serviceName)) { + return { error: "supervisor.serviceName must name one systemd .service unit" }; + } + + // execPath is required, not optional: without a declared executable there is + // nothing to check the listening process against, and attachment would accept + // any live local listener that answers the health probe. + const execPath = requireNonEmptyString(value.execPath, "supervisor.execPath"); + if (typeof execPath !== "string") return execPath; + if (!path.isAbsolute(execPath)) { + return { error: `supervisor.execPath must be an absolute path, got ${execPath}` }; + } + + return { kind: kind as GatewaySupervisorKind, serviceName, execPath }; +} + +/** + * Parse and validate a gateway-management declaration. Fail-closed: any problem + * is reported as an error, never coerced into a usable default. + */ +export function parseGatewayManagementDeclaration(raw: unknown): GatewayManagementParseResult { + if (!isRecord(raw)) { + return { ok: false, reason: "declaration must be a JSON object" }; + } + const unknown = rejectUnknownKeys(raw, DECLARATION_KEYS, "declaration"); + if (unknown) return { ok: false, reason: unknown }; + + if (raw.version !== GATEWAY_MANAGEMENT_CONTRACT_VERSION) { + return { + ok: false, + reason: + `unsupported gateway-management contract version ${JSON.stringify(raw.version)}; ` + + `this NemoClaw build supports version ${GATEWAY_MANAGEMENT_CONTRACT_VERSION}`, + }; + } + + const mode = raw.mode; + if (mode !== "nemoclaw-managed" && mode !== "externally-supervised") { + return { + ok: false, + reason: `mode must be nemoclaw-managed or externally-supervised, got ${JSON.stringify(mode)}`, + }; + } + + let endpoint: string | null = null; + let stateDir: string | null = null; + if (mode === "externally-supervised") { + const parsedEndpoint = parseEndpoint(raw.endpoint); + if (typeof parsedEndpoint !== "string") { + return { ok: false, reason: parsedEndpoint.error }; + } + const parsedStateDir = parseStateDir(raw.stateDir); + if (typeof parsedStateDir !== "string") { + return { ok: false, reason: parsedStateDir.error }; + } + endpoint = parsedEndpoint; + stateDir = parsedStateDir; + } else if (raw.endpoint !== undefined || raw.stateDir !== undefined) { + return { + ok: false, + reason: "endpoint and stateDir must not be declared for mode nemoclaw-managed", + }; + } + + const supervisor = parseSupervisor(raw.supervisor, mode); + if (supervisor && "error" in supervisor) return { ok: false, reason: supervisor.error }; + + const requiredCapabilities = parseCapabilities(raw.requiredCapabilities); + if ("error" in requiredCapabilities) { + return { ok: false, reason: requiredCapabilities.error }; + } + + return { + ok: true, + declaration: { + version: GATEWAY_MANAGEMENT_CONTRACT_VERSION, + mode, + endpoint, + stateDir, + supervisor, + requiredCapabilities, + }, + }; +} + +export interface LoadGatewayManagementOptions { + env?: NodeJS.ProcessEnv; + readFile?: (filePath: string) => string; + /** Declaration supplied in-process (e.g. by a platform profile) instead of via file. */ + declaration?: unknown; +} + +export type GatewayManagementLoadResult = + | { + ok: true; + declaration: GatewayManagementDeclaration | null; + source: "profile" | "file" | null; + } + | { ok: false; reason: string }; + +/** + * Resolve the declaration for this run. An in-process declaration (a platform + * profile) wins over the environment file; absent both, the caller gets `null` + * and keeps NemoClaw's historical self-managed behavior. + */ +export function loadGatewayManagementDeclaration( + options: LoadGatewayManagementOptions = {}, +): GatewayManagementLoadResult { + if (options.declaration !== undefined && options.declaration !== null) { + const parsed = parseGatewayManagementDeclaration(options.declaration); + if (!parsed.ok) return { ok: false, reason: `gateway management profile: ${parsed.reason}` }; + return { ok: true, declaration: parsed.declaration, source: "profile" }; + } + + const env = options.env ?? process.env; + const configuredPath = env[GATEWAY_MANAGEMENT_ENV_VAR]?.trim(); + if (!configuredPath) return { ok: true, declaration: null, source: null }; + + const readFile = options.readFile ?? ((filePath: string) => fs.readFileSync(filePath, "utf-8")); + let contents: string; + try { + contents = readFile(path.resolve(configuredPath)); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + return { + ok: false, + reason: `${GATEWAY_MANAGEMENT_ENV_VAR}=${configuredPath} could not be read: ${detail}`, + }; + } + + let raw: unknown; + try { + raw = JSON.parse(contents); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + return { + ok: false, + reason: `${GATEWAY_MANAGEMENT_ENV_VAR}=${configuredPath} is not valid JSON: ${detail}`, + }; + } + + const parsed = parseGatewayManagementDeclaration(raw); + if (!parsed.ok) { + return { + ok: false, + reason: `${GATEWAY_MANAGEMENT_ENV_VAR}=${configuredPath}: ${parsed.reason}`, + }; + } + return { ok: true, declaration: parsed.declaration, source: "file" }; +} diff --git a/src/lib/onboard/gateway-ownership.test.ts b/src/lib/onboard/gateway-ownership.test.ts new file mode 100644 index 00000000000..492980d7b0f --- /dev/null +++ b/src/lib/onboard/gateway-ownership.test.ts @@ -0,0 +1,322 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import type { GatewayManagementDeclaration } from "./gateway-management"; +import { + assertGatewayEffectAllowed, + cgroupBelongsToUnit, + describeGatewayOwner, + evaluateGatewayAttachment, + type GatewayAttachmentProbe, + type GatewayLifecycleEffect, + GatewayOwnershipError, + resolveGatewayOwner, +} from "./gateway-ownership"; + +const externalDeclaration: GatewayManagementDeclaration = { + version: 1, + mode: "externally-supervised", + endpoint: "http://127.0.0.1:8080", + stateDir: "/var/lib/openshell/gateway", + supervisor: { + kind: "systemd-system", + serviceName: "openshell-gateway.service", + execPath: "/usr/local/bin/openshell-gateway", + }, + requiredCapabilities: ["sandbox.create"], +}; + +const externalOwner = resolveGatewayOwner({ + gatewayName: "nemoclaw", + gatewayPort: 8080, + declaration: externalDeclaration, + hasPackagedService: false, +}); + +function probe(overrides: Partial = {}): GatewayAttachmentProbe { + return { + gatewayPort: 8080, + httpReady: true, + portOccupied: true, + listenerPids: [4242], + listenerScanComplete: true, + supervisorActive: true, + listenerExecPath: "/usr/local/bin/openshell-gateway", + listenerSupervisorMatch: true, + ...overrides, + }; +} + +describe("gateway owner resolution", () => { + it("treats a declaration as the lifecycle authority (#6576)", () => { + expect(externalOwner).toEqual({ + gatewayName: "nemoclaw", + gatewayPort: 8080, + mode: "externally-supervised", + source: "declared", + endpoint: "http://127.0.0.1:8080", + stateDir: "/var/lib/openshell/gateway", + supervisor: externalDeclaration.supervisor, + requiredCapabilities: ["sandbox.create"], + }); + }); + + it("owns the packaged service when nothing is declared and it is installed (#6576)", () => { + expect( + resolveGatewayOwner({ + gatewayName: "nemoclaw", + gatewayPort: 31818, + declaration: null, + hasPackagedService: true, + }), + ).toMatchObject({ + mode: "nemoclaw-managed", + source: "packaged-service", + }); + }); + + it("falls back to standalone self-management only when nothing is declared (#6576)", () => { + expect( + resolveGatewayOwner({ + gatewayName: "nemoclaw", + gatewayPort: 31818, + declaration: null, + hasPackagedService: false, + }), + ).toMatchObject({ + mode: "nemoclaw-managed", + source: "standalone", + }); + }); +}); + +describe("gateway lifecycle effect enforcement", () => { + const effects: GatewayLifecycleEffect[] = [ + "start", + "stop", + "restart", + "destroy", + "replace", + "standalone-fallback", + ]; + + it.each(effects)("refuses to %s an externally supervised gateway (#6576)", (effect) => { + expect(() => assertGatewayEffectAllowed(externalOwner, effect)).toThrow(GatewayOwnershipError); + try { + assertGatewayEffectAllowed(externalOwner, effect); + } catch (error) { + expect((error as GatewayOwnershipError).code).toBe("external_supervision_forbids_effect"); + expect((error as GatewayOwnershipError).message).toContain("openshell-gateway.service"); + } + }); + + it.each(effects)("permits %s when NemoClaw owns the lifecycle (#6576)", (effect) => { + const owner = resolveGatewayOwner({ + gatewayName: "nemoclaw", + gatewayPort: 31818, + declaration: null, + hasPackagedService: true, + }); + + expect(() => assertGatewayEffectAllowed(owner, effect)).not.toThrow(); + }); +}); + +describe("externally supervised gateway attachment", () => { + it("attaches to a healthy gateway held by the declared supervisor (#6576)", () => { + expect(evaluateGatewayAttachment(externalOwner, probe())).toEqual({ + ok: true, + owner: externalOwner, + }); + }); + + it("rejects a declaration whose endpoint port is not the one this process operates (#6576)", () => { + const result = evaluateGatewayAttachment(externalOwner, probe({ gatewayPort: 31818 })); + + expect(result).toMatchObject({ ok: false, code: "endpoint_port_mismatch" }); + expect(result.ok === false && result.message).toMatch(/port 8080.*port 31818/s); + }); + + it("rejects a required capability this build does not provide (#6576)", () => { + const owner = { + ...externalOwner, + requiredCapabilities: ["gateway.teleport"], + } as unknown as typeof externalOwner; + + expect(evaluateGatewayAttachment(owner, probe())).toMatchObject({ + ok: false, + code: "capability_unsupported", + }); + }); + + it("fails when a competing listener also holds the port (#6576)", () => { + const result = evaluateGatewayAttachment(externalOwner, probe({ listenerPids: [4242, 4243] })); + + expect(result).toMatchObject({ ok: false, code: "multiple_owners" }); + }); + + it("fails when the declared supervisor is inactive rather than starting the gateway (#6576)", () => { + const result = evaluateGatewayAttachment(externalOwner, probe({ supervisorActive: false })); + + expect(result).toMatchObject({ ok: false, code: "supervisor_inactive" }); + expect(result.ok === false && result.message).toMatch( + /does not start an externally supervised/, + ); + }); + + it("fails instead of launching a gateway when nothing holds the port (#6576)", () => { + const result = evaluateGatewayAttachment( + externalOwner, + probe({ portOccupied: false, listenerPids: [], httpReady: false }), + ); + + expect(result).toMatchObject({ ok: false, code: "gateway_unreachable" }); + expect(result.ok === false && result.message).toMatch(/will not start a competing gateway/); + }); + + it("fails when an unrecognized process holds the gateway port (#6576)", () => { + const result = evaluateGatewayAttachment( + externalOwner, + probe({ listenerPids: [], listenerExecPath: null }), + ); + + expect(result).toMatchObject({ ok: false, code: "unknown_listener" }); + }); + + it("fails when the listener set cannot be fully enumerated (#6576)", () => { + const result = evaluateGatewayAttachment(externalOwner, probe({ listenerScanComplete: false })); + + expect(result).toMatchObject({ ok: false, code: "unknown_listener" }); + }); + + it("rejects a same-binary process that is not part of the declared unit (#6576)", () => { + // The impostor answers the health probe and runs the declared executable, + // but its PID is not in the unit's cgroup — the exact gap an exec-path match + // alone would miss. + const result = evaluateGatewayAttachment( + externalOwner, + probe({ listenerSupervisorMatch: false }), + ); + + expect(result).toMatchObject({ ok: false, code: "identity_mismatch" }); + expect(result.ok === false && result.message).toMatch(/not part of openshell-gateway\.service/); + }); + + it("fails closed when the listener cannot be bound to the declared unit (#6576)", () => { + const result = evaluateGatewayAttachment( + externalOwner, + probe({ listenerSupervisorMatch: null }), + ); + + expect(result).toMatchObject({ ok: false, code: "unknown_listener" }); + expect(result.ok === false && result.message).toMatch(/could not confirm/); + }); + + it("fails when the running gateway is not the declared executable (#6576)", () => { + const result = evaluateGatewayAttachment( + externalOwner, + probe({ listenerExecPath: "/opt/brev/bin/openshell-gateway" }), + ); + + expect(result).toMatchObject({ ok: false, code: "identity_mismatch" }); + }); + + it("fails when the listener identity cannot be verified against the declaration (#6576)", () => { + const result = evaluateGatewayAttachment(externalOwner, probe({ listenerExecPath: null })); + + expect(result).toMatchObject({ ok: false, code: "unknown_listener" }); + }); + + it("fails when the supervised gateway does not answer a health check (#6576)", () => { + const result = evaluateGatewayAttachment(externalOwner, probe({ httpReady: false })); + + expect(result).toMatchObject({ ok: false, code: "gateway_unreachable" }); + expect(result.ok === false && result.message).toMatch(/will not replace it/); + }); + + it("does not gate a NemoClaw-managed gateway on attachment checks (#6576)", () => { + const owner = resolveGatewayOwner({ + gatewayName: "nemoclaw", + gatewayPort: 31818, + declaration: null, + hasPackagedService: true, + }); + + expect( + evaluateGatewayAttachment(owner, probe({ portOccupied: false, listenerPids: [] })), + ).toMatchObject({ ok: true }); + }); +}); + +describe("gateway owner diagnostics", () => { + it("reports the owner identity without exposing credentials (#6576)", () => { + expect(describeGatewayOwner(externalOwner)).toEqual({ + gatewayName: "nemoclaw", + gatewayPort: 8080, + mode: "externally-supervised", + source: "declared", + endpoint: "http://127.0.0.1:8080/", + supervisor: { + kind: "systemd-system", + serviceName: "openshell-gateway.service", + execPath: "/usr/local/bin/openshell-gateway", + }, + requiredCapabilities: ["sandbox.create"], + }); + }); + + it("omits an endpoint that was never declared (#6576)", () => { + const owner = resolveGatewayOwner({ + gatewayName: "nemoclaw", + gatewayPort: 31818, + declaration: null, + hasPackagedService: false, + }); + + expect(describeGatewayOwner(owner)).toMatchObject({ + mode: "nemoclaw-managed", + source: "standalone", + endpoint: null, + supervisor: null, + }); + }); +}); + +describe("cgroupBelongsToUnit", () => { + const UNIT = "openshell-gateway.service"; + + it("matches a cgroup v2 process in the unit's system slice (#6576)", () => { + expect(cgroupBelongsToUnit(`0::/system.slice/${UNIT}\n`, UNIT)).toBe(true); + }); + + it("matches a cgroup v1 process listed under the unit (#6576)", () => { + const v1 = [ + "12:pids:/system.slice/openshell-gateway.service", + "0:name=systemd:/system.slice/openshell-gateway.service", + ].join("\n"); + expect(cgroupBelongsToUnit(v1, UNIT)).toBe(true); + }); + + it("matches a user-manager unit path (#6576)", () => { + expect( + cgroupBelongsToUnit( + `0::/user.slice/user-1000.slice/user@1000.service/app.slice/${UNIT}`, + UNIT, + ), + ).toBe(true); + }); + + it("rejects a same-binary process in a login session scope (#6576)", () => { + expect(cgroupBelongsToUnit("0::/user.slice/user-1000.slice/session-3.scope", UNIT)).toBe(false); + }); + + it("rejects a different unit that merely shares a prefix (#6576)", () => { + expect(cgroupBelongsToUnit("0::/system.slice/openshell-gateway.service.d", UNIT)).toBe(false); + }); + + it("rejects empty or unreadable cgroup text (#6576)", () => { + expect(cgroupBelongsToUnit("", UNIT)).toBe(false); + }); +}); diff --git a/src/lib/onboard/gateway-ownership.ts b/src/lib/onboard/gateway-ownership.ts new file mode 100644 index 00000000000..e9740d2ae37 --- /dev/null +++ b/src/lib/onboard/gateway-ownership.ts @@ -0,0 +1,470 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Gateway lifecycle authority (#6576). + * + * At every point exactly one component owns the OpenShell gateway lifecycle. + * This module resolves that owner, decides which lifecycle effects the owner + * permits, and evaluates whether NemoClaw may attach to a gateway it does not + * own — all as pure functions, so the decision can be made and tested before + * any provider, policy, sandbox, or registry effect runs. + * + * The failure this prevents: when an external supervisor (for example a + * platform image's own gateway service) owns the process and NemoClaw silently + * falls back to launching a standalone gateway, both layers believe they own + * the same port. One replaces the live listener, the other restarts it, and the + * host is left with a persistent bind conflict and no authoritative owner. + * Externally supervised mode therefore never permits a standalone fallback: it + * attaches and validates, or it fails. + * + * Source boundary: OpenShell exposes no gateway capability-discovery endpoint, + * so `requiredCapabilities` is enforced against what this NemoClaw build + * implements rather than interrogated from the running gateway. That still + * fails closed when a newer platform profile declares a capability an older + * NemoClaw cannot honor, which is the case this contract must not let through. + * Gateway-side discovery can replace this check when OpenShell reports it. + */ + +import type { JsonObject } from "../core/json-types"; +import { redactUrl } from "../security/redact"; +import { + type GatewayCapability, + type GatewayManagementDeclaration, + type GatewayManagementMode, + type GatewaySupervisorDeclaration, + SUPPORTED_GATEWAY_CAPABILITIES, +} from "./gateway-management"; + +/** + * Port named by a declared endpoint, defaulting to the scheme's port when the + * URL omits one. Returns null when no endpoint was declared. + */ +function declaredEndpointPort(endpoint: string | null): number | null { + if (!endpoint) return null; + const url = new URL(endpoint); + if (url.port) return Number(url.port); + return url.protocol === "https:" ? 443 : 80; +} + +/** + * How the owner was established. `declared` means a profile or operator + * declaration named it; the other two are NemoClaw's own detected defaults when + * nothing was declared. + */ +export type GatewayOwnerSource = "declared" | "packaged-service" | "standalone"; + +export interface GatewayOwner { + /** Canonical OpenShell registration selected for every downstream command. */ + gatewayName: string; + /** Host port paired with `gatewayName` by the current per-gateway binding. */ + gatewayPort: number; + mode: GatewayManagementMode; + source: GatewayOwnerSource; + /** Present only when declared. */ + endpoint: string | null; + /** Present only when declared. */ + stateDir: string | null; + /** Present only for an externally supervised owner. */ + supervisor: GatewaySupervisorDeclaration | null; + requiredCapabilities: readonly GatewayCapability[]; +} + +/** Lifecycle effects whose legality depends on who owns the gateway. */ +export type GatewayLifecycleEffect = + | "start" + | "stop" + | "restart" + | "destroy" + | "replace" + | "standalone-fallback"; + +export type GatewayOwnershipFailureCode = + | "external_supervision_forbids_effect" + | "gateway_unreachable" + | "supervisor_inactive" + | "identity_mismatch" + | "unknown_listener" + | "multiple_owners" + | "endpoint_port_mismatch" + | "gateway_registration_failed" + | "capability_unsupported"; + +export class GatewayOwnershipError extends Error { + readonly code: GatewayOwnershipFailureCode; + readonly owner: GatewayOwner; + + constructor(code: GatewayOwnershipFailureCode, message: string, owner: GatewayOwner) { + super(message); + this.name = "GatewayOwnershipError"; + this.code = code; + this.owner = owner; + } +} + +export interface ResolveGatewayOwnerOptions { + gatewayName: string; + gatewayPort: number; + /** Validated declaration, or null when nothing was declared. */ + declaration: GatewayManagementDeclaration | null; + /** Whether the packaged canonical gateway user service is installed. */ + hasPackagedService: boolean; +} + +/** + * Establish the single lifecycle authority for this run. + * + * A declaration is authoritative. Without one, NemoClaw keeps its historical + * behavior — prefer the packaged canonical service, otherwise manage a + * standalone gateway — but the owner is now explicit rather than implied by + * whichever code path happened to run first. + */ +export function resolveGatewayOwner({ + gatewayName, + gatewayPort, + declaration, + hasPackagedService, +}: ResolveGatewayOwnerOptions): GatewayOwner { + if (declaration) { + return { + gatewayName, + gatewayPort, + mode: declaration.mode, + source: "declared", + endpoint: declaration.endpoint, + stateDir: declaration.stateDir, + supervisor: declaration.supervisor, + requiredCapabilities: declaration.requiredCapabilities, + }; + } + return { + gatewayName, + gatewayPort, + mode: "nemoclaw-managed", + source: hasPackagedService ? "packaged-service" : "standalone", + endpoint: null, + stateDir: null, + supervisor: null, + requiredCapabilities: [], + }; +} + +export function isExternallySupervised(owner: GatewayOwner): boolean { + return owner.mode === "externally-supervised"; +} + +function sameRequiredCapabilities( + a: readonly GatewayCapability[], + b: readonly GatewayCapability[], +): boolean { + const aSet = new Set(a); + const bSet = new Set(b); + return aSet.size === bSet.size && Array.from(aSet).every((capability) => bSet.has(capability)); +} + +/** + * Whether two resolutions describe the same lifecycle authority. Used to keep + * one authority per run: a later resolution that differs is a migration, not a + * silent switch (#6576). + */ +export function sameGatewayOwner(a: GatewayOwner, b: GatewayOwner): boolean { + return ( + a.gatewayName === b.gatewayName && + a.gatewayPort === b.gatewayPort && + a.mode === b.mode && + a.source === b.source && + a.endpoint === b.endpoint && + a.stateDir === b.stateDir && + a.supervisor?.kind === b.supervisor?.kind && + a.supervisor?.serviceName === b.supervisor?.serviceName && + a.supervisor?.execPath === b.supervisor?.execPath && + sameRequiredCapabilities(a.requiredCapabilities, b.requiredCapabilities) + ); +} + +/** Short owner description for error messages; carries no credential material. */ +export function describeGatewayOwnerForError(owner: GatewayOwner): string { + const authority = owner.supervisor?.serviceName ?? owner.source; + return `${owner.gatewayName}@${owner.gatewayPort}:${owner.mode}:${authority}`; +} + +function describeEffect(effect: GatewayLifecycleEffect): string { + switch (effect) { + case "standalone-fallback": + return "start a standalone gateway"; + case "replace": + return "replace the gateway"; + default: + return `${effect} the gateway`; + } +} + +/** + * Guard every lifecycle effect against the declared owner. An externally + * supervised gateway is attached to and validated, never started, stopped, or + * replaced — including via the standalone fallback. + * + * Callers must invoke this before the effect, not after. + */ +export function assertGatewayEffectAllowed( + owner: GatewayOwner, + effect: GatewayLifecycleEffect, +): void { + if (!isExternallySupervised(owner)) return; + const supervisor = owner.supervisor?.serviceName ?? "an external supervisor"; + throw new GatewayOwnershipError( + "external_supervision_forbids_effect", + `Refusing to ${describeEffect(effect)}: the gateway lifecycle is owned by ${supervisor}. ` + + "NemoClaw attaches to an externally supervised gateway but never manages its process. " + + "Fix the supervisor, or declare mode nemoclaw-managed to hand the lifecycle back to NemoClaw.", + owner, + ); +} + +/** Observations about the running gateway, gathered before any effect. */ +export interface GatewayAttachmentProbe { + /** + * The port this NemoClaw process is configured to operate the gateway on. + * Every downstream consumer — sandbox bridge, registry, dashboard forwards — + * is bound to it, so a declaration naming a different port is a + * misconfiguration rather than a second gateway to probe. + */ + gatewayPort: number; + /** The declared endpoint answered a health check. */ + httpReady: boolean; + /** Anything at all holds the gateway port. */ + portOccupied: boolean; + /** Identity-verified gateway processes listening on the port. */ + listenerPids: readonly number[]; + /** False when the listener set could not be authoritatively enumerated. */ + listenerScanComplete: boolean; + /** Whether the declared supervisor unit reports active; null when unprobeable. */ + supervisorActive: boolean | null; + /** + * Executable backing the listening process, when it could be read. A value + * that disagrees with the declared `execPath` means someone else owns the + * port. + */ + listenerExecPath: string | null; + /** + * Whether the listening process is authoritatively bound to the declared + * supervisor unit (for systemd, via cgroup membership). + * + * `true` proves the PID belongs to the declared unit; `false` proves it does + * not (a foreign process, possibly the same binary launched outside the + * unit); `null` means the relationship could not be established. A matching + * executable path is not a substitute — a second process running the same + * binary would pass an exec check while the named unit is merely active. + */ + listenerSupervisorMatch: boolean | null; +} + +/** + * Whether a process cgroup path names the given systemd unit. + * + * `/proc//cgroup` lists the process's cgroup path; a process managed by a + * systemd unit sits under a slice segment named for the unit (for example + * `/system.slice/openshell-gateway.service` or a `...//...` subpath). A + * same-binary process started outside the unit lands in a different cgroup + * (a login session scope, the user slice), so this distinguishes the unit's + * own process from an impostor holding the same port. + */ +export function cgroupBelongsToUnit(cgroupText: string, serviceName: string): boolean { + const unit = serviceName.trim(); + if (!unit) return false; + for (const line of cgroupText.split(/\r?\n/)) { + // cgroup v2: "0::/system.slice/unit.service"; v1: "N:controller:/path". + const cgroupPath = line.slice(line.lastIndexOf(":") + 1).trim(); + if (!cgroupPath) continue; + const segments = cgroupPath.split("/").filter(Boolean); + if (segments.includes(unit)) return true; + } + return false; +} + +export type GatewayAttachmentResult = + | { ok: true; owner: GatewayOwner } + | { ok: false; code: GatewayOwnershipFailureCode; message: string }; + +/** + * Decide whether NemoClaw may attach to an externally supervised gateway. + * + * Every failure here must be raised before provider, policy, sandbox, or + * registry mutation: an ambiguous or multiply owned gateway is precisely the + * state that must not be papered over by starting another one. + */ +export function evaluateGatewayAttachment( + owner: GatewayOwner, + probe: GatewayAttachmentProbe, +): GatewayAttachmentResult { + if (!isExternallySupervised(owner)) { + return { ok: true, owner }; + } + + const supervisorName = owner.supervisor?.serviceName ?? "the declared supervisor"; + + // Configuration errors are reported before any runtime observation: probing + // is only meaningful once we know the declaration describes the gateway this + // process actually operates on. + const declaredPort = declaredEndpointPort(owner.endpoint); + if (declaredPort !== null && declaredPort !== probe.gatewayPort) { + return { + ok: false, + code: "endpoint_port_mismatch", + message: + `The declared gateway endpoint uses port ${declaredPort}, but this NemoClaw process operates ` + + `the gateway on port ${probe.gatewayPort}. Attaching would validate one gateway and then use ` + + `another. Point the declaration at port ${probe.gatewayPort}, or re-run with ` + + `NEMOCLAW_GATEWAY_PORT=${declaredPort}.`, + }; + } + + const unsupported = owner.requiredCapabilities.filter( + (capability) => !SUPPORTED_GATEWAY_CAPABILITIES.includes(capability), + ); + if (unsupported.length > 0) { + return { + ok: false, + code: "capability_unsupported", + message: + `The declaration requires unsupported gateway capabilities ${unsupported.join(", ")}, which this NemoClaw ` + + `build does not provide. Onboarding stops before any effect rather than attaching to a gateway ` + + `it cannot drive as declared.`, + }; + } + + if (probe.listenerPids.length > 1) { + return { + ok: false, + code: "multiple_owners", + message: + `The gateway port has ${probe.listenerPids.length} listening processes, so ownership is ambiguous. ` + + `Leave exactly one process — the one supervised by ${supervisorName} — and retry.`, + }; + } + + if (probe.supervisorActive === false) { + return { + ok: false, + code: "supervisor_inactive", + message: + `${supervisorName} is not active, and NemoClaw does not start an externally supervised gateway. ` + + `Start it through the platform supervisor, then re-run onboarding.`, + }; + } + + if (!probe.portOccupied) { + return { + ok: false, + code: "gateway_unreachable", + message: + `No process is listening on the declared gateway endpoint and ${supervisorName} owns its lifecycle. ` + + `NemoClaw will not start a competing gateway. Bring the supervised gateway up and re-run onboarding.`, + }; + } + + if (probe.listenerPids.length === 0) { + return { + ok: false, + code: "unknown_listener", + message: + `The gateway port is held by a process that is not a recognizable OpenShell gateway. ` + + `NemoClaw will not attach to it or replace it. Identify the process holding the port, ` + + `stop only that process, and let ${supervisorName} own the gateway.`, + }; + } + + if (!probe.listenerScanComplete) { + return { + ok: false, + code: "unknown_listener", + message: + `The set of processes holding the gateway port could not be enumerated, so a second gateway ` + + `cannot be ruled out. NemoClaw fails closed rather than attach to an unproven single owner.`, + }; + } + + // Authoritative identity: the listening PID must belong to the declared + // supervisor unit. This is the check the executable path alone cannot make — + // a second process running the same binary would pass an exec comparison + // while the named unit is only active. Fail closed unless the relationship is + // positively established. + if (probe.listenerSupervisorMatch === false) { + return { + ok: false, + code: "identity_mismatch", + message: + `The process holding the gateway port is not part of ${supervisorName}. ` + + `A different process — possibly the same binary started outside the unit — holds the port. ` + + `NemoClaw will not attach to it. Stop it and let ${supervisorName} own the gateway.`, + }; + } + + if (probe.listenerSupervisorMatch === null) { + return { + ok: false, + code: "unknown_listener", + message: + `NemoClaw could not confirm that the process holding the gateway port belongs to ${supervisorName}. ` + + `It attaches only to a listener authoritatively bound to the declared supervisor, and fails closed otherwise.`, + }; + } + + const declaredExecPath = owner.supervisor?.execPath ?? null; + if (declaredExecPath && probe.listenerExecPath && probe.listenerExecPath !== declaredExecPath) { + return { + ok: false, + code: "identity_mismatch", + message: + `The process holding the gateway port does not match the declared gateway identity ` + + `(running ${probe.listenerExecPath}, declared ${declaredExecPath}). ` + + `NemoClaw will not attach to an unrecognized gateway.`, + }; + } + + if (declaredExecPath && !probe.listenerExecPath) { + return { + ok: false, + code: "unknown_listener", + message: + `The gateway port is held by a process whose identity could not be verified against ` + + `the declared gateway (${declaredExecPath}). NemoClaw will not attach to an unverified listener.`, + }; + } + + if (!probe.httpReady) { + return { + ok: false, + code: "gateway_unreachable", + message: + `The gateway supervised by ${supervisorName} is running but did not answer a health check. ` + + `NemoClaw will not replace it. Check the supervisor and re-run onboarding.`, + }; + } + + return { ok: true, owner }; +} + +/** + * Redacted owner identity for status, read-only diagnostics, and machine + * events. The declaration cannot carry credentials by construction, but the + * endpoint still goes through URL redaction so this stays safe if the contract + * ever widens. + */ +export function describeGatewayOwner(owner: GatewayOwner): JsonObject { + return { + gatewayName: owner.gatewayName, + gatewayPort: owner.gatewayPort, + mode: owner.mode, + source: owner.source, + endpoint: owner.endpoint ? (redactUrl(owner.endpoint) ?? null) : null, + supervisor: owner.supervisor + ? { + kind: owner.supervisor.kind, + serviceName: owner.supervisor.serviceName, + execPath: owner.supervisor.execPath, + } + : null, + requiredCapabilities: [...owner.requiredCapabilities], + }; +} diff --git a/src/lib/onboard/gateway-port-conflict.test.ts b/src/lib/onboard/gateway-port-conflict.test.ts index ce5a6f9ad4d..833e871be4b 100644 --- a/src/lib/onboard/gateway-port-conflict.test.ts +++ b/src/lib/onboard/gateway-port-conflict.test.ts @@ -26,6 +26,7 @@ describe("gateway port conflict", () => { await failFastOnForeignGatewayPortConflict({ gatewayPort: 8080, + externallySupervised: false, checkPortAvailable, getGatewayPortCheckOptions: () => ({ host: "127.0.0.1" }), isDockerDriverGatewayPortListener: () => false, @@ -47,6 +48,7 @@ describe("gateway port conflict", () => { await failFastOnForeignGatewayPortConflict({ gatewayPort: 8080, + externallySupervised: false, checkPortAvailable, getGatewayPortCheckOptions: () => ({ host: "127.0.0.1" }), isDockerDriverGatewayPortListener: () => false, @@ -59,4 +61,21 @@ describe("gateway port conflict", () => { it("keeps Docker-driver gateway listeners on the gateway reuse path", () => { expect(couldBeNemoClawGatewayPortListener(blockedPort("python3"), () => true)).toBe(true); }); + + it("defers an arbitrary declared external executable to exact attachment validation (#6576)", async () => { + const checkPortAvailable = vi.fn().mockResolvedValue(blockedPort("gatewayd")); + const exitProcess = vi.fn(); + + await failFastOnForeignGatewayPortConflict({ + gatewayPort: 8080, + externallySupervised: true, + checkPortAvailable, + getGatewayPortCheckOptions: () => ({ host: "127.0.0.1" }), + isDockerDriverGatewayPortListener: () => false, + exitProcess, + }); + + expect(checkPortAvailable).not.toHaveBeenCalled(); + expect(exitProcess).not.toHaveBeenCalled(); + }); }); diff --git a/src/lib/onboard/gateway-port-conflict.ts b/src/lib/onboard/gateway-port-conflict.ts index abc7501c2ab..e5d7ae50a67 100644 --- a/src/lib/onboard/gateway-port-conflict.ts +++ b/src/lib/onboard/gateway-port-conflict.ts @@ -10,6 +10,7 @@ type DockerGatewayPortListenerClassifier = (portCheck: PortProbeResult) => boole export interface GatewayPortConflictDeps { gatewayPort: number; + externallySupervised: boolean; checkPortAvailable: CheckPortAvailable; getGatewayPortCheckOptions: () => CheckPortOpts; isDockerDriverGatewayPortListener: DockerGatewayPortListenerClassifier; @@ -46,6 +47,7 @@ export function couldBeNemoClawGatewayPortListener( export async function failFastOnForeignGatewayPortConflict({ gatewayPort, + externallySupervised, checkPortAvailable, getGatewayPortCheckOptions, isDockerDriverGatewayPortListener, @@ -53,6 +55,10 @@ export async function failFastOnForeignGatewayPortConflict({ serviceHints = getPortConflictServiceHints(), writeError, }: GatewayPortConflictDeps): Promise { + // External ownership was resolved and bound by the caller before reaching + // this legacy name heuristic. Defer its arbitrary executable to the exact + // downstream supervisor/cgroup/executable/capability/health validation. + if (externallySupervised) return; const portCheck = await checkPortAvailable(gatewayPort, getGatewayPortCheckOptions()); if ( portCheck.ok || diff --git a/src/lib/onboard/gateway-recovery.test.ts b/src/lib/onboard/gateway-recovery.test.ts index 3ca4d02651c..af71abc565f 100644 --- a/src/lib/onboard/gateway-recovery.test.ts +++ b/src/lib/onboard/gateway-recovery.test.ts @@ -283,3 +283,33 @@ describe("gateway recovery", () => { expect(deps.startGatewayWithOptions).not.toHaveBeenCalled(); }); }); + +describe("gateway lifecycle authority during recovery", () => { + it("starts no gateway on any recovery branch when an external supervisor owns it (#6576)", async () => { + const ownershipError = new Error("owned by openshell-gateway.service"); + const deps = createDeps({ + assertGatewayStartAllowed: vi.fn(() => { + throw ownershipError; + }), + }); + + // The cross-port, non-default-name target is the branch that reaches a raw + // `openshell gateway start` without going through startGatewayWithOptions. + await expect( + startGatewayForRecovery({ gatewayName: "other", gatewayPort: 8080 }, deps), + ).rejects.toThrow(ownershipError); + + expect(deps.assertGatewayStartAllowed).toHaveBeenCalledWith(false); + expect(deps.runOpenshell).not.toHaveBeenCalled(); + expect(deps.startGatewayWithOptions).not.toHaveBeenCalled(); + }); + + it("still recovers normally when NemoClaw owns the gateway lifecycle (#6576)", async () => { + const deps = createDeps({ assertGatewayStartAllowed: vi.fn() }); + + await startGatewayForRecovery({}, deps); + + expect(deps.assertGatewayStartAllowed).toHaveBeenCalledWith(false); + expect(deps.startGatewayWithOptions).toHaveBeenCalledOnce(); + }); +}); diff --git a/src/lib/onboard/gateway-recovery.ts b/src/lib/onboard/gateway-recovery.ts index fbec2303249..cbc90b25fe9 100644 --- a/src/lib/onboard/gateway-recovery.ts +++ b/src/lib/onboard/gateway-recovery.ts @@ -48,6 +48,12 @@ type GatewayStartResult = { }; export type GatewayRecoveryDeps = { + /** + * Fail closed before any recovery branch starts a gateway process an external + * supervisor owns (#6576). Optional so existing test harnesses keep working; + * production wiring always supplies it. + */ + assertGatewayStartAllowed?(exitOnFailure: boolean): void; getGatewayClusterContainerState?(gatewayName: string): string; getGatewayStartEnv(): Record; runCaptureOpenshell(args: string[], opts?: RunCaptureOpenshellOptions): string; @@ -261,6 +267,10 @@ export async function startGatewayForRecovery( options: StartGatewayForRecoveryOptions, deps: GatewayRecoveryDeps, ): Promise { + // Guard every recovery branch, including the cross-port / non-default-name + // path below that reaches `openshell gateway start` without going through + // startGatewayWithOptions. + deps.assertGatewayStartAllowed?.(false); const target = resolveGatewayRecoveryTarget(options); const linuxDockerDriverEnabled = ( deps.isLinuxDockerDriverGatewayEnabled ?? isLinuxDockerDriverGatewayEnabled diff --git a/src/lib/onboard/gateway-stale-port-reuse.test.ts b/src/lib/onboard/gateway-stale-port-reuse.test.ts index 5d9da0522c8..35e7f387836 100644 --- a/src/lib/onboard/gateway-stale-port-reuse.test.ts +++ b/src/lib/onboard/gateway-stale-port-reuse.test.ts @@ -71,13 +71,14 @@ describe("classifyGatewayPortReuse", () => { }); const BASE_INPUT = { + kind: "gateway" as const, port: 8080, - gatewayPort: 8080, dashboardPort: 18789, label: "OpenShell gateway", runtimeDisplayName: "NemoClaw", gatewayName: "nemoclaw", gatewayReuseState: "healthy" as GatewayReuseState, + externallySupervised: false, portCheckOptions: undefined, supportsLifecycleCommands: true, }; @@ -99,9 +100,10 @@ describe("applyHealthyPortReuse", () => { expect(result).toBeNull(); }); - it("returns null for ports that are neither the gateway nor dashboard port", async () => { + it("returns null for an unrelated preflight port role", async () => { const result = await applyHealthyPortReuse({ ...BASE_INPUT, + kind: "other", port: 9999, destroyGateway: () => true, runOpenshell: vi.fn(), @@ -111,6 +113,65 @@ describe("applyHealthyPortReuse", () => { expect(result).toBeNull(); }); + it.each([ + "healthy", + "missing", + ] as GatewayReuseState[])("preserves an externally supervised gateway port for downstream attachment from %s state (#6576)", async (gatewayReuseState) => { + const destroyGateway = vi.fn(() => true); + const runOpenshell = vi.fn(); + const checkPortAvailable = vi.fn(); + const verifyGatewayContainerRunning = vi.fn(() => "missing" as GatewayContainerState); + + const result = await applyHealthyPortReuse({ + ...BASE_INPUT, + gatewayReuseState, + externallySupervised: true, + destroyGateway, + runOpenshell, + checkPortAvailable, + verifyGatewayContainerRunning, + }); + + expect(result).toBe("continue"); + expect(verifyGatewayContainerRunning).not.toHaveBeenCalled(); + expect(destroyGateway).not.toHaveBeenCalled(); + expect(runOpenshell).not.toHaveBeenCalled(); + expect(checkPortAvailable).not.toHaveBeenCalled(); + }); + + it.each([ + { gatewayReuseState: "missing", port: 18789, relationship: "distinct" }, + { gatewayReuseState: "healthy", port: 18789, relationship: "distinct" }, + { gatewayReuseState: "missing", port: 8080, relationship: "equal-number" }, + { gatewayReuseState: "healthy", port: 8080, relationship: "equal-number" }, + ] as const)("retains conflict handling for a $relationship external dashboard from $gatewayReuseState state (#6576)", async ({ + gatewayReuseState, + port, + }) => { + const destroyGateway = vi.fn(() => true); + const runOpenshell = vi.fn(); + const checkPortAvailable = vi.fn(); + const verifyGatewayContainerRunning = vi.fn(); + + const result = await applyHealthyPortReuse({ + ...BASE_INPUT, + kind: "dashboard", + port, + gatewayReuseState, + externallySupervised: true, + destroyGateway, + runOpenshell, + checkPortAvailable, + verifyGatewayContainerRunning, + }); + + expect(result).toBeNull(); + expect(verifyGatewayContainerRunning).not.toHaveBeenCalled(); + expect(destroyGateway).not.toHaveBeenCalled(); + expect(runOpenshell).not.toHaveBeenCalled(); + expect(checkPortAvailable).not.toHaveBeenCalled(); + }); + it("cleans up stale metadata and returns downgraded state when the port frees up", async () => { const log = vi.spyOn(console, "log").mockImplementation(() => undefined); const destroyGateway = vi.fn(() => true); @@ -217,6 +278,7 @@ describe("applyHealthyPortReuse", () => { const result = await applyHealthyPortReuse({ ...BASE_INPUT, + kind: "dashboard", port: 18789, label: "NemoClaw dashboard", destroyGateway: () => true, diff --git a/src/lib/onboard/gateway-stale-port-reuse.ts b/src/lib/onboard/gateway-stale-port-reuse.ts index 780bca18312..f938841c31d 100644 --- a/src/lib/onboard/gateway-stale-port-reuse.ts +++ b/src/lib/onboard/gateway-stale-port-reuse.ts @@ -5,6 +5,7 @@ import type { GatewayReuseState } from "../state/gateway"; import { destroyGatewayForReuse } from "./gateway-cleanup"; import type { GatewayContainerState } from "./gateway-container-running"; import type { CheckPortOpts, PortProbeResult } from "./preflight"; +import type { PreflightPortKind } from "./preflight-ports"; export type GatewayPortReuseDecision = "stale" | "reuse" | "skip"; @@ -40,13 +41,14 @@ export function classifyGatewayPortReuse(input: GatewayPortReuseInput): GatewayP } export interface HealthyPortReuseInput { + kind: PreflightPortKind; port: number; - gatewayPort: number; dashboardPort: number; label: string; runtimeDisplayName: string; gatewayName: string; gatewayReuseState: GatewayReuseState; + externallySupervised: boolean; portCheckOptions: CheckPortOpts | undefined; supportsLifecycleCommands: boolean; destroyGateway: () => boolean; @@ -69,14 +71,22 @@ export type HealthyPortReuseOutcome = export async function applyHealthyPortReuse( input: HealthyPortReuseInput, ): Promise { - const { port, gatewayPort, dashboardPort, label, runtimeDisplayName, gatewayName } = input; + const { kind, port, dashboardPort, label, runtimeDisplayName, gatewayName } = input; + if (kind === "other") return null; + // An occupied externally supervised gateway port is expected. Preserve it + // unchanged so the FSM can perform the authoritative listener, supervisor, + // executable, capability, and health checks before any later effect. This + // must run before the healthy-state gate: a first attachment can have no + // NemoClaw gateway metadata yet, while the supervisor already owns the port. + // The explicit entry kind is authoritative: a dashboard entry can have the + // same numeric port and must still retain normal conflict handling. + if (input.externallySupervised) return kind === "gateway" ? "continue" : null; if (input.gatewayReuseState !== "healthy") return null; - if (port !== gatewayPort && port !== dashboardPort) return null; // Only probe the container when lifecycle commands are advertised — for // package-managed gateways without lifecycle commands the openshell-cluster-* // container intentionally doesn't exist and the probe would always report // "missing", which is then ignored by classifyGatewayPortReuse anyway. - if (port === gatewayPort && input.supportsLifecycleCommands) { + if (kind === "gateway" && input.supportsLifecycleCommands) { const decision = classifyGatewayPortReuse({ gatewayReuseState: input.gatewayReuseState, supportsLifecycleCommands: true, diff --git a/src/lib/onboard/machine/handlers/gateway.test.ts b/src/lib/onboard/machine/handlers/gateway.test.ts index f5be1e5a55d..de755406323 100644 --- a/src/lib/onboard/machine/handlers/gateway.test.ts +++ b/src/lib/onboard/machine/handlers/gateway.test.ts @@ -9,9 +9,33 @@ import type { GatewayReuseState } from "../../../state/gateway"; import { createSession, type Session } from "../../../state/onboard-session"; import { flushTrace, resetTraceForTests, TRACE_FILE_ENV, type TraceArtifact } from "../../../trace"; import type { GatewayContainerState } from "../../gateway-container-running"; +import { + type GatewayAttachmentProbe, + type GatewayOwner, + GatewayOwnershipError, + resolveGatewayOwner, +} from "../../gateway-ownership"; import { ONBOARD_TRACE_PHASE_NAMES } from "../../tracing"; import { type GatewayStateOptions, handleGatewayState } from "./gateway"; +const EXTERNAL_OWNER: GatewayOwner = resolveGatewayOwner({ + gatewayName: "nemoclaw", + gatewayPort: 8080, + declaration: { + version: 1, + mode: "externally-supervised", + endpoint: "http://127.0.0.1:8080", + stateDir: "/var/lib/openshell/gateway", + supervisor: { + kind: "systemd-system", + serviceName: "openshell-gateway.service", + execPath: "/usr/local/bin/openshell-gateway", + }, + requiredCapabilities: [], + }, + hasPackagedService: false, +}); + type Gpu = { type: string } | null; function createDeps(overrides: Partial["deps"]> = {}) { @@ -39,10 +63,35 @@ function createDeps(overrides: Partial["deps"]> = {}) { exit: vi.fn((code: number): never => { throw new Error(`exit ${code}`); }), + resolveOwner: vi.fn( + (): GatewayOwner => + resolveGatewayOwner({ + gatewayName: "nemoclaw", + gatewayPort: 8080, + declaration: null, + hasPackagedService: false, + }), + ), + attachGateway: vi.fn(), + probeAttachment: vi.fn( + async (): Promise => ({ + gatewayPort: 8080, + httpReady: true, + portOccupied: true, + listenerPids: [4242], + listenerScanComplete: true, + supervisorActive: true, + listenerExecPath: "/usr/local/bin/openshell-gateway", + listenerSupervisorMatch: true, + }), + ), }; return { calls, deps: { + resolveGatewayOwner: calls.resolveOwner, + attachGateway: calls.attachGateway, + probeGatewayAttachment: calls.probeAttachment, refreshDockerDriverGatewayReuseState: calls.refresh, gatewayCliSupportsLifecycleCommands: calls.lifecycle, verifyGatewayContainerRunning: calls.verifyContainer, @@ -130,7 +179,19 @@ describe("handleGatewayState", () => { next: "provider_selection", transitionKind: "advance", updates: undefined, - metadata: { state: "gateway", gatewayReuseState: "missing" }, + metadata: { + state: "gateway", + gatewayReuseState: "missing", + gatewayOwner: { + gatewayName: "nemoclaw", + gatewayPort: 8080, + mode: "nemoclaw-managed", + source: "standalone", + endpoint: null, + supervisor: null, + requiredCapabilities: [], + }, + }, }); }); @@ -400,3 +461,109 @@ describe("handleGatewayState", () => { expect(result.gatewayReuseState).toBe("missing"); }); }); + +describe("externally supervised gateway lifecycle authority", () => { + function externalDeps(probe: Partial = {}) { + const { calls, deps } = createDeps(); + calls.resolveOwner.mockReturnValue(EXTERNAL_OWNER); + calls.probeAttachment.mockResolvedValue({ + gatewayPort: 8080, + httpReady: true, + portOccupied: true, + listenerPids: [4242], + listenerScanComplete: true, + supervisorActive: true, + listenerExecPath: "/usr/local/bin/openshell-gateway", + listenerSupervisorMatch: true, + ...probe, + }); + return { calls, deps }; + } + + it("attaches to the supervised gateway without running any lifecycle effect (#6576)", async () => { + const order: string[] = []; + const { calls, deps } = externalDeps(); + calls.attachGateway.mockImplementation(() => order.push("attach")); + calls.complete.mockImplementation(async () => { + order.push("complete"); + return createSession(); + }); + + const result = await handleGatewayState(baseOptions(deps, "missing")); + + expect(calls.startGateway).not.toHaveBeenCalled(); + expect(calls.attachGateway).toHaveBeenCalledWith(EXTERNAL_OWNER); + expect(calls.destroy).not.toHaveBeenCalled(); + expect(calls.destroyForReuse).not.toHaveBeenCalled(); + expect(calls.retireLegacy).not.toHaveBeenCalled(); + expect(calls.complete).toHaveBeenCalledWith("gateway"); + expect(order).toEqual(["attach", "complete"]); + expect(result.stateResult).toMatchObject({ + metadata: { gatewayOwner: { mode: "externally-supervised", source: "declared" } }, + }); + }); + + it("does not cross the provider-mutation boundary when exact registration fails (#6576)", async () => { + const { calls, deps } = externalDeps(); + calls.attachGateway.mockImplementation(() => { + throw new GatewayOwnershipError( + "gateway_registration_failed", + "registration failed", + EXTERNAL_OWNER, + ); + }); + + await expect(handleGatewayState(baseOptions(deps, "missing"))).rejects.toMatchObject({ + code: "gateway_registration_failed", + }); + expect(calls.recordSkip).not.toHaveBeenCalled(); + expect(calls.complete).not.toHaveBeenCalled(); + }); + + it("never falls back to a standalone gateway when the supervisor is inactive (#6576)", async () => { + const { calls, deps } = externalDeps({ supervisorActive: false }); + + await expect(handleGatewayState(baseOptions(deps, "missing"))).rejects.toThrow( + GatewayOwnershipError, + ); + expect(calls.startGateway).not.toHaveBeenCalled(); + expect(calls.destroy).not.toHaveBeenCalled(); + }); + + it("fails before any effect when a competing listener holds the port (#6576)", async () => { + const { calls, deps } = externalDeps({ listenerPids: [4242, 4243] }); + + await expect(handleGatewayState(baseOptions(deps, "healthy"))).rejects.toMatchObject({ + code: "multiple_owners", + }); + expect(calls.startGateway).not.toHaveBeenCalled(); + expect(calls.destroyForReuse).not.toHaveBeenCalled(); + }); + + it("fails before any effect when the running gateway is not the declared one (#6576)", async () => { + const { calls, deps } = externalDeps({ listenerExecPath: "/opt/other/openshell-gateway" }); + + await expect(handleGatewayState(baseOptions(deps, "healthy"))).rejects.toMatchObject({ + code: "identity_mismatch", + }); + expect(calls.startGateway).not.toHaveBeenCalled(); + }); + + it("revalidates the declared owner on resume rather than trusting the recorded step (#6576)", async () => { + const { calls, deps } = externalDeps({ + portOccupied: false, + listenerPids: [], + httpReady: false, + }); + const session = createSession(); + session.steps = { + gateway: { status: "complete", startedAt: null, completedAt: null, error: null }, + }; + + await expect( + handleGatewayState({ ...baseOptions(deps, "healthy", session), resume: true }), + ).rejects.toMatchObject({ code: "gateway_unreachable" }); + expect(calls.probeAttachment).toHaveBeenCalledOnce(); + expect(calls.startGateway).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/onboard/machine/handlers/gateway.ts b/src/lib/onboard/machine/handlers/gateway.ts index bd3401b977c..1c3e2456b2d 100644 --- a/src/lib/onboard/machine/handlers/gateway.ts +++ b/src/lib/onboard/machine/handlers/gateway.ts @@ -5,6 +5,14 @@ import type { NvidiaPlatform } from "../../../inference/nim"; import type { GatewayReuseState } from "../../../state/gateway"; import type { Session } from "../../../state/onboard-session"; import type { GatewayContainerState } from "../../gateway-container-running"; +import { + describeGatewayOwner, + evaluateGatewayAttachment, + type GatewayAttachmentProbe, + type GatewayOwner, + GatewayOwnershipError, + isExternallySupervised, +} from "../../gateway-ownership"; import { withGatewayTrace } from "../../tracing"; import { advanceTo, type OnboardStateTransitionResult } from "../result"; @@ -19,6 +27,14 @@ export interface GatewayStateOptions { requestedSandboxName: string | null; recreateSandbox: boolean; deps: { + /** + * The single declared lifecycle authority for this run (#6576). Resolved + * before any effect so an externally supervised gateway is attached to + * rather than started, replaced, or destroyed. + */ + resolveGatewayOwner(): GatewayOwner; + probeGatewayAttachment(owner: GatewayOwner): Promise; + attachGateway(owner: GatewayOwner): void; refreshDockerDriverGatewayReuseState(state: GatewayReuseState): Promise; gatewayCliSupportsLifecycleCommands(): boolean; verifyGatewayContainerRunning(gatewayName: string): GatewayContainerState; @@ -91,6 +107,15 @@ async function handleGatewayStatePhase({ recreateSandbox, deps, }: GatewayStateOptions): Promise { + // Establish the lifecycle authority before anything in this phase can touch + // the gateway. Resume takes the same path as a fresh run: a recorded + // "complete" gateway step is not evidence that the declared owner still holds + // the port. + const owner = deps.resolveGatewayOwner(); + if (isExternallySupervised(owner)) { + return attachToExternallySupervisedGateway(owner, deps); + } + let gatewayReuseState = await deps.refreshDockerDriverGatewayReuseState(initialGatewayReuseState); const supportsLifecycleCommands = deps.gatewayCliSupportsLifecycleCommands(); @@ -239,7 +264,48 @@ async function handleGatewayStatePhase({ gatewayReuseState, session, stateResult: advanceTo("provider_selection", { - metadata: { state: "gateway", gatewayReuseState }, + metadata: { state: "gateway", gatewayReuseState, gatewayOwner: describeGatewayOwner(owner) }, + }), + }; +} + +/** + * Externally supervised gateways are attached to, never managed. This path runs + * no destructive effect: it validates the declared owner still holds the port + * and fails closed otherwise, so an unknown listener, a mismatched identity, or + * multiple owners is reported before any provider, policy, sandbox, or registry + * mutation can run. + */ +async function attachToExternallySupervisedGateway( + owner: GatewayOwner, + deps: GatewayStateOptions["deps"], +): Promise { + const supervisor = owner.supervisor?.serviceName ?? "an external supervisor"; + const attachment = evaluateGatewayAttachment(owner, await deps.probeGatewayAttachment(owner)); + if (!attachment.ok) { + throw new GatewayOwnershipError(attachment.code, attachment.message, owner); + } + deps.attachGateway(owner); + + deps.skippedStepMessage("gateway", `supervised by ${supervisor}`, "reuse"); + deps.note(` Attached to externally supervised OpenShell gateway (${supervisor}).`); + await deps.recordStateSkipped("gateway", { + reason: "external-supervision", + gatewayOwner: describeGatewayOwner(owner), + }); + + return { + // No NemoClaw-owned gateway runtime exists to reuse or recreate; the + // supervisor owns it, and downstream reuse decisions must not treat this as + // a gateway NemoClaw may recycle. + gatewayReuseState: "healthy", + session: await deps.recordStepComplete("gateway"), + stateResult: advanceTo("provider_selection", { + metadata: { + state: "gateway", + gatewayReuseState: "healthy", + gatewayOwner: describeGatewayOwner(owner), + }, }), }; } diff --git a/src/lib/onboard/machine/handlers/sandbox-checkpoint-crash-recovery.test.ts b/src/lib/onboard/machine/handlers/sandbox-checkpoint-crash-recovery.test.ts index 4d15d0fa1cc..4bb45fd938c 100644 --- a/src/lib/onboard/machine/handlers/sandbox-checkpoint-crash-recovery.test.ts +++ b/src/lib/onboard/machine/handlers/sandbox-checkpoint-crash-recovery.test.ts @@ -47,6 +47,7 @@ function crashedCheckpoint(overrides: Partial = {}): OnboardC webSearch: decisionUnset(), messaging: decisionUnset(), resourceProfile: decisionUnset(), + gatewayAuthority: decisionUnset(), effectGroups: { sandbox_create: { completedAt: "2026-01-01T00:00:00.000Z", diff --git a/src/lib/onboard/machine/handlers/sandbox-messaging.test.ts b/src/lib/onboard/machine/handlers/sandbox-messaging.test.ts index 84a7912dc4a..6056233577c 100644 --- a/src/lib/onboard/machine/handlers/sandbox-messaging.test.ts +++ b/src/lib/onboard/machine/handlers/sandbox-messaging.test.ts @@ -173,6 +173,7 @@ function withMessagingCheckpoint( webSearch: decisionUnset(), messaging: decisionSelected({ selectedChannels, disabledChannels }), resourceProfile: decisionUnset(), + gatewayAuthority: decisionUnset(), effectGroups: {}, bindings: { credentialEnvs: [], registeredProviders: [] }, }; diff --git a/src/lib/onboard/machine/handlers/sandbox.test.ts b/src/lib/onboard/machine/handlers/sandbox.test.ts index 6a8de4d223d..45fdcb4e85d 100644 --- a/src/lib/onboard/machine/handlers/sandbox.test.ts +++ b/src/lib/onboard/machine/handlers/sandbox.test.ts @@ -147,6 +147,7 @@ describe("handleSandboxState", () => { webSearch: decisionUnset(), messaging: decisionUnset(), resourceProfile: decisionUnset(), + gatewayAuthority: decisionUnset(), effectGroups: { web_search_provider: { completedAt: "2026-01-01T00:00:00.000Z", @@ -602,6 +603,7 @@ describe("handleSandboxState", () => { webSearch: decisionUnset(), messaging: decisionUnset(), resourceProfile: decisionUnset(), + gatewayAuthority: decisionUnset(), effectGroups: {}, bindings: { credentialEnvs: [], registeredProviders: [] }, }; @@ -642,6 +644,7 @@ describe("handleSandboxState", () => { webSearch: decisionUnset(), messaging: decisionUnset(), resourceProfile: decisionUnset(), + gatewayAuthority: decisionUnset(), effectGroups: {}, bindings: { credentialEnvs: [], registeredProviders: [] }, }; @@ -675,6 +678,7 @@ describe("handleSandboxState", () => { webSearch: decisionSelected({ fetchEnabled: true, provider: "brave" }), messaging: decisionUnset(), resourceProfile: decisionUnset(), + gatewayAuthority: decisionUnset(), effectGroups: {}, bindings: { credentialEnvs: [], registeredProviders: [] }, }; @@ -716,6 +720,7 @@ describe("handleSandboxState", () => { webSearch: decisionDeclined(), messaging: decisionDeclined(), resourceProfile: decisionSelected({ cpu: "4", memory: "8Gi" }), + gatewayAuthority: decisionUnset(), effectGroups: {}, bindings: { credentialEnvs: [], registeredProviders: [] }, }; diff --git a/src/lib/onboard/machine/initial-flow-phases.test.ts b/src/lib/onboard/machine/initial-flow-phases.test.ts index 0274cc19c21..2daf0a2c926 100644 --- a/src/lib/onboard/machine/initial-flow-phases.test.ts +++ b/src/lib/onboard/machine/initial-flow-phases.test.ts @@ -5,6 +5,7 @@ import { describe, expect, it, vi } from "vitest"; import { createSession, type Session } from "../../state/onboard-session"; import { recordInvalidatedTargets } from "../__test-helpers__/machine-recorders"; +import { resolveGatewayOwner } from "../gateway-ownership"; import { createInitialOnboardFlowPhases, type InitialOnboardFlowContext, @@ -118,6 +119,24 @@ describe("initial onboard flow phases", () => { gatewayName: "nemoclaw", recreateSandbox: () => false, gatewayDeps: { + resolveGatewayOwner: () => + resolveGatewayOwner({ + gatewayName: "nemoclaw", + gatewayPort: 31818, + declaration: null, + hasPackagedService: false, + }), + attachGateway: vi.fn(), + probeGatewayAttachment: async () => ({ + gatewayPort: 31818, + httpReady: true, + portOccupied: true, + listenerPids: [4242], + listenerScanComplete: true, + supervisorActive: null, + listenerExecPath: null, + listenerSupervisorMatch: null, + }), refreshDockerDriverGatewayReuseState: async (state) => state, gatewayCliSupportsLifecycleCommands: () => false, verifyGatewayContainerRunning: () => "running", @@ -154,7 +173,19 @@ describe("initial onboard flow phases", () => { expect(preflight.context.gpuPassthrough).toBe(true); expect(gateway.result).toEqual( advanceTo("provider_selection", { - metadata: { state: "gateway", gatewayReuseState: "healthy" }, + metadata: { + state: "gateway", + gatewayReuseState: "healthy", + gatewayOwner: { + gatewayName: "nemoclaw", + gatewayPort: 31818, + mode: "nemoclaw-managed", + source: "standalone", + endpoint: null, + supervisor: null, + requiredCapabilities: [], + }, + }, }), ); expect(notes).toContain( @@ -330,6 +361,24 @@ describe("initial onboard flow phases", () => { gatewayName: "nemoclaw", recreateSandbox: () => false, gatewayDeps: { + resolveGatewayOwner: () => + resolveGatewayOwner({ + gatewayName: "nemoclaw", + gatewayPort: 31818, + declaration: null, + hasPackagedService: false, + }), + attachGateway: vi.fn(), + probeGatewayAttachment: async () => ({ + gatewayPort: 31818, + httpReady: true, + portOccupied: true, + listenerPids: [4242], + listenerScanComplete: true, + supervisorActive: null, + listenerExecPath: null, + listenerSupervisorMatch: null, + }), refreshDockerDriverGatewayReuseState: vi.fn(async (state) => { calls.push("refresh-gateway-reuse"); return state; diff --git a/src/lib/onboard/preflight-gateway-cleanup-decision.test.ts b/src/lib/onboard/preflight-gateway-cleanup-decision.test.ts index 6e8021f12bb..6f91d96aea1 100644 --- a/src/lib/onboard/preflight-gateway-cleanup-decision.test.ts +++ b/src/lib/onboard/preflight-gateway-cleanup-decision.test.ts @@ -59,6 +59,18 @@ describe("preflightGatewayCleanupDecision", () => { ).toBe("destroy-legacy"); }); + it("returns noop for an externally supervised gateway even when stale (#6576)", () => { + for (const state of ["stale", "active-unnamed"] as const) { + expect( + preflightGatewayCleanupDecision({ + gatewayReuseState: state, + isDockerDriverGatewayEnabled: false, + externallySupervised: true, + }), + ).toBe("noop"); + } + }); + it("returns noop for non-stale states regardless of driver", () => { for (const state of ["healthy", "missing", "foreign-active"] as const) { expect( @@ -81,6 +93,7 @@ describe("applyPreflightGatewayCleanup", () => { function makeDeps(overrides: { gatewayReuseState: GatewayReuseState; isDockerDriverGatewayEnabled: boolean; + externallySupervised?: boolean; }) { const log = vi.fn(); const warn = vi.fn(); @@ -96,6 +109,7 @@ describe("applyPreflightGatewayCleanup", () => { deps: { gatewayReuseState: overrides.gatewayReuseState, isDockerDriverGatewayEnabled: overrides.isDockerDriverGatewayEnabled, + externallySupervised: overrides.externallySupervised, cliDisplayName: "NemoClaw", dashboardPort: 8081, log, @@ -126,6 +140,21 @@ describe("applyPreflightGatewayCleanup", () => { expect(ctx.runOpenshell).not.toHaveBeenCalled(); }); + it("performs no destroy for an externally supervised gateway on the legacy path (#6576)", () => { + const ctx = makeDeps({ + gatewayReuseState: "stale", + isDockerDriverGatewayEnabled: false, + externallySupervised: true, + }); + + const next = applyPreflightGatewayCleanup(ctx.deps); + + expect(next).toBe("stale"); + expect(ctx.destroyGateway).not.toHaveBeenCalled(); + expect(ctx.destroyGatewayForReuse).not.toHaveBeenCalled(); + expect(ctx.runOpenshell).not.toHaveBeenCalled(); + }); + it("prints the deferral warning without ANSI when NO_COLOR is set", () => { stubStderrColorDepth(24); vi.stubEnv("NO_COLOR", "1"); diff --git a/src/lib/onboard/preflight-gateway-cleanup-decision.ts b/src/lib/onboard/preflight-gateway-cleanup-decision.ts index 1b4e47da93a..359bdfaa393 100644 --- a/src/lib/onboard/preflight-gateway-cleanup-decision.ts +++ b/src/lib/onboard/preflight-gateway-cleanup-decision.ts @@ -12,7 +12,13 @@ export const PREFLIGHT_DEFERRED_RECREATE_MESSAGE = export function preflightGatewayCleanupDecision(opts: { gatewayReuseState: GatewayReuseState; isDockerDriverGatewayEnabled: boolean; + externallySupervised?: boolean; }): PreflightGatewayCleanupAction { + // An externally supervised gateway is never cleaned up by NemoClaw, even from + // preflight: stale metadata or an interrupted prior run must not let this + // destroy a gateway whose declared supervisor is the sole lifecycle authority + // (#6576). The FSM attach path is the only thing that touches it. + if (opts.externallySupervised) return "noop"; if (opts.gatewayReuseState !== "stale" && opts.gatewayReuseState !== "active-unnamed") { return "noop"; } @@ -22,6 +28,7 @@ export function preflightGatewayCleanupDecision(opts: { export interface PreflightGatewayCleanupDeps { gatewayReuseState: GatewayReuseState; isDockerDriverGatewayEnabled: boolean; + externallySupervised?: boolean; cliDisplayName: string; dashboardPort: number; log: (line: string) => void; @@ -39,6 +46,7 @@ export function applyPreflightGatewayCleanup(deps: PreflightGatewayCleanupDeps): const action = preflightGatewayCleanupDecision({ gatewayReuseState: deps.gatewayReuseState, isDockerDriverGatewayEnabled: deps.isDockerDriverGatewayEnabled, + externallySupervised: deps.externallySupervised, }); if (action === "defer") { deps.warn(warnLine(PREFLIGHT_DEFERRED_RECREATE_MESSAGE)); diff --git a/src/lib/onboard/preflight-gateway-reuse.test.ts b/src/lib/onboard/preflight-gateway-reuse.test.ts index 5d5a805fc79..0b1b7eadf61 100644 --- a/src/lib/onboard/preflight-gateway-reuse.test.ts +++ b/src/lib/onboard/preflight-gateway-reuse.test.ts @@ -2,12 +2,11 @@ // SPDX-License-Identifier: Apache-2.0 import { describe, expect, it, vi } from "vitest"; - -import type { GatewayContainerState } from "./gateway-container-running"; import type { GatewayReuseState } from "../state/gateway"; +import type { GatewayContainerState } from "./gateway-container-running"; import { - reconcilePreflightGatewayReuseState, type PreflightGatewayReuseDeps, + reconcilePreflightGatewayReuseState, } from "./preflight-gateway-reuse"; function makeDeps(overrides: Partial = {}): PreflightGatewayReuseDeps { @@ -55,6 +54,26 @@ describe("reconcilePreflightGatewayReuseState", () => { expect(verify).not.toHaveBeenCalled(); }); + it("performs no recover or recreate for an externally supervised gateway (#6576)", async () => { + const verify = vi.fn(() => "stopped" as GatewayContainerState); + const recover = vi.fn(async () => true); + const destroyForReuse = vi.fn(() => "missing" as GatewayReuseState); + const deps = makeDeps({ + gatewayReuseState: "healthy", + externallySupervised: true, + verifyGatewayContainerRunning: verify, + recoverGatewayRuntime: recover, + destroyGatewayForReuse: destroyForReuse, + }); + + const result = await reconcilePreflightGatewayReuseState(deps); + + expect(result).toBe("healthy"); + expect(verify).not.toHaveBeenCalled(); + expect(recover).not.toHaveBeenCalled(); + expect(destroyForReuse).not.toHaveBeenCalled(); + }); + it("recovers a stopped container without removing volumes (#4187)", async () => { const recover = vi.fn(async () => true); const destroyForReuse = vi.fn(() => "missing" as GatewayReuseState); diff --git a/src/lib/onboard/preflight-gateway-reuse.ts b/src/lib/onboard/preflight-gateway-reuse.ts index 8bf77a06963..c07b6b3489b 100644 --- a/src/lib/onboard/preflight-gateway-reuse.ts +++ b/src/lib/onboard/preflight-gateway-reuse.ts @@ -1,12 +1,14 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import type { GatewayContainerState } from "./gateway-container-running"; import type { GatewayReuseState } from "../state/gateway"; +import type { GatewayContainerState } from "./gateway-container-running"; export interface PreflightGatewayReuseDeps { gatewayReuseState: GatewayReuseState; supportsLifecycleCommands: boolean; + /** When true, NemoClaw performs no recover/recreate here (#6576). */ + externallySupervised?: boolean; gatewayName: string; verifyGatewayContainerRunning(name: string): GatewayContainerState; recoverGatewayRuntime(): Promise; @@ -39,6 +41,9 @@ export async function reconcilePreflightGatewayReuseState( deps: PreflightGatewayReuseDeps, ): Promise { let gatewayReuseState = deps.gatewayReuseState; + // An externally supervised gateway is validated and attached to by the FSM, + // never recovered or recreated here (#6576). + if (deps.externallySupervised) return gatewayReuseState; if (gatewayReuseState !== "healthy" || !deps.supportsLifecycleCommands) { return gatewayReuseState; } diff --git a/src/lib/onboard/preflight-gateway-sequence.test.ts b/src/lib/onboard/preflight-gateway-sequence.test.ts new file mode 100644 index 00000000000..cb217e8b37d --- /dev/null +++ b/src/lib/onboard/preflight-gateway-sequence.test.ts @@ -0,0 +1,178 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import type { GatewayReuseState } from "../state/gateway"; +import type { GatewayContainerState } from "./gateway-container-running"; +import { + type PreflightGatewaySequenceDeps, + runPreflightGatewaySequence, +} from "./preflight-gateway-sequence"; + +interface SequenceHarness { + deps: PreflightGatewaySequenceDeps; + destructive: Record>; +} + +function harness(overrides: { + gatewayReuseState: GatewayReuseState; + externallySupervised: boolean; + containerState?: GatewayContainerState; + orphanContainerPresent?: boolean; + httpReady?: boolean; + imageDrift?: { currentVersion: string; expectedVersion: string } | null; + destroyedReuseState?: GatewayReuseState; +}): SequenceHarness { + const destroyGateway = vi.fn(() => true); + const destroyGatewayForReuse = vi.fn( + (): GatewayReuseState => overrides.destroyedReuseState ?? "missing", + ); + const runOpenshell = vi.fn(); + const dockerStop = vi.fn(); + const dockerRm = vi.fn(); + const dockerRemoveVolumesByPrefix = vi.fn(); + const clearRegistry = vi.fn(); + const stopDashboardForward = vi.fn(); + const stopAllDashboardForwards = vi.fn(); + const exitProcess = vi.fn((code: number): never => { + throw new Error(`exit ${code}`); + }); + let inspectCalls = 0; + const deps: PreflightGatewaySequenceDeps = { + gatewayReuseState: overrides.gatewayReuseState, + externallySupervised: overrides.externallySupervised, + supportsLifecycleCommands: true, + isDockerDriverGatewayEnabled: false, + gatewayName: "nemoclaw", + cliDisplayName: "NemoClaw", + dashboardPort: 3000, + verifyGatewayContainerRunning: () => overrides.containerState ?? "running", + recoverGatewayRuntime: async () => true, + waitForGatewayHttpReady: async () => overrides.httpReady ?? true, + getGatewayLocalEndpoint: () => "http://127.0.0.1:8080", + stopDashboardForward, + stopAllDashboardForwards, + getGatewayClusterImageDrift: () => overrides.imageDrift ?? null, + exitProcess: exitProcess as unknown as (code: number) => never, + destroyGateway, + destroyGatewayForReuse, + runOpenshell, + dockerInspect: () => { + inspectCalls += 1; + // Only the first inspect finds the orphan; the post-removal inspect + // reports the container gone so volume/registry cleanup proceeds. + const present = (overrides.orphanContainerPresent ?? false) && inspectCalls === 1; + return { status: present ? 0 : 1 }; + }, + dockerStop, + dockerRm, + dockerRemoveVolumesByPrefix, + clearRegistry, + log: () => {}, + warn: () => {}, + }; + return { + deps, + destructive: { + destroyGateway, + destroyGatewayForReuse, + dockerStop, + dockerRm, + dockerRemoveVolumesByPrefix, + clearRegistry, + runOpenshell, + stopDashboardForward, + stopAllDashboardForwards, + }, + }; +} + +function expectNoDestructiveEffect(h: SequenceHarness): void { + for (const [name, fn] of Object.entries(h.destructive)) { + expect(fn, `${name} must not run under external supervision`).not.toHaveBeenCalled(); + } +} + +describe("full preflight gateway sequence under external supervision (#6576)", () => { + it("crosses the whole path with zero destructive effects when metadata is missing and an orphan-looking container exists", async () => { + // The original regression: "missing" only means NemoClaw holds no + // metadata, but the container is the supervisor's live gateway. + const h = harness({ + gatewayReuseState: "missing", + externallySupervised: true, + orphanContainerPresent: true, + }); + const result = await runPreflightGatewaySequence(h.deps); + expect(result).toBe("missing"); + expectNoDestructiveEffect(h); + }); + + it.each([ + "healthy", + "stale", + "active-unnamed", + "foreign-active", + ])("performs no cleanup and preserves reuse state %s", async (gatewayReuseState) => { + const h = harness({ + gatewayReuseState, + externallySupervised: true, + containerState: "missing", + orphanContainerPresent: true, + httpReady: false, + imageDrift: { currentVersion: "1.0.0", expectedVersion: "2.0.0" }, + }); + const result = await runPreflightGatewaySequence(h.deps); + expect(result).toBe(gatewayReuseState); + expectNoDestructiveEffect(h); + }); +}); + +describe("full preflight gateway sequence when NemoClaw owns the gateway (#6576)", () => { + it("still removes a genuinely orphaned container end-to-end", async () => { + const h = harness({ + gatewayReuseState: "missing", + externallySupervised: false, + orphanContainerPresent: true, + }); + await runPreflightGatewaySequence(h.deps); + expect(h.destructive.dockerStop).toHaveBeenCalledWith("openshell-cluster-nemoclaw", { + ignoreError: true, + suppressOutput: true, + }); + expect(h.destructive.dockerRm).toHaveBeenCalled(); + expect(h.destructive.dockerRemoveVolumesByPrefix).toHaveBeenCalled(); + expect(h.destructive.clearRegistry).toHaveBeenCalled(); + }); + + it("still destroys a stale legacy session through the cleanup stage", async () => { + const h = harness({ + gatewayReuseState: "stale", + externallySupervised: false, + destroyedReuseState: "missing", + }); + await runPreflightGatewaySequence(h.deps); + expect(h.destructive.destroyGatewayForReuse).toHaveBeenCalledTimes(1); + expect(h.destructive.runOpenshell).toHaveBeenCalledWith(["forward", "stop", "3000"], { + ignoreError: true, + }); + }); + + it("feeds each stage the reuse state the previous stage produced", async () => { + // Reconciliation downgrades "healthy" (stale metadata, container gone) to + // "missing"; the orphan stage must then consume that downgraded state and + // clean up. A composition regression that re-reads the original input + // would skip the orphan stage entirely. + const h = harness({ + gatewayReuseState: "healthy", + externallySupervised: false, + containerState: "missing", + orphanContainerPresent: true, + destroyedReuseState: "missing", + }); + const result = await runPreflightGatewaySequence(h.deps); + expect(h.destructive.destroyGatewayForReuse).toHaveBeenCalledTimes(1); + expect(h.destructive.dockerStop).toHaveBeenCalled(); + expect(result).toBe("missing"); + }); +}); diff --git a/src/lib/onboard/preflight-gateway-sequence.ts b/src/lib/onboard/preflight-gateway-sequence.ts new file mode 100644 index 00000000000..a4b29ce017c --- /dev/null +++ b/src/lib/onboard/preflight-gateway-sequence.ts @@ -0,0 +1,110 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { GatewayReuseState } from "../state/gateway"; +import type { GatewayContainerState } from "./gateway-container-running"; +import { applyPreflightGatewayCleanup } from "./preflight-gateway-cleanup-decision"; +import { reconcilePreflightGatewayReuseState } from "./preflight-gateway-reuse"; +import { cleanupOrphanedGatewayContainer } from "./preflight-orphan-gateway-cleanup"; + +export interface PreflightGatewaySequenceDeps { + gatewayReuseState: GatewayReuseState; + /** Resolved once by the caller before this sequence runs (#6576). */ + externallySupervised: boolean; + supportsLifecycleCommands: boolean; + isDockerDriverGatewayEnabled: boolean; + gatewayName: string; + cliDisplayName: string; + dashboardPort: number; + verifyGatewayContainerRunning(name: string): GatewayContainerState; + recoverGatewayRuntime(): Promise; + waitForGatewayHttpReady(): Promise; + getGatewayLocalEndpoint(): string; + stopDashboardForward(): void; + stopAllDashboardForwards(): void; + getGatewayClusterImageDrift(): { currentVersion: string; expectedVersion: string } | null; + exitProcess(code: number): never; + destroyGateway(): boolean; + destroyGatewayForReuse( + destroyGateway: () => boolean, + successMessage: string, + failureMessage: string, + ): GatewayReuseState; + runOpenshell(args: string[], options: { ignoreError: true }): unknown; + dockerInspect( + args: string[], + opts: { ignoreError: true; suppressOutput: true }, + ): { status: number | null }; + dockerStop(name: string, opts: { ignoreError: true; suppressOutput: true }): unknown; + dockerRm(name: string, opts: { ignoreError: true; suppressOutput: true }): unknown; + dockerRemoveVolumesByPrefix( + prefix: string, + opts: { ignoreError: true; suppressOutput: true }, + ): unknown; + clearRegistry(): void; + log(message: string): void; + warn(message: string): void; +} + +/** + * The full preflight gateway-mutating sequence, composed exactly as onboard + * runs it: reuse reconciliation, then stale/unnamed cleanup, then orphaned + * container cleanup — each stage consuming the reuse state the previous stage + * produced. + * + * The one lifecycle authority is resolved by the caller before this sequence + * and threaded to every stage, so an externally supervised gateway crosses the + * whole path with zero destructive effects. Keeping the composition in one + * unit makes that guarantee testable end-to-end: the original #6576 regression + * was two guarded stages followed by an unguarded one, which per-stage tests + * cannot catch. + */ +export async function runPreflightGatewaySequence( + deps: PreflightGatewaySequenceDeps, +): Promise { + let gatewayReuseState = await reconcilePreflightGatewayReuseState({ + gatewayReuseState: deps.gatewayReuseState, + supportsLifecycleCommands: deps.supportsLifecycleCommands, + externallySupervised: deps.externallySupervised, + gatewayName: deps.gatewayName, + verifyGatewayContainerRunning: deps.verifyGatewayContainerRunning, + recoverGatewayRuntime: deps.recoverGatewayRuntime, + waitForGatewayHttpReady: deps.waitForGatewayHttpReady, + getGatewayLocalEndpoint: deps.getGatewayLocalEndpoint, + stopDashboardForward: deps.stopDashboardForward, + stopAllDashboardForwards: deps.stopAllDashboardForwards, + destroyGateway: deps.destroyGateway, + destroyGatewayForReuse: deps.destroyGatewayForReuse, + getGatewayClusterImageDrift: deps.getGatewayClusterImageDrift, + exitProcess: deps.exitProcess, + }); + + gatewayReuseState = applyPreflightGatewayCleanup({ + gatewayReuseState, + isDockerDriverGatewayEnabled: deps.isDockerDriverGatewayEnabled, + externallySupervised: deps.externallySupervised, + cliDisplayName: deps.cliDisplayName, + dashboardPort: deps.dashboardPort, + log: deps.log, + warn: deps.warn, + runOpenshell: deps.runOpenshell, + destroyGateway: deps.destroyGateway, + destroyGatewayForReuse: deps.destroyGatewayForReuse, + }); + + cleanupOrphanedGatewayContainer({ + gatewayReuseState, + isDockerDriverGatewayEnabled: deps.isDockerDriverGatewayEnabled, + externallySupervised: deps.externallySupervised, + gatewayName: deps.gatewayName, + dockerInspect: deps.dockerInspect, + dockerStop: deps.dockerStop, + dockerRm: deps.dockerRm, + dockerRemoveVolumesByPrefix: deps.dockerRemoveVolumesByPrefix, + clearRegistry: deps.clearRegistry, + log: deps.log, + warn: deps.warn, + }); + + return gatewayReuseState; +} diff --git a/src/lib/onboard/preflight-orphan-gateway-cleanup.test.ts b/src/lib/onboard/preflight-orphan-gateway-cleanup.test.ts new file mode 100644 index 00000000000..55d86c2a068 --- /dev/null +++ b/src/lib/onboard/preflight-orphan-gateway-cleanup.test.ts @@ -0,0 +1,96 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { + cleanupOrphanedGatewayContainer, + type OrphanGatewayCleanupDeps, +} from "./preflight-orphan-gateway-cleanup"; + +function makeDeps(overrides: Partial = {}) { + const deps: OrphanGatewayCleanupDeps = { + gatewayReuseState: "missing", + isDockerDriverGatewayEnabled: false, + externallySupervised: false, + gatewayName: "nemoclaw", + // Container exists, and is gone after removal. + dockerInspect: vi.fn(() => ({ status: 0 })), + dockerStop: vi.fn(), + dockerRm: vi.fn(), + dockerRemoveVolumesByPrefix: vi.fn(), + clearRegistry: vi.fn(), + log: vi.fn(), + warn: vi.fn(), + ...overrides, + }; + return deps; +} + +describe("cleanupOrphanedGatewayContainer (#6576)", () => { + it("performs no container, volume, or registry mutation under external supervision", () => { + // "missing" only means NemoClaw holds no metadata; the supervisor's + // container is still the live gateway. + const deps = makeDeps({ externallySupervised: true }); + + cleanupOrphanedGatewayContainer(deps); + + expect(deps.dockerInspect).not.toHaveBeenCalled(); + expect(deps.dockerStop).not.toHaveBeenCalled(); + expect(deps.dockerRm).not.toHaveBeenCalled(); + expect(deps.dockerRemoveVolumesByPrefix).not.toHaveBeenCalled(); + expect(deps.clearRegistry).not.toHaveBeenCalled(); + }); + + it("removes a genuinely orphaned container when NemoClaw owns the gateway", () => { + let inspectCalls = 0; + const deps = makeDeps({ + // Present first, absent after removal. + dockerInspect: vi.fn(() => ({ status: inspectCalls++ === 0 ? 0 : 1 })), + }); + + cleanupOrphanedGatewayContainer(deps); + + expect(deps.dockerStop).toHaveBeenCalledWith("openshell-cluster-nemoclaw", expect.anything()); + expect(deps.dockerRm).toHaveBeenCalledWith("openshell-cluster-nemoclaw", expect.anything()); + expect(deps.dockerRemoveVolumesByPrefix).toHaveBeenCalledWith( + "openshell-cluster-nemoclaw", + expect.anything(), + ); + expect(deps.clearRegistry).toHaveBeenCalledOnce(); + }); + + it("warns without clearing state when the container survives removal", () => { + const deps = makeDeps({ dockerInspect: vi.fn(() => ({ status: 0 })) }); + + cleanupOrphanedGatewayContainer(deps); + + expect(deps.dockerRemoveVolumesByPrefix).not.toHaveBeenCalled(); + expect(deps.clearRegistry).not.toHaveBeenCalled(); + expect(deps.warn).toHaveBeenCalledWith( + " ! Found an orphaned gateway container, but automatic cleanup failed.", + ); + }); + + it("does nothing when the gateway is reusable or on the Docker-driver path", () => { + for (const overrides of [ + { gatewayReuseState: "healthy" }, + { isDockerDriverGatewayEnabled: true }, + ]) { + const deps = makeDeps(overrides); + + cleanupOrphanedGatewayContainer(deps); + + expect(deps.dockerInspect).not.toHaveBeenCalled(); + } + }); + + it("does nothing when no orphaned container is present", () => { + const deps = makeDeps({ dockerInspect: vi.fn(() => ({ status: 1 })) }); + + cleanupOrphanedGatewayContainer(deps); + + expect(deps.dockerStop).not.toHaveBeenCalled(); + expect(deps.clearRegistry).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/onboard/preflight-orphan-gateway-cleanup.ts b/src/lib/onboard/preflight-orphan-gateway-cleanup.ts new file mode 100644 index 00000000000..e3f3066abfc --- /dev/null +++ b/src/lib/onboard/preflight-orphan-gateway-cleanup.ts @@ -0,0 +1,65 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export interface OrphanGatewayCleanupDeps { + gatewayReuseState: string; + isDockerDriverGatewayEnabled: boolean; + /** When true, NemoClaw performs no container/volume/registry cleanup (#6576). */ + externallySupervised: boolean; + gatewayName: string; + dockerInspect( + args: string[], + opts: { ignoreError: true; suppressOutput: true }, + ): { status: number | null }; + dockerStop(name: string, opts: { ignoreError: true; suppressOutput: true }): unknown; + dockerRm(name: string, opts: { ignoreError: true; suppressOutput: true }): unknown; + dockerRemoveVolumesByPrefix( + prefix: string, + opts: { ignoreError: true; suppressOutput: true }, + ): unknown; + clearRegistry(): void; + log(message: string): void; + warn(message: string): void; +} + +/** + * Remove a gateway container orphaned by an interrupted onboard (e.g. Ctrl+C + * during gateway start): OpenShell can have no metadata for it + * (`gatewayReuseState === "missing"`) while the container still runs. + * + * This is destructive — it stops and removes the container, deletes matching + * volumes, and clears the sandbox registry — so it must never run against a + * gateway NemoClaw does not own. Under external supervision, "missing" only + * means NemoClaw holds no metadata; the supervisor's container is still the + * live gateway, and destroying it here would happen before the FSM attachment + * check ever runs (#6576). + */ +export function cleanupOrphanedGatewayContainer(deps: OrphanGatewayCleanupDeps): void { + if (deps.externallySupervised) return; + if (deps.gatewayReuseState !== "missing" || deps.isDockerDriverGatewayEnabled) return; + + const containerName = `openshell-cluster-${deps.gatewayName}`; + const inspectResult = deps.dockerInspect( + ["--type", "container", "--format", "{{.State.Status}}", containerName], + { ignoreError: true, suppressOutput: true }, + ); + if (inspectResult.status !== 0) return; + + deps.log(" Cleaning up orphaned gateway container..."); + deps.dockerStop(containerName, { ignoreError: true, suppressOutput: true }); + deps.dockerRm(containerName, { ignoreError: true, suppressOutput: true }); + const postInspectResult = deps.dockerInspect(["--type", "container", containerName], { + ignoreError: true, + suppressOutput: true, + }); + if (postInspectResult.status !== 0) { + deps.dockerRemoveVolumesByPrefix(containerName, { + ignoreError: true, + suppressOutput: true, + }); + deps.clearRegistry(); + deps.log(" ✓ Orphaned gateway container removed"); + } else { + deps.warn(" ! Found an orphaned gateway container, but automatic cleanup failed."); + } +} diff --git a/src/lib/onboard/preflight-ports.ts b/src/lib/onboard/preflight-ports.ts index ec814717862..db7a9a4eb60 100644 --- a/src/lib/onboard/preflight-ports.ts +++ b/src/lib/onboard/preflight-ports.ts @@ -6,7 +6,10 @@ import { HERMES_OPENAI_API_PORT } from "../core/ports"; /** Agent-neutral rejection message when {@link HERMES_OPENAI_API_PORT} is requested as a dashboard port; shared by both #4984 guards. */ export const RESERVED_HERMES_DASHBOARD_PORT_MESSAGE = `[SECURITY] Invalid dashboard port ${HERMES_OPENAI_API_PORT} - reserved for the Hermes OpenAI-compatible API`; +export type PreflightPortKind = "gateway" | "dashboard" | "other"; + export interface PreflightPort { + kind: PreflightPortKind; port: number; label: string; envVar: string; @@ -24,10 +27,16 @@ export function buildRequiredPreflightPorts(opts: { dashboardLabel: string; }): PreflightPort[] { return [ - { port: opts.gatewayPort, label: "OpenShell gateway", envVar: "NEMOCLAW_GATEWAY_PORT" }, + { + kind: "gateway", + port: opts.gatewayPort, + label: "OpenShell gateway", + envVar: "NEMOCLAW_GATEWAY_PORT", + }, ...(opts.dashboardPort !== null ? [ { + kind: "dashboard" as const, port: opts.dashboardPort, label: opts.dashboardLabel, envVar: "NEMOCLAW_DASHBOARD_PORT", diff --git a/src/lib/onboard/session-bootstrap.test.ts b/src/lib/onboard/session-bootstrap.test.ts index 91eb44d3f8e..9f8cc37fb0b 100644 --- a/src/lib/onboard/session-bootstrap.test.ts +++ b/src/lib/onboard/session-bootstrap.test.ts @@ -382,6 +382,7 @@ describe("prepareOnboardSession", () => { webSearch: decisionUnset(), messaging: decisionUnset(), resourceProfile: decisionUnset(), + gatewayAuthority: decisionUnset(), effectGroups: {}, bindings: { credentialEnvs: [], registeredProviders: [] }, }; @@ -432,6 +433,7 @@ describe("prepareOnboardSession", () => { webSearch: decisionUnset(), messaging: decisionUnset(), resourceProfile: decisionUnset(), + gatewayAuthority: decisionUnset(), effectGroups: {}, bindings: { credentialEnvs: [], registeredProviders: [] }, }; diff --git a/src/lib/state/onboard-checkpoint-migrate.test.ts b/src/lib/state/onboard-checkpoint-migrate.test.ts index 81de24c5023..9a8e3fc18ae 100644 --- a/src/lib/state/onboard-checkpoint-migrate.test.ts +++ b/src/lib/state/onboard-checkpoint-migrate.test.ts @@ -87,6 +87,7 @@ describe("resolveCheckpointForResume", () => { webSearch: decisionUnset(), messaging: decisionUnset(), resourceProfile: decisionUnset(), + gatewayAuthority: decisionUnset(), effectGroups: {}, bindings: { credentialEnvs: [], registeredProviders: [] }, }; diff --git a/src/lib/state/onboard-checkpoint-migrate.ts b/src/lib/state/onboard-checkpoint-migrate.ts index b8d404b8a19..99774350ca0 100644 --- a/src/lib/state/onboard-checkpoint-migrate.ts +++ b/src/lib/state/onboard-checkpoint-migrate.ts @@ -77,6 +77,7 @@ export function deriveCheckpointFromSession(session: Session): OnboardCheckpoint webSearch: webSearchDecision(session), messaging: messagingDecision(session), resourceProfile: resourceDecision(session), + gatewayAuthority: decisionUnset(), effectGroups: {}, bindings: { // Provider/inference resume owns and revalidates the primary inference @@ -99,7 +100,7 @@ export function resolveCheckpointForResume(rawSession: unknown): CheckpointLoadR const session = normalizeSession(rawSession as JsonValue); if (!session) return { status: "none" }; - if (inspected.status === "loaded") { + if (inspected.status === "loaded" || inspected.status === "migrated") { // A checkpoint copied from another session's file would otherwise supply // identity, bindings, and effect receipts for the wrong onboarding run. if (inspected.checkpoint.sessionId !== session.sessionId) return { status: "corrupt" }; diff --git a/src/lib/state/onboard-checkpoint-types.ts b/src/lib/state/onboard-checkpoint-types.ts index 21af3f595e5..837bec3c6e0 100644 --- a/src/lib/state/onboard-checkpoint-types.ts +++ b/src/lib/state/onboard-checkpoint-types.ts @@ -4,7 +4,7 @@ import type { WebSearchConfig } from "../inference/web-search"; import type { OnboardMachineState } from "../onboard/machine/types"; -export const CHECKPOINT_SCHEMA_VERSION = 1 as const; +export const CHECKPOINT_SCHEMA_VERSION = 2 as const; export type CheckpointSchemaVersion = typeof CHECKPOINT_SCHEMA_VERSION; @@ -45,6 +45,24 @@ export interface CheckpointProviderBinding { readonly credentialEnv: string; } +export interface CheckpointGatewaySupervisor { + readonly kind: "systemd-system" | "systemd-user"; + readonly serviceName: string; + readonly execPath: string; +} + +/** Secret-free lifecycle authority bound to one canonical gateway name and port. */ +export interface CheckpointGatewayAuthority { + readonly gatewayName: string; + readonly gatewayPort: number; + readonly mode: "nemoclaw-managed" | "externally-supervised"; + readonly source: "declared" | "packaged-service" | "standalone"; + readonly endpoint: string | null; + readonly stateDir: string | null; + readonly supervisor: CheckpointGatewaySupervisor | null; + readonly requiredCapabilities: readonly string[]; +} + export interface CheckpointBindings { readonly credentialEnvs: readonly string[]; readonly registeredProviders: readonly CheckpointProviderBinding[]; @@ -59,6 +77,7 @@ export interface OnboardCheckpoint { readonly webSearch: CheckpointDecision; readonly messaging: CheckpointDecision; readonly resourceProfile: CheckpointDecision; + readonly gatewayAuthority: CheckpointDecision; readonly effectGroups: Readonly< Partial> >; diff --git a/src/lib/state/onboard-checkpoint.test.ts b/src/lib/state/onboard-checkpoint.test.ts index c87811a1757..1ddbdbaa889 100644 --- a/src/lib/state/onboard-checkpoint.test.ts +++ b/src/lib/state/onboard-checkpoint.test.ts @@ -28,6 +28,7 @@ function baseCheckpoint(overrides: Partial = {}): OnboardChec webSearch: decisionUnset(), messaging: decisionUnset(), resourceProfile: decisionDeclined(), + gatewayAuthority: decisionUnset(), effectGroups: { sandbox_create: { completedAt: ISO, fingerprint: "fp-create" } }, bindings: { credentialEnvs: ["OPENAI_API_KEY"], @@ -92,12 +93,72 @@ describe("checkpoint schema inspection", () => { }); }); - it("loads and round-trips a valid v1 checkpoint", () => { + it("loads and round-trips a valid v2 checkpoint", () => { const checkpoint = baseCheckpoint(); const result = inspectCheckpoint(serializeCheckpoint(checkpoint)); expect(result).toEqual({ status: "loaded", checkpoint }); }); + it("migrates a valid v1 checkpoint with an unset gateway authority", () => { + const serialized = serializeCheckpoint(baseCheckpoint()); + serialized.schemaVersion = 1; + delete serialized.gatewayAuthority; + + const result = inspectCheckpoint(serialized); + + expect(result).toMatchObject({ status: "migrated", fromVersion: 1 }); + expect(result.status === "migrated" && result.checkpoint.gatewayAuthority).toEqual( + decisionUnset(), + ); + }); + + it("round-trips a selected externally supervised gateway authority", () => { + const checkpoint = baseCheckpoint({ + gatewayAuthority: decisionSelected({ + gatewayName: "nemoclaw", + gatewayPort: 8080, + mode: "externally-supervised", + source: "declared", + endpoint: "https://127.0.0.1:8080", + stateDir: "/var/lib/openshell/gateway", + supervisor: { + kind: "systemd-system", + serviceName: "openshell-gateway.service", + execPath: "/usr/local/bin/openshell-gateway", + }, + requiredCapabilities: ["gateway.health"], + }), + }); + + expect(inspectCheckpoint(serializeCheckpoint(checkpoint))).toEqual({ + status: "loaded", + checkpoint, + }); + }); + + it("rejects a checkpoint whose external authority targets a different port", () => { + const serialized = serializeCheckpoint( + baseCheckpoint({ + gatewayAuthority: decisionSelected({ + gatewayName: "nemoclaw", + gatewayPort: 8080, + mode: "externally-supervised", + source: "declared", + endpoint: "http://127.0.0.1:9443", + stateDir: "/var/lib/openshell/gateway", + supervisor: { + kind: "systemd-system", + serviceName: "openshell-gateway.service", + execPath: "/usr/local/bin/openshell-gateway", + }, + requiredCapabilities: [], + }), + }), + ); + + expect(inspectCheckpoint(serialized)).toEqual({ status: "corrupt" }); + }); + it("rejects a checkpoint whose sandbox identity value is malformed", () => { const checkpoint = serializeCheckpoint(baseCheckpoint()); (checkpoint as Record).sandboxIdentity = { diff --git a/src/lib/state/onboard-checkpoint.ts b/src/lib/state/onboard-checkpoint.ts index 40558766034..07134d22ba7 100644 --- a/src/lib/state/onboard-checkpoint.ts +++ b/src/lib/state/onboard-checkpoint.ts @@ -1,9 +1,12 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import path from "node:path"; import { isObjectRecord } from "../core/json-types"; +import { DEFAULT_GATEWAY_PORT } from "../core/ports"; import { normalizeWebSearchConfig, type WebSearchConfig } from "../inference/web-search"; import { NAME_MAX_LENGTH, NAME_VALID_PATTERN } from "../name-validation"; +import { SUPPORTED_GATEWAY_CAPABILITIES } from "../onboard/gateway-management"; import { isOnboardMachineState } from "../onboard/machine/transitions"; import { parseCheckpointDecision } from "./onboard-checkpoint-decision"; import { @@ -12,6 +15,8 @@ import { type CheckpointDecision, type CheckpointEffectGroupName, type CheckpointEffectGroupRecord, + type CheckpointGatewayAuthority, + type CheckpointGatewaySupervisor, type CheckpointLoadResult, type CheckpointMessagingSelection, type CheckpointProviderBinding, @@ -122,6 +127,104 @@ function parseProviderBindings(value: unknown): CheckpointProviderBinding[] | nu return bindings; } +function parseGatewaySupervisor(value: unknown): CheckpointGatewaySupervisor | null { + if (!isObjectRecord(value)) return null; + const kind = value.kind; + const serviceName = readString(value.serviceName); + const execPath = readString(value.execPath); + if (kind !== "systemd-system" && kind !== "systemd-user") return null; + if (!serviceName || !/^[A-Za-z0-9][A-Za-z0-9:_.@-]*\.service$/.test(serviceName)) return null; + if (!execPath || !path.isAbsolute(execPath)) return null; + return { kind, serviceName, execPath }; +} + +function parseGatewayAuthorityValue(value: unknown): CheckpointGatewayAuthority | null { + if (!isObjectRecord(value)) return null; + const gatewayName = readString(value.gatewayName); + const gatewayPort = value.gatewayPort; + const mode = value.mode; + const source = value.source; + const endpoint = value.endpoint === null ? null : readString(value.endpoint); + const stateDir = value.stateDir === null ? null : readString(value.stateDir); + const requiredCapabilities = readStringArray(value.requiredCapabilities); + if ( + !gatewayName || + !Number.isInteger(gatewayPort) || + Number(gatewayPort) < 1 || + Number(gatewayPort) > 65535 + ) { + return null; + } + const canonicalName = + gatewayPort === DEFAULT_GATEWAY_PORT ? "nemoclaw" : `nemoclaw-${String(gatewayPort)}`; + if (gatewayName !== canonicalName) return null; + if (mode !== "nemoclaw-managed" && mode !== "externally-supervised") return null; + if (source !== "declared" && source !== "packaged-service" && source !== "standalone") + return null; + if (!requiredCapabilities) return null; + if ( + requiredCapabilities.some( + (capability) => + !SUPPORTED_GATEWAY_CAPABILITIES.includes( + capability as (typeof SUPPORTED_GATEWAY_CAPABILITIES)[number], + ), + ) + ) { + return null; + } + + if (mode === "nemoclaw-managed") { + if (endpoint !== null || stateDir !== null || value.supervisor !== null) return null; + return { + gatewayName, + gatewayPort: Number(gatewayPort), + mode, + source, + endpoint: null, + stateDir: null, + supervisor: null, + requiredCapabilities, + }; + } + + if (source !== "declared" || !endpoint || !stateDir || !path.isAbsolute(stateDir)) return null; + let parsedEndpoint: URL; + try { + parsedEndpoint = new URL(endpoint); + } catch { + return null; + } + if ( + (parsedEndpoint.protocol !== "http:" && parsedEndpoint.protocol !== "https:") || + !["127.0.0.1", "[::1]", "::1"].includes(parsedEndpoint.hostname) || + parsedEndpoint.username || + parsedEndpoint.password || + parsedEndpoint.search || + parsedEndpoint.hash || + (parsedEndpoint.pathname && parsedEndpoint.pathname !== "/") + ) { + return null; + } + const endpointPort = parsedEndpoint.port + ? Number(parsedEndpoint.port) + : parsedEndpoint.protocol === "https:" + ? 443 + : 80; + if (endpointPort !== gatewayPort) return null; + const supervisor = parseGatewaySupervisor(value.supervisor); + if (!supervisor) return null; + return { + gatewayName, + gatewayPort: Number(gatewayPort), + mode, + source, + endpoint: parsedEndpoint.origin, + stateDir, + supervisor, + requiredCapabilities, + }; +} + function parseBindings(value: unknown): CheckpointBindings | null { if (!isObjectRecord(value)) return null; const credentialEnvs = readStringArray(value.credentialEnvs); @@ -137,7 +240,10 @@ function requireDecision( return parseCheckpointDecision(raw, parseValue); } -function parseCurrentSchema(value: Record): OnboardCheckpoint | null { +function parseSchema( + value: Record, + gatewayAuthorityRaw: unknown, +): OnboardCheckpoint | null { const sessionId = readString(value.sessionId); const machineState = value.machineState; const updatedAt = readCanonicalIsoTimestamp(value.updatedAt); @@ -148,9 +254,12 @@ function parseCurrentSchema(value: Record): OnboardCheckpoint | const webSearch = requireDecision(value.webSearch, parseWebSearchValue); const messaging = requireDecision(value.messaging, parseMessagingValue); const resourceProfile = requireDecision(value.resourceProfile, parseResourceProfileValue); + const gatewayAuthority = requireDecision(gatewayAuthorityRaw, parseGatewayAuthorityValue); const effectGroups = parseEffectGroups(value.effectGroups); const bindings = parseBindings(value.bindings); - if (!sandboxIdentity || !webSearch || !messaging || !resourceProfile) return null; + if (!sandboxIdentity || !webSearch || !messaging || !resourceProfile || !gatewayAuthority) { + return null; + } if (!effectGroups || !bindings) return null; return { @@ -162,6 +271,7 @@ function parseCurrentSchema(value: Record): OnboardCheckpoint | webSearch, messaging, resourceProfile, + gatewayAuthority, effectGroups, bindings, }; @@ -179,9 +289,13 @@ export function inspectCheckpoint(raw: unknown): CheckpointLoadResult { return { status: "unsupported_future", foundVersion: version }; } if (version === CHECKPOINT_SCHEMA_VERSION) { - const checkpoint = parseCurrentSchema(raw); + const checkpoint = parseSchema(raw, raw.gatewayAuthority); return checkpoint ? { status: "loaded", checkpoint } : { status: "corrupt" }; } + if (version === 1) { + const checkpoint = parseSchema(raw, { kind: "unset" }); + return checkpoint ? { status: "migrated", checkpoint, fromVersion: 1 } : { status: "corrupt" }; + } return { status: "corrupt" }; } @@ -195,6 +309,7 @@ export function serializeCheckpoint(checkpoint: OnboardCheckpoint): Record Date: Wed, 22 Jul 2026 09:41:55 -0400 Subject: [PATCH 02/20] fix(onboard): harden external gateway identity checks Signed-off-by: Julie Yaunches --- .../gateway-lifecycle-authority.mdx | 9 ++- src/lib/core/gateway-capabilities.ts | 18 +++++ .../gateway-authority-checkpoint.test.ts | 10 ++- .../onboard/gateway-authority-checkpoint.ts | 6 +- src/lib/onboard/gateway-host-runtime.test.ts | 41 +++++++++++ src/lib/onboard/gateway-host-runtime.ts | 68 +++++++++++++++---- src/lib/onboard/gateway-management.ts | 24 +++---- src/lib/onboard/gateway-ownership.test.ts | 7 ++ src/lib/onboard/gateway-ownership.ts | 14 ++-- src/lib/state/onboard-checkpoint.ts | 2 +- 10 files changed, 155 insertions(+), 44 deletions(-) create mode 100644 src/lib/core/gateway-capabilities.ts diff --git a/docs/deployment/gateway-lifecycle-authority.mdx b/docs/deployment/gateway-lifecycle-authority.mdx index 0c1d729ae7f..4976a719665 100644 --- a/docs/deployment/gateway-lifecycle-authority.mdx +++ b/docs/deployment/gateway-lifecycle-authority.mdx @@ -106,7 +106,11 @@ For an HTTP endpoint, NemoClaw sends the readiness request to the exact declared ## Validate and attach the gateway NemoClaw validates external authority before provider, policy, sandbox, or sandbox-registry mutations. -It rejects a systemd unit that reports inactive and checks that exactly one recognizable process holds the configured port, the process belongs to the unit's cgroup, the executable matches `execPath`, and the exact endpoint answers its health check. +NemoClaw requires systemd to confirm that the declared unit is active. +It checks that exactly one recognizable process holds the configured port. +The process identity must remain stable while NemoClaw reads `/proc` evidence and confirms the listener a second time. +The process must belong to the unit's cgroup, and its executable must match `execPath`. +The exact endpoint must answer its health check. After validation, NemoClaw registers the exact endpoint under the canonical gateway name and selects it as the active OpenShell gateway. Port `8080` uses the name `nemoclaw`, and another port uses `nemoclaw-`. @@ -117,11 +121,12 @@ Onboarding fails when any of the following conditions applies: - The endpoint port differs from the gateway port for this process. - A required capability is unsupported. -- The declared supervisor is inactive. +- The declared supervisor is inactive, or systemd cannot confirm its active state. - Nothing is listening on the declared endpoint. - The listener set cannot be enumerated completely. - No recognizable gateway process holds the port. - More than one process holds the port. +- The listener process or listener set changes during identity verification. - The listener does not belong to the declared systemd unit. - The listener executable does not match `execPath`. - The endpoint does not answer the required health check. diff --git a/src/lib/core/gateway-capabilities.ts b/src/lib/core/gateway-capabilities.ts new file mode 100644 index 00000000000..c176af187a3 --- /dev/null +++ b/src/lib/core/gateway-capabilities.ts @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Gateway capabilities implemented by this NemoClaw build. + * + * This domain allowlist is shared by declaration parsing, runtime ownership + * checks, and durable checkpoint validation. Keeping it in core prevents state + * restoration from depending on onboarding orchestration. + */ +export const SUPPORTED_GATEWAY_CAPABILITIES = [ + "gateway.health", + "sandbox.create", + "sandbox.exec", + "gpu.passthrough", +] as const; + +export type GatewayCapability = (typeof SUPPORTED_GATEWAY_CAPABILITIES)[number]; diff --git a/src/lib/onboard/gateway-authority-checkpoint.test.ts b/src/lib/onboard/gateway-authority-checkpoint.test.ts index d0ed11c4870..fe3565c47a3 100644 --- a/src/lib/onboard/gateway-authority-checkpoint.test.ts +++ b/src/lib/onboard/gateway-authority-checkpoint.test.ts @@ -50,22 +50,20 @@ describe("durable gateway lifecycle authority", () => { expect(bindGatewayAuthorityToCheckpoint(session, owner)).toEqual(owner); const authority = session.checkpoint?.gatewayAuthority; - expect(authority).toBeDefined(); - if (!authority) throw new Error("expected gateway authority checkpoint"); - expect(isDecisionSelected(authority)).toBe(true); - expect(authority.kind === "selected" && authority.value).toEqual( - checkpointGatewayAuthority(owner), - ); + expect(authority).toEqual({ kind: "selected", value: checkpointGatewayAuthority(owner) }); + expect(isDecisionSelected(authority!)).toBe(true); }); it("accepts the same authority after a process resume round-trip (#6576)", () => { const firstProcess = createSession({ sessionId: "authority-session" }); const owner = externalOwner(); bindGatewayAuthorityToCheckpoint(firstProcess, owner); + const checkpointUpdatedAt = firstProcess.checkpoint?.updatedAt; const resumed = normalizeSession(JSON.parse(JSON.stringify(firstProcess)) as never); expect(resumed).not.toBeNull(); expect(bindGatewayAuthorityToCheckpoint(resumed!, owner)).toEqual(owner); + expect(resumed!.checkpoint?.updatedAt).toBe(checkpointUpdatedAt); }); it("rejects external-to-managed drift after a process resume before effects (#6576)", () => { diff --git a/src/lib/onboard/gateway-authority-checkpoint.ts b/src/lib/onboard/gateway-authority-checkpoint.ts index e0611b7612e..d29da93dc41 100644 --- a/src/lib/onboard/gateway-authority-checkpoint.ts +++ b/src/lib/onboard/gateway-authority-checkpoint.ts @@ -1,15 +1,15 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import type { GatewayCapability } from "../core/gateway-capabilities"; import { + decisionSelected, isDecisionDeclined, isDecisionSelected, - decisionSelected, } from "../state/onboard-checkpoint-decision"; +import { deriveCheckpointFromSession } from "../state/onboard-checkpoint-migrate"; import type { CheckpointGatewayAuthority } from "../state/onboard-checkpoint-types"; import type { Session } from "../state/onboard-session"; -import { deriveCheckpointFromSession } from "../state/onboard-checkpoint-migrate"; -import type { GatewayCapability } from "./gateway-management"; import { describeGatewayOwnerForError, type GatewayOwner, diff --git a/src/lib/onboard/gateway-host-runtime.test.ts b/src/lib/onboard/gateway-host-runtime.test.ts index 31c116cd15e..817f260ec2e 100644 --- a/src/lib/onboard/gateway-host-runtime.test.ts +++ b/src/lib/onboard/gateway-host-runtime.test.ts @@ -59,6 +59,7 @@ function createDeps(overrides: Partial = {}): GatewayHos // test overwriting them. readProcExe: () => SYSTEMD_GATEWAY_EXEC, readProcCgroup: () => `0::/system.slice/${DECLARATION.supervisor.serviceName}\n`, + readProcStartTime: () => "710024", waitForGatewayHttpReady: async () => true, ...overrides, }; @@ -236,6 +237,42 @@ describe("gateway host runtime attachment probe", () => { }); }); + it("fails closed when the listener PID changes during identity verification (#6576)", async () => { + declareExternalSupervision(); + const getGatewayPortListenerRawScan = vi + .fn() + .mockReturnValueOnce({ pids: [SYSTEMD_GATEWAY_PID], complete: true }) + .mockReturnValueOnce({ pids: [4343], complete: true }); + const runtime = createGatewayHostRuntime(createDeps({ getGatewayPortListenerRawScan })); + const owner = runtime.getGatewayOwner(); + + const probe = await runtime.probeGatewayAttachment(owner); + + expect(probe.listenerPids).toEqual([4343]); + expect(probe.listenerExecPath).toBeNull(); + expect(probe.listenerSupervisorMatch).toBeNull(); + expect(evaluateGatewayAttachment(owner, probe)).toMatchObject({ + ok: false, + code: "unknown_listener", + }); + }); + + it("fails closed when a PID is reused while its identity is read (#6576)", async () => { + declareExternalSupervision(); + const readProcStartTime = vi.fn().mockReturnValueOnce("710024").mockReturnValueOnce("710025"); + const runtime = createGatewayHostRuntime(createDeps({ readProcStartTime })); + const owner = runtime.getGatewayOwner(); + + const probe = await runtime.probeGatewayAttachment(owner); + + expect(probe.listenerExecPath).toBeNull(); + expect(probe.listenerSupervisorMatch).toBeNull(); + expect(evaluateGatewayAttachment(owner, probe)).toMatchObject({ + ok: false, + code: "unknown_listener", + }); + }); + it("reports an unprobeable supervisor rather than guessing (#6576)", async () => { declareExternalSupervision(); const runtime = createGatewayHostRuntime( @@ -247,6 +284,10 @@ describe("gateway host runtime attachment probe", () => { const probe = await runtime.probeGatewayAttachment(runtime.getGatewayOwner()); expect(probe.supervisorActive).toBeNull(); + expect(evaluateGatewayAttachment(runtime.getGatewayOwner(), probe)).toMatchObject({ + ok: false, + code: "supervisor_inactive", + }); }); it("reads the authoritative gateway port lazily, not at construction (#6576)", () => { diff --git a/src/lib/onboard/gateway-host-runtime.ts b/src/lib/onboard/gateway-host-runtime.ts index 5bb80c003f3..243252eee16 100644 --- a/src/lib/onboard/gateway-host-runtime.ts +++ b/src/lib/onboard/gateway-host-runtime.ts @@ -73,6 +73,8 @@ export interface GatewayHostRuntimeDeps { readProcExe?(pid: number): string | null; /** Overrides `/proc//cgroup` reads; defaults to the real file. */ readProcCgroup?(pid: number): string | null; + /** Overrides `/proc//stat` start-time reads; defaults to the real file. */ + readProcStartTime?(pid: number): string | null; waitForGatewayHttpReady(): Promise; } @@ -176,6 +178,23 @@ export function createGatewayHostRuntime(deps: GatewayHostRuntimeDeps): GatewayH } } + function readProcStartTime(pid: number): string | null { + if (deps.readProcStartTime) return deps.readProcStartTime(pid); + try { + const stat = fs.readFileSync(`/proc/${pid}/stat`, "utf-8"); + const commandEnd = stat.lastIndexOf(")"); + // Fields after the command begin at field 3; process start time is field 22. + return commandEnd >= 0 + ? (stat + .slice(commandEnd + 1) + .trim() + .split(/\s+/)[19] ?? null) + : null; + } catch { + return null; + } + } + /** * Bind a listening PID to the declared supervisor unit via cgroup membership * — the authoritative evidence that the process is the unit's, not merely a @@ -183,17 +202,23 @@ export function createGatewayHostRuntime(deps: GatewayHostRuntimeDeps): GatewayH * cannot be established (no PID or an unreadable cgroup), and the caller then * fails closed. */ - function readListenerSupervisorMatch( - owner: GatewayOwner, - pid: number | undefined, - ): boolean | null { + function readListenerSupervisorMatch(owner: GatewayOwner, pid: number): boolean | null { const supervisor = owner.supervisor; - if (!supervisor || typeof pid !== "number") return null; + if (!supervisor) return null; const cgroupText = readProcCgroup(pid); if (cgroupText === null) return null; return cgroupBelongsToUnit(cgroupText, supervisor.serviceName); } + /** Read executable and cgroup evidence only while the PID names one process. */ + function readStableListenerIdentity(owner: GatewayOwner, pid: number) { + const startTime = readProcStartTime(pid); + const execPath = readListenerExecPath(pid); + const supervisorMatch = readListenerSupervisorMatch(owner, pid); + const endTime = readProcStartTime(pid); + return startTime !== null && endTime === startTime ? { execPath, supervisorMatch } : null; + } + /** * Probe the declared endpoint rather than the process default, so a * declaration is never assessed against a different local listener. A @@ -222,20 +247,39 @@ export function createGatewayHostRuntime(deps: GatewayHostRuntimeDeps): GatewayH */ async function probeGatewayAttachment(owner: GatewayOwner): Promise { const portCheck = await deps.checkGatewayPortAvailable(); + const httpReady = await waitForDeclaredGatewayHttpReady(owner); + const supervisorActive = isSupervisorUnitActive(owner); + const gatewayBin = deps.resolveOpenShellGatewayBinary(); const scan = deps.getGatewayPortListenerRawScan(portCheck, { - gatewayBin: deps.resolveOpenShellGatewayBinary(), + gatewayBin, }); const [firstPid] = scan.pids; + const identity = + scan.complete && scan.pids.length === 1 && typeof firstPid === "number" + ? readStableListenerIdentity(owner, firstPid) + : null; + const verifiedScan = + typeof firstPid === "number" + ? deps.getGatewayPortListenerRawScan(portCheck, { + gatewayBin, + }) + : scan; + const listenerStayedStable = + scan.complete && + verifiedScan.complete && + scan.pids.length === 1 && + verifiedScan.pids.length === 1 && + verifiedScan.pids[0] === firstPid; return { gatewayPort: deps.gatewayPort(), - httpReady: await waitForDeclaredGatewayHttpReady(owner), + httpReady, // `ok` means the port is free; anything else means something holds it. portOccupied: !portCheck.ok, - listenerPids: scan.pids, - listenerScanComplete: scan.complete, - supervisorActive: isSupervisorUnitActive(owner), - listenerExecPath: typeof firstPid === "number" ? readListenerExecPath(firstPid) : null, - listenerSupervisorMatch: readListenerSupervisorMatch(owner, firstPid), + listenerPids: verifiedScan.pids, + listenerScanComplete: scan.complete && verifiedScan.complete, + supervisorActive, + listenerExecPath: listenerStayedStable ? (identity?.execPath ?? null) : null, + listenerSupervisorMatch: listenerStayedStable ? (identity?.supervisorMatch ?? null) : null, }; } diff --git a/src/lib/onboard/gateway-management.ts b/src/lib/onboard/gateway-management.ts index 45f8942ba09..ef14e2c9341 100644 --- a/src/lib/onboard/gateway-management.ts +++ b/src/lib/onboard/gateway-management.ts @@ -22,6 +22,15 @@ import fs from "node:fs"; import path from "node:path"; +import { + type GatewayCapability, + SUPPORTED_GATEWAY_CAPABILITIES, +} from "../core/gateway-capabilities"; + +export { + type GatewayCapability, + SUPPORTED_GATEWAY_CAPABILITIES, +} from "../core/gateway-capabilities"; /** Bump only for a breaking change to the declaration shape. */ export const GATEWAY_MANAGEMENT_CONTRACT_VERSION = 1; @@ -31,21 +40,6 @@ export const GATEWAY_MANAGEMENT_ENV_VAR = "NEMOCLAW_GATEWAY_MANAGEMENT"; export type GatewayManagementMode = "nemoclaw-managed" | "externally-supervised"; -/** - * Capabilities a gateway must expose for canonical onboarding to attach to it. - * A declaration requiring a capability NemoClaw does not implement fails closed - * rather than attaching and discovering the gap after provider or sandbox - * effects have already run. - */ -export const SUPPORTED_GATEWAY_CAPABILITIES = [ - "gateway.health", - "sandbox.create", - "sandbox.exec", - "gpu.passthrough", -] as const; - -export type GatewayCapability = (typeof SUPPORTED_GATEWAY_CAPABILITIES)[number]; - /** How the external supervisor runs the gateway, and how to recognize it. */ /** * Supervisor kinds NemoClaw can authoritatively bind a listening PID to. Only diff --git a/src/lib/onboard/gateway-ownership.test.ts b/src/lib/onboard/gateway-ownership.test.ts index 492980d7b0f..eaa9e37bed4 100644 --- a/src/lib/onboard/gateway-ownership.test.ts +++ b/src/lib/onboard/gateway-ownership.test.ts @@ -166,6 +166,13 @@ describe("externally supervised gateway attachment", () => { ); }); + it("fails closed when the declared supervisor status cannot be determined (#6576)", () => { + const result = evaluateGatewayAttachment(externalOwner, probe({ supervisorActive: null })); + + expect(result).toMatchObject({ ok: false, code: "supervisor_inactive" }); + expect(result.ok === false && result.message).toMatch(/could not be confirmed active/); + }); + it("fails instead of launching a gateway when nothing holds the port (#6576)", () => { const result = evaluateGatewayAttachment( externalOwner, diff --git a/src/lib/onboard/gateway-ownership.ts b/src/lib/onboard/gateway-ownership.ts index e9740d2ae37..be61965434c 100644 --- a/src/lib/onboard/gateway-ownership.ts +++ b/src/lib/onboard/gateway-ownership.ts @@ -26,14 +26,16 @@ * Gateway-side discovery can replace this check when OpenShell reports it. */ +import { + type GatewayCapability, + SUPPORTED_GATEWAY_CAPABILITIES, +} from "../core/gateway-capabilities"; import type { JsonObject } from "../core/json-types"; import { redactUrl } from "../security/redact"; import { - type GatewayCapability, type GatewayManagementDeclaration, type GatewayManagementMode, type GatewaySupervisorDeclaration, - SUPPORTED_GATEWAY_CAPABILITIES, } from "./gateway-management"; /** @@ -234,7 +236,7 @@ export interface GatewayAttachmentProbe { httpReady: boolean; /** Anything at all holds the gateway port. */ portOccupied: boolean; - /** Identity-verified gateway processes listening on the port. */ + /** All processes observed listening on the port; identity is established separately. */ listenerPids: readonly number[]; /** False when the listener set could not be authoritatively enumerated. */ listenerScanComplete: boolean; @@ -343,12 +345,14 @@ export function evaluateGatewayAttachment( }; } - if (probe.supervisorActive === false) { + if (probe.supervisorActive !== true) { + const status = + probe.supervisorActive === false ? "is not active" : "could not be confirmed active"; return { ok: false, code: "supervisor_inactive", message: - `${supervisorName} is not active, and NemoClaw does not start an externally supervised gateway. ` + + `${supervisorName} ${status}, and NemoClaw does not start an externally supervised gateway. ` + `Start it through the platform supervisor, then re-run onboarding.`, }; } diff --git a/src/lib/state/onboard-checkpoint.ts b/src/lib/state/onboard-checkpoint.ts index 07134d22ba7..eb87c63b5d6 100644 --- a/src/lib/state/onboard-checkpoint.ts +++ b/src/lib/state/onboard-checkpoint.ts @@ -2,11 +2,11 @@ // SPDX-License-Identifier: Apache-2.0 import path from "node:path"; +import { SUPPORTED_GATEWAY_CAPABILITIES } from "../core/gateway-capabilities"; import { isObjectRecord } from "../core/json-types"; import { DEFAULT_GATEWAY_PORT } from "../core/ports"; import { normalizeWebSearchConfig, type WebSearchConfig } from "../inference/web-search"; import { NAME_MAX_LENGTH, NAME_VALID_PATTERN } from "../name-validation"; -import { SUPPORTED_GATEWAY_CAPABILITIES } from "../onboard/gateway-management"; import { isOnboardMachineState } from "../onboard/machine/transitions"; import { parseCheckpointDecision } from "./onboard-checkpoint-decision"; import { From c5bd3559cb1831f7b4d5fb24cf28ae5767498705 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 22 Jul 2026 11:53:21 -0400 Subject: [PATCH 03/20] fix(onboard): revalidate external gateway attachment Signed-off-by: Julie Yaunches --- .../gateway-lifecycle-authority.mdx | 3 +- src/lib/onboard/gateway-host-runtime.test.ts | 44 +++++++++-- src/lib/onboard/gateway-host-runtime.ts | 74 +++++++++++++++---- .../onboard/machine/handlers/gateway.test.ts | 13 +++- src/lib/onboard/machine/handlers/gateway.ts | 7 +- 5 files changed, 113 insertions(+), 28 deletions(-) diff --git a/docs/deployment/gateway-lifecycle-authority.mdx b/docs/deployment/gateway-lifecycle-authority.mdx index 4976a719665..2314696083f 100644 --- a/docs/deployment/gateway-lifecycle-authority.mdx +++ b/docs/deployment/gateway-lifecycle-authority.mdx @@ -115,7 +115,8 @@ The exact endpoint must answer its health check. After validation, NemoClaw registers the exact endpoint under the canonical gateway name and selects it as the active OpenShell gateway. Port `8080` uses the name `nemoclaw`, and another port uses `nemoclaw-`. NemoClaw verifies the named and active registrations before advancing to provider selection. -If registration, selection, or verification fails, onboarding stops instead of continuing with a different gateway. +It then repeats the health, supervisor, listener, executable, cgroup, and process identity checks and compares them with the initial evidence. +If registration, selection, or repeated validation fails, NemoClaw removes the attempted registration and stops before provider selection. Onboarding fails when any of the following conditions applies: diff --git a/src/lib/onboard/gateway-host-runtime.test.ts b/src/lib/onboard/gateway-host-runtime.test.ts index 817f260ec2e..61fe0cada3c 100644 --- a/src/lib/onboard/gateway-host-runtime.test.ts +++ b/src/lib/onboard/gateway-host-runtime.test.ts @@ -299,13 +299,15 @@ describe("gateway host runtime attachment probe", () => { expect(runtime.getGatewayStartEnv()).toMatchObject({ OPENSHELL_SERVER_PORT: "9443" }); }); - it("registers and selects the exact declared endpoint without prior gateway metadata (#6576)", () => { + it("registers and selects the exact declared endpoint without prior gateway metadata (#6576)", async () => { declareExternalSupervision(); process.env.OPENSHELL_GATEWAY = "ambient-sibling"; const runOpenshell = vi.fn((_args: string[]) => ({ status: 0 })); const runtime = createGatewayHostRuntime(createDeps({ runOpenshell })); - runtime.attachGateway(runtime.getGatewayOwner()); + const owner = runtime.getGatewayOwner(); + const expectedProbe = await runtime.probeGatewayAttachment(owner); + await runtime.attachGateway(owner, expectedProbe); expect(runOpenshell.mock.calls).toEqual([ [ @@ -317,13 +319,15 @@ describe("gateway host runtime attachment probe", () => { expect(process.env.OPENSHELL_GATEWAY).toBe("nemoclaw"); }); - it("replaces stale registration before selecting the declared endpoint (#6576)", () => { + it("replaces stale registration before selecting the declared endpoint (#6576)", async () => { declareExternalSupervision(); const statuses = [1, 0, 0]; const runOpenshell = vi.fn((_args: string[]) => ({ status: statuses.shift() ?? 0 })); const runtime = createGatewayHostRuntime(createDeps({ runOpenshell })); - runtime.attachGateway(runtime.getGatewayOwner()); + const owner = runtime.getGatewayOwner(); + const expectedProbe = await runtime.probeGatewayAttachment(owner); + await runtime.attachGateway(owner, expectedProbe); expect(runOpenshell.mock.calls.map(([args]) => args)).toEqual([ ["gateway", "add", "http://127.0.0.1:8080", "--local", "--name", "nemoclaw"], @@ -333,14 +337,17 @@ describe("gateway host runtime attachment probe", () => { ]); }); - it("removes the attempted registration when exact gateway selection is unhealthy (#6576)", () => { + it("removes the attempted registration when exact gateway selection is unhealthy (#6576)", async () => { declareExternalSupervision(); const runOpenshell = vi.fn((_args: string[]) => ({ status: 0 })); const runtime = createGatewayHostRuntime( createDeps({ isGatewayHealthy: () => false, runOpenshell }), ); - expect(() => runtime.attachGateway(runtime.getGatewayOwner())).toThrow( + const owner = runtime.getGatewayOwner(); + const expectedProbe = await runtime.probeGatewayAttachment(owner); + + await expect(runtime.attachGateway(owner, expectedProbe)).rejects.toThrow( /Failed to register and select/, ); expect(runOpenshell).toHaveBeenLastCalledWith(["gateway", "remove", "nemoclaw"], { @@ -349,4 +356,29 @@ describe("gateway host runtime attachment probe", () => { }); expect(process.env.OPENSHELL_GATEWAY).toBeUndefined(); }); + + it("removes the registration when the listener changes during attachment (#6576)", async () => { + declareExternalSupervision(); + const getGatewayPortListenerRawScan = vi + .fn() + .mockReturnValueOnce({ pids: [SYSTEMD_GATEWAY_PID], complete: true }) + .mockReturnValueOnce({ pids: [SYSTEMD_GATEWAY_PID], complete: true }) + .mockReturnValueOnce({ pids: [4343], complete: true }) + .mockReturnValueOnce({ pids: [4343], complete: true }); + const runOpenshell = vi.fn((_args: string[]) => ({ status: 0 })); + const runtime = createGatewayHostRuntime( + createDeps({ getGatewayPortListenerRawScan, runOpenshell }), + ); + const owner = runtime.getGatewayOwner(); + const expectedProbe = await runtime.probeGatewayAttachment(owner); + + await expect(runtime.attachGateway(owner, expectedProbe)).rejects.toMatchObject({ + code: "identity_mismatch", + }); + expect(runOpenshell).toHaveBeenLastCalledWith(["gateway", "remove", "nemoclaw"], { + ignoreError: true, + suppressOutput: true, + }); + expect(process.env.OPENSHELL_GATEWAY).toBeUndefined(); + }); }); diff --git a/src/lib/onboard/gateway-host-runtime.ts b/src/lib/onboard/gateway-host-runtime.ts index 243252eee16..7b1b0773a9d 100644 --- a/src/lib/onboard/gateway-host-runtime.ts +++ b/src/lib/onboard/gateway-host-runtime.ts @@ -28,6 +28,7 @@ import { assertGatewayEffectAllowed, cgroupBelongsToUnit, describeGatewayOwnerForError, + evaluateGatewayAttachment, type GatewayAttachmentProbe, type GatewayOwner, GatewayOwnershipError, @@ -84,7 +85,7 @@ export interface GatewayHostRuntime { * owns. Applies to onboarding, rebuild, and recovery alike. */ assertGatewayStartAllowed(exitOnFailure: boolean): void; - attachGateway(owner: GatewayOwner): void; + attachGateway(owner: GatewayOwner, expectedProbe: GatewayAttachmentProbe): Promise; bindGatewayOwner(owner: GatewayOwner): void; /** HTTPS endpoint of the gateway this process operates. */ getGatewayLocalEndpoint(): string; @@ -97,7 +98,7 @@ export interface GatewayHostRuntime { machineGatewayOwnerDeps: { probeGatewayAttachment(owner: GatewayOwner): Promise; resolveGatewayOwner(): GatewayOwner; - attachGateway(owner: GatewayOwner): void; + attachGateway(owner: GatewayOwner, expectedProbe: GatewayAttachmentProbe): Promise; }; probeGatewayAttachment(owner: GatewayOwner): Promise; } @@ -314,10 +315,43 @@ export function createGatewayHostRuntime(deps: GatewayHostRuntimeDeps): GatewayH process.env.OPENSHELL_LOCAL_TLS_DIR = localTlsDir; } + function sameAttachmentEvidence( + expected: GatewayAttachmentProbe, + actual: GatewayAttachmentProbe, + ): boolean { + return ( + expected.gatewayPort === actual.gatewayPort && + expected.httpReady === actual.httpReady && + expected.portOccupied === actual.portOccupied && + expected.listenerScanComplete === actual.listenerScanComplete && + expected.supervisorActive === actual.supervisorActive && + expected.listenerExecPath === actual.listenerExecPath && + expected.listenerSupervisorMatch === actual.listenerSupervisorMatch && + expected.listenerPids.length === actual.listenerPids.length && + expected.listenerPids.every((pid, index) => pid === actual.listenerPids[index]) + ); + } + /** Register and select the exact endpoint whose listener identity was validated. */ - function attachGateway(owner: GatewayOwner): void { + async function attachGateway( + owner: GatewayOwner, + expectedProbe: GatewayAttachmentProbe, + ): Promise { if (!isExternallySupervised(owner) || !owner.endpoint) return; + const expectedAttachment = evaluateGatewayAttachment(owner, expectedProbe); + if (!expectedAttachment.ok) { + throw new GatewayOwnershipError(expectedAttachment.code, expectedAttachment.message, owner); + } prepareExternalGatewayClient(owner); + const removeAttemptedRegistration = () => { + deps.runOpenshell(["gateway", "remove", owner.gatewayName], { + ignoreError: true, + suppressOutput: true, + }); + if (process.env.OPENSHELL_GATEWAY === owner.gatewayName) { + delete process.env.OPENSHELL_GATEWAY; + } + }; const add = () => deps.runOpenshell( ["gateway", "add", owner.endpoint as string, "--local", "--name", owner.gatewayName], @@ -325,10 +359,7 @@ export function createGatewayHostRuntime(deps: GatewayHostRuntimeDeps): GatewayH ); let addResult = add(); if (addResult.status !== 0) { - deps.runOpenshell(["gateway", "remove", owner.gatewayName], { - ignoreError: true, - suppressOutput: true, - }); + removeAttemptedRegistration(); addResult = add(); } const selectResult = deps.runOpenshell(["gateway", "select", owner.gatewayName], { @@ -345,19 +376,34 @@ export function createGatewayHostRuntime(deps: GatewayHostRuntimeDeps): GatewayH selectResult.status !== 0 || !(deps.isGatewayHealthy ?? isGatewayHealthy)(status, namedInfo, activeInfo, owner.gatewayName) ) { - deps.runOpenshell(["gateway", "remove", owner.gatewayName], { - ignoreError: true, - suppressOutput: true, - }); - if (process.env.OPENSHELL_GATEWAY === owner.gatewayName) { - delete process.env.OPENSHELL_GATEWAY; - } + removeAttemptedRegistration(); throw new GatewayOwnershipError( "gateway_registration_failed", `Failed to register and select externally supervised gateway '${owner.gatewayName}' at ${owner.endpoint}.`, owner, ); } + + let currentProbe: GatewayAttachmentProbe; + try { + currentProbe = await probeGatewayAttachment(owner); + } catch (error) { + removeAttemptedRegistration(); + throw error; + } + const currentAttachment = evaluateGatewayAttachment(owner, currentProbe); + if (!currentAttachment.ok || !sameAttachmentEvidence(expectedProbe, currentProbe)) { + removeAttemptedRegistration(); + if (!currentAttachment.ok) { + throw new GatewayOwnershipError(currentAttachment.code, currentAttachment.message, owner); + } + throw new GatewayOwnershipError( + "identity_mismatch", + `The externally supervised gateway listener changed while '${owner.gatewayName}' was registered. ` + + "The registration was removed; stabilize the supervisor and re-run onboarding.", + owner, + ); + } process.env.OPENSHELL_GATEWAY = owner.gatewayName; } diff --git a/src/lib/onboard/machine/handlers/gateway.test.ts b/src/lib/onboard/machine/handlers/gateway.test.ts index de755406323..c63d4a22e88 100644 --- a/src/lib/onboard/machine/handlers/gateway.test.ts +++ b/src/lib/onboard/machine/handlers/gateway.test.ts @@ -72,7 +72,7 @@ function createDeps(overrides: Partial["deps"]> = {}) { hasPackagedService: false, }), ), - attachGateway: vi.fn(), + attachGateway: vi.fn(async () => undefined), probeAttachment: vi.fn( async (): Promise => ({ gatewayPort: 8080, @@ -483,7 +483,9 @@ describe("externally supervised gateway lifecycle authority", () => { it("attaches to the supervised gateway without running any lifecycle effect (#6576)", async () => { const order: string[] = []; const { calls, deps } = externalDeps(); - calls.attachGateway.mockImplementation(() => order.push("attach")); + calls.attachGateway.mockImplementation(async () => { + order.push("attach"); + }); calls.complete.mockImplementation(async () => { order.push("complete"); return createSession(); @@ -492,7 +494,10 @@ describe("externally supervised gateway lifecycle authority", () => { const result = await handleGatewayState(baseOptions(deps, "missing")); expect(calls.startGateway).not.toHaveBeenCalled(); - expect(calls.attachGateway).toHaveBeenCalledWith(EXTERNAL_OWNER); + expect(calls.attachGateway).toHaveBeenCalledWith( + EXTERNAL_OWNER, + expect.objectContaining({ listenerPids: [4242], listenerSupervisorMatch: true }), + ); expect(calls.destroy).not.toHaveBeenCalled(); expect(calls.destroyForReuse).not.toHaveBeenCalled(); expect(calls.retireLegacy).not.toHaveBeenCalled(); @@ -505,7 +510,7 @@ describe("externally supervised gateway lifecycle authority", () => { it("does not cross the provider-mutation boundary when exact registration fails (#6576)", async () => { const { calls, deps } = externalDeps(); - calls.attachGateway.mockImplementation(() => { + calls.attachGateway.mockImplementation(async () => { throw new GatewayOwnershipError( "gateway_registration_failed", "registration failed", diff --git a/src/lib/onboard/machine/handlers/gateway.ts b/src/lib/onboard/machine/handlers/gateway.ts index 1c3e2456b2d..9b98e17abbe 100644 --- a/src/lib/onboard/machine/handlers/gateway.ts +++ b/src/lib/onboard/machine/handlers/gateway.ts @@ -34,7 +34,7 @@ export interface GatewayStateOptions { */ resolveGatewayOwner(): GatewayOwner; probeGatewayAttachment(owner: GatewayOwner): Promise; - attachGateway(owner: GatewayOwner): void; + attachGateway(owner: GatewayOwner, expectedProbe: GatewayAttachmentProbe): Promise; refreshDockerDriverGatewayReuseState(state: GatewayReuseState): Promise; gatewayCliSupportsLifecycleCommands(): boolean; verifyGatewayContainerRunning(gatewayName: string): GatewayContainerState; @@ -281,11 +281,12 @@ async function attachToExternallySupervisedGateway( deps: GatewayStateOptions["deps"], ): Promise { const supervisor = owner.supervisor?.serviceName ?? "an external supervisor"; - const attachment = evaluateGatewayAttachment(owner, await deps.probeGatewayAttachment(owner)); + const probe = await deps.probeGatewayAttachment(owner); + const attachment = evaluateGatewayAttachment(owner, probe); if (!attachment.ok) { throw new GatewayOwnershipError(attachment.code, attachment.message, owner); } - deps.attachGateway(owner); + await deps.attachGateway(owner, probe); deps.skippedStepMessage("gateway", `supervised by ${supervisor}`, "reuse"); deps.note(` Attached to externally supervised OpenShell gateway (${supervisor}).`); From dfbd96d3bb3a0a8a86beac3fc28487d37b439e48 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 22 Jul 2026 12:38:55 -0400 Subject: [PATCH 04/20] fix(onboard): harden external gateway identity Signed-off-by: Julie Yaunches --- src/lib/onboard/gateway-host-runtime.test.ts | 50 +++++++++++++++++++ src/lib/onboard/gateway-host-runtime.ts | 28 +++++++++-- src/lib/onboard/gateway-management.test.ts | 18 ++++++- src/lib/onboard/gateway-management.ts | 28 +++++------ src/lib/onboard/gateway-ownership.test.ts | 1 + src/lib/onboard/gateway-ownership.ts | 2 + .../onboard/machine/handlers/gateway.test.ts | 2 + .../machine/initial-flow-phases.test.ts | 2 + 8 files changed, 110 insertions(+), 21 deletions(-) diff --git a/src/lib/onboard/gateway-host-runtime.test.ts b/src/lib/onboard/gateway-host-runtime.test.ts index 61fe0cada3c..d299222e671 100644 --- a/src/lib/onboard/gateway-host-runtime.test.ts +++ b/src/lib/onboard/gateway-host-runtime.test.ts @@ -170,6 +170,7 @@ describe("gateway host runtime attachment probe", () => { portOccupied: true, listenerPids: [SYSTEMD_GATEWAY_PID], listenerScanComplete: true, + listenerStartTime: "710024", supervisorActive: true, listenerExecPath: SYSTEMD_GATEWAY_EXEC, listenerSupervisorMatch: true, @@ -273,6 +274,28 @@ describe("gateway host runtime attachment probe", () => { }); }); + it("fails closed when the listener PID is reused between identity and port confirmation (#6576)", async () => { + declareExternalSupervision(); + const readProcStartTime = vi + .fn() + .mockReturnValueOnce("710024") + .mockReturnValueOnce("710024") + .mockReturnValueOnce("710025") + .mockReturnValueOnce("710025"); + const runtime = createGatewayHostRuntime(createDeps({ readProcStartTime })); + const owner = runtime.getGatewayOwner(); + + const probe = await runtime.probeGatewayAttachment(owner); + + expect(probe.listenerStartTime).toBeNull(); + expect(probe.listenerExecPath).toBeNull(); + expect(probe.listenerSupervisorMatch).toBeNull(); + expect(evaluateGatewayAttachment(owner, probe)).toMatchObject({ + ok: false, + code: "unknown_listener", + }); + }); + it("reports an unprobeable supervisor rather than guessing (#6576)", async () => { declareExternalSupervision(); const runtime = createGatewayHostRuntime( @@ -381,4 +404,31 @@ describe("gateway host runtime attachment probe", () => { }); expect(process.env.OPENSHELL_GATEWAY).toBeUndefined(); }); + + it("removes the registration when the listener process generation changes (#6576)", async () => { + declareExternalSupervision(); + const readProcStartTime = vi + .fn() + .mockReturnValueOnce("710024") + .mockReturnValueOnce("710024") + .mockReturnValueOnce("710024") + .mockReturnValueOnce("710024") + .mockReturnValueOnce("710025") + .mockReturnValueOnce("710025") + .mockReturnValueOnce("710025") + .mockReturnValueOnce("710025"); + const runOpenshell = vi.fn((_args: string[]) => ({ status: 0 })); + const runtime = createGatewayHostRuntime(createDeps({ readProcStartTime, runOpenshell })); + const owner = runtime.getGatewayOwner(); + const expectedProbe = await runtime.probeGatewayAttachment(owner); + + await expect(runtime.attachGateway(owner, expectedProbe)).rejects.toMatchObject({ + code: "identity_mismatch", + }); + expect(runOpenshell).toHaveBeenLastCalledWith(["gateway", "remove", "nemoclaw"], { + ignoreError: true, + suppressOutput: true, + }); + expect(process.env.OPENSHELL_GATEWAY).toBeUndefined(); + }); }); diff --git a/src/lib/onboard/gateway-host-runtime.ts b/src/lib/onboard/gateway-host-runtime.ts index 7b1b0773a9d..616c156fb38 100644 --- a/src/lib/onboard/gateway-host-runtime.ts +++ b/src/lib/onboard/gateway-host-runtime.ts @@ -217,7 +217,9 @@ export function createGatewayHostRuntime(deps: GatewayHostRuntimeDeps): GatewayH const execPath = readListenerExecPath(pid); const supervisorMatch = readListenerSupervisorMatch(owner, pid); const endTime = readProcStartTime(pid); - return startTime !== null && endTime === startTime ? { execPath, supervisorMatch } : null; + return startTime !== null && endTime === startTime + ? { startTime, execPath, supervisorMatch } + : null; } /** @@ -255,7 +257,7 @@ export function createGatewayHostRuntime(deps: GatewayHostRuntimeDeps): GatewayH gatewayBin, }); const [firstPid] = scan.pids; - const identity = + const initialIdentity = scan.complete && scan.pids.length === 1 && typeof firstPid === "number" ? readStableListenerIdentity(owner, firstPid) : null; @@ -265,12 +267,26 @@ export function createGatewayHostRuntime(deps: GatewayHostRuntimeDeps): GatewayH gatewayBin, }) : scan; + const confirmedIdentity = + scan.complete && + verifiedScan.complete && + scan.pids.length === 1 && + verifiedScan.pids.length === 1 && + verifiedScan.pids[0] === firstPid && + typeof firstPid === "number" + ? readStableListenerIdentity(owner, firstPid) + : null; const listenerStayedStable = scan.complete && verifiedScan.complete && scan.pids.length === 1 && verifiedScan.pids.length === 1 && - verifiedScan.pids[0] === firstPid; + verifiedScan.pids[0] === firstPid && + initialIdentity !== null && + confirmedIdentity !== null && + initialIdentity.startTime === confirmedIdentity.startTime && + initialIdentity.execPath === confirmedIdentity.execPath && + initialIdentity.supervisorMatch === confirmedIdentity.supervisorMatch; return { gatewayPort: deps.gatewayPort(), httpReady, @@ -278,9 +294,10 @@ export function createGatewayHostRuntime(deps: GatewayHostRuntimeDeps): GatewayH portOccupied: !portCheck.ok, listenerPids: verifiedScan.pids, listenerScanComplete: scan.complete && verifiedScan.complete, + listenerStartTime: listenerStayedStable ? confirmedIdentity.startTime : null, supervisorActive, - listenerExecPath: listenerStayedStable ? (identity?.execPath ?? null) : null, - listenerSupervisorMatch: listenerStayedStable ? (identity?.supervisorMatch ?? null) : null, + listenerExecPath: listenerStayedStable ? confirmedIdentity.execPath : null, + listenerSupervisorMatch: listenerStayedStable ? confirmedIdentity.supervisorMatch : null, }; } @@ -324,6 +341,7 @@ export function createGatewayHostRuntime(deps: GatewayHostRuntimeDeps): GatewayH expected.httpReady === actual.httpReady && expected.portOccupied === actual.portOccupied && expected.listenerScanComplete === actual.listenerScanComplete && + expected.listenerStartTime === actual.listenerStartTime && expected.supervisorActive === actual.supervisorActive && expected.listenerExecPath === actual.listenerExecPath && expected.listenerSupervisorMatch === actual.listenerSupervisorMatch && diff --git a/src/lib/onboard/gateway-management.test.ts b/src/lib/onboard/gateway-management.test.ts index 7f28a578512..f6405513a5d 100644 --- a/src/lib/onboard/gateway-management.test.ts +++ b/src/lib/onboard/gateway-management.test.ts @@ -182,6 +182,20 @@ describe("gateway management declaration", () => { expect(result.ok === false && result.reason).toMatch(/unsupported capability/); }); + it("does not echo malformed declaration values in errors (#6576)", () => { + const secret = "sk-live-not-a-real-token"; + for (const declaration of [ + externalDeclaration({ version: secret }), + externalDeclaration({ mode: secret }), + externalDeclaration({ endpoint: secret }), + externalDeclaration({ requiredCapabilities: [secret] }), + ]) { + const result = parseGatewayManagementDeclaration(declaration); + expect(result).toMatchObject({ ok: false }); + expect(result.ok === false && result.reason).not.toContain(secret); + } + }); + it("rejects a relative state directory (#6576)", () => { const result = parseGatewayManagementDeclaration( externalDeclaration({ stateDir: "relative/state" }), @@ -222,12 +236,14 @@ describe("gateway management declaration loading", () => { }); it("fails closed on malformed JSON rather than self-managing the gateway (#6576)", () => { + const secret = "sk-live-not-a-real-token"; const result = loadGatewayManagementDeclaration({ env: { [GATEWAY_MANAGEMENT_ENV_VAR]: "/etc/nemoclaw/gateway.json" }, - readFile: () => "{ not json", + readFile: () => `{ "token": "${secret}"`, }); expect(result.ok === false && result.reason).toMatch(/not valid JSON/); + expect(result.ok === false && result.reason).not.toContain(secret); }); it("prefers an in-process profile declaration over the environment file (#6576)", () => { diff --git a/src/lib/onboard/gateway-management.ts b/src/lib/onboard/gateway-management.ts index ef14e2c9341..5400fbf54f5 100644 --- a/src/lib/onboard/gateway-management.ts +++ b/src/lib/onboard/gateway-management.ts @@ -135,10 +135,10 @@ function parseEndpoint(value: unknown): string | { error: string } { try { url = new URL(raw); } catch { - return { error: `endpoint is not a valid URL: ${raw}` }; + return { error: "endpoint is not a valid URL" }; } if (url.protocol !== "http:" && url.protocol !== "https:") { - return { error: `endpoint must use http or https, got ${url.protocol.replace(":", "")}` }; + return { error: "endpoint must use http or https" }; } if (url.username || url.password) { return { error: "endpoint must not embed credentials" }; @@ -152,7 +152,7 @@ function parseEndpoint(value: unknown): string | { error: string } { if (!SUPPORTED_GATEWAY_ENDPOINT_HOSTS.has(url.hostname)) { return { error: - `endpoint host ${url.hostname} is not a supported local gateway origin; ` + + `endpoint host is not a supported local gateway origin; ` + `the declared gateway is supervised on this machine, so the endpoint must be loopback ` + `(one of: ${[...SUPPORTED_GATEWAY_ENDPOINT_HOSTS].join(", ")})`, }; @@ -167,7 +167,7 @@ function parseStateDir(value: unknown): string | { error: string } { const raw = requireNonEmptyString(value, "stateDir"); if (typeof raw !== "string") return raw; if (!path.isAbsolute(raw)) { - return { error: `stateDir must be an absolute path, got ${raw}` }; + return { error: "stateDir must be an absolute path" }; } return raw; } @@ -183,7 +183,7 @@ function parseCapabilities(value: unknown): readonly GatewayCapability[] | { err if (typeof entry !== "string" || !supported.has(entry)) { return { error: - `unsupported capability ${JSON.stringify(entry)}; ` + + `unsupported capability; ` + `this NemoClaw build provides: ${SUPPORTED_GATEWAY_CAPABILITIES.join(", ")}`, }; } @@ -230,7 +230,7 @@ function parseSupervisor( const execPath = requireNonEmptyString(value.execPath, "supervisor.execPath"); if (typeof execPath !== "string") return execPath; if (!path.isAbsolute(execPath)) { - return { error: `supervisor.execPath must be an absolute path, got ${execPath}` }; + return { error: "supervisor.execPath must be an absolute path" }; } return { kind: kind as GatewaySupervisorKind, serviceName, execPath }; @@ -251,7 +251,7 @@ export function parseGatewayManagementDeclaration(raw: unknown): GatewayManageme return { ok: false, reason: - `unsupported gateway-management contract version ${JSON.stringify(raw.version)}; ` + + "unsupported gateway-management contract version; " + `this NemoClaw build supports version ${GATEWAY_MANAGEMENT_CONTRACT_VERSION}`, }; } @@ -260,7 +260,7 @@ export function parseGatewayManagementDeclaration(raw: unknown): GatewayManageme if (mode !== "nemoclaw-managed" && mode !== "externally-supervised") { return { ok: false, - reason: `mode must be nemoclaw-managed or externally-supervised, got ${JSON.stringify(mode)}`, + reason: "mode must be nemoclaw-managed or externally-supervised", }; } @@ -342,22 +342,20 @@ export function loadGatewayManagementDeclaration( let contents: string; try { contents = readFile(path.resolve(configuredPath)); - } catch (error) { - const detail = error instanceof Error ? error.message : String(error); + } catch { return { ok: false, - reason: `${GATEWAY_MANAGEMENT_ENV_VAR}=${configuredPath} could not be read: ${detail}`, + reason: `${GATEWAY_MANAGEMENT_ENV_VAR} declaration file could not be read`, }; } let raw: unknown; try { raw = JSON.parse(contents); - } catch (error) { - const detail = error instanceof Error ? error.message : String(error); + } catch { return { ok: false, - reason: `${GATEWAY_MANAGEMENT_ENV_VAR}=${configuredPath} is not valid JSON: ${detail}`, + reason: `${GATEWAY_MANAGEMENT_ENV_VAR} declaration file is not valid JSON`, }; } @@ -365,7 +363,7 @@ export function loadGatewayManagementDeclaration( if (!parsed.ok) { return { ok: false, - reason: `${GATEWAY_MANAGEMENT_ENV_VAR}=${configuredPath}: ${parsed.reason}`, + reason: `${GATEWAY_MANAGEMENT_ENV_VAR} declaration file: ${parsed.reason}`, }; } return { ok: true, declaration: parsed.declaration, source: "file" }; diff --git a/src/lib/onboard/gateway-ownership.test.ts b/src/lib/onboard/gateway-ownership.test.ts index eaa9e37bed4..0647b159713 100644 --- a/src/lib/onboard/gateway-ownership.test.ts +++ b/src/lib/onboard/gateway-ownership.test.ts @@ -42,6 +42,7 @@ function probe(overrides: Partial = {}): GatewayAttachme portOccupied: true, listenerPids: [4242], listenerScanComplete: true, + listenerStartTime: "710024", supervisorActive: true, listenerExecPath: "/usr/local/bin/openshell-gateway", listenerSupervisorMatch: true, diff --git a/src/lib/onboard/gateway-ownership.ts b/src/lib/onboard/gateway-ownership.ts index be61965434c..e45c6c18418 100644 --- a/src/lib/onboard/gateway-ownership.ts +++ b/src/lib/onboard/gateway-ownership.ts @@ -240,6 +240,8 @@ export interface GatewayAttachmentProbe { listenerPids: readonly number[]; /** False when the listener set could not be authoritatively enumerated. */ listenerScanComplete: boolean; + /** Linux process start time proving which generation of the PID was inspected. */ + listenerStartTime: string | null; /** Whether the declared supervisor unit reports active; null when unprobeable. */ supervisorActive: boolean | null; /** diff --git a/src/lib/onboard/machine/handlers/gateway.test.ts b/src/lib/onboard/machine/handlers/gateway.test.ts index c63d4a22e88..8661f1d7566 100644 --- a/src/lib/onboard/machine/handlers/gateway.test.ts +++ b/src/lib/onboard/machine/handlers/gateway.test.ts @@ -80,6 +80,7 @@ function createDeps(overrides: Partial["deps"]> = {}) { portOccupied: true, listenerPids: [4242], listenerScanComplete: true, + listenerStartTime: "710024", supervisorActive: true, listenerExecPath: "/usr/local/bin/openshell-gateway", listenerSupervisorMatch: true, @@ -472,6 +473,7 @@ describe("externally supervised gateway lifecycle authority", () => { portOccupied: true, listenerPids: [4242], listenerScanComplete: true, + listenerStartTime: "710024", supervisorActive: true, listenerExecPath: "/usr/local/bin/openshell-gateway", listenerSupervisorMatch: true, diff --git a/src/lib/onboard/machine/initial-flow-phases.test.ts b/src/lib/onboard/machine/initial-flow-phases.test.ts index 2daf0a2c926..93a32f82038 100644 --- a/src/lib/onboard/machine/initial-flow-phases.test.ts +++ b/src/lib/onboard/machine/initial-flow-phases.test.ts @@ -133,6 +133,7 @@ describe("initial onboard flow phases", () => { portOccupied: true, listenerPids: [4242], listenerScanComplete: true, + listenerStartTime: null, supervisorActive: null, listenerExecPath: null, listenerSupervisorMatch: null, @@ -375,6 +376,7 @@ describe("initial onboard flow phases", () => { portOccupied: true, listenerPids: [4242], listenerScanComplete: true, + listenerStartTime: null, supervisorActive: null, listenerExecPath: null, listenerSupervisorMatch: null, From 122bb63a28f99872f620eb8b2d09de9f47da091a Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 22 Jul 2026 13:33:47 -0400 Subject: [PATCH 05/20] chore(ci): retry transient E2E gate Signed-off-by: Julie Yaunches From 9881a094d089445284a062bc0cbde888305591d6 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 22 Jul 2026 14:16:53 -0400 Subject: [PATCH 06/20] chore(ci): refresh terminal E2E gate Signed-off-by: Julie Yaunches From 7958a0ca00f8a98fa8914881dd31932f2bd65c01 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Thu, 23 Jul 2026 13:33:39 -0700 Subject: [PATCH 07/20] fix(onboard): revalidate gateway authority before attach --- .../gateway-lifecycle-authority.mdx | 5 ++-- src/lib/onboard/gateway-host-runtime.test.ts | 24 +++++++++++++++++++ src/lib/onboard/gateway-host-runtime.ts | 1 + src/lib/onboard/gateway-ownership.test.ts | 2 +- src/lib/onboard/gateway-ownership.ts | 2 +- 5 files changed, 30 insertions(+), 4 deletions(-) diff --git a/docs/deployment/gateway-lifecycle-authority.mdx b/docs/deployment/gateway-lifecycle-authority.mdx index 2314696083f..ed79cd51198 100644 --- a/docs/deployment/gateway-lifecycle-authority.mdx +++ b/docs/deployment/gateway-lifecycle-authority.mdx @@ -107,12 +107,13 @@ For an HTTP endpoint, NemoClaw sends the readiness request to the exact declared NemoClaw validates external authority before provider, policy, sandbox, or sandbox-registry mutations. NemoClaw requires systemd to confirm that the declared unit is active. -It checks that exactly one recognizable process holds the configured port. +It checks that exactly one live process holds the configured port. The process identity must remain stable while NemoClaw reads `/proc` evidence and confirms the listener a second time. The process must belong to the unit's cgroup, and its executable must match `execPath`. The exact endpoint must answer its health check. After validation, NemoClaw registers the exact endpoint under the canonical gateway name and selects it as the active OpenShell gateway. +Immediately before registration, NemoClaw reloads the authority and rejects any change from the authority bound to the run. Port `8080` uses the name `nemoclaw`, and another port uses `nemoclaw-`. NemoClaw verifies the named and active registrations before advancing to provider selection. It then repeats the health, supervisor, listener, executable, cgroup, and process identity checks and compares them with the initial evidence. @@ -125,7 +126,7 @@ Onboarding fails when any of the following conditions applies: - The declared supervisor is inactive, or systemd cannot confirm its active state. - Nothing is listening on the declared endpoint. - The listener set cannot be enumerated completely. -- No recognizable gateway process holds the port. +- NemoClaw cannot identify a listener process for the occupied port. - More than one process holds the port. - The listener process or listener set changes during identity verification. - The listener does not belong to the declared systemd unit. diff --git a/src/lib/onboard/gateway-host-runtime.test.ts b/src/lib/onboard/gateway-host-runtime.test.ts index d299222e671..b5f4168ece7 100644 --- a/src/lib/onboard/gateway-host-runtime.test.ts +++ b/src/lib/onboard/gateway-host-runtime.test.ts @@ -342,6 +342,30 @@ describe("gateway host runtime attachment probe", () => { expect(process.env.OPENSHELL_GATEWAY).toBe("nemoclaw"); }); + it("rejects changed authority before gateway registration (#6576)", async () => { + process.env[GATEWAY_MANAGEMENT_ENV_VAR] = "/etc/nemoclaw/gateway-management.json"; + let declaration = DECLARATION; + vi.spyOn(require("node:fs") as typeof import("node:fs"), "readFileSync").mockImplementation( + () => JSON.stringify(declaration) as never, + ); + const runOpenshell = vi.fn((_args: string[]) => ({ status: 0 })); + const runtime = createGatewayHostRuntime(createDeps({ runOpenshell })); + const owner = runtime.getGatewayOwner(); + const expectedProbe = await runtime.probeGatewayAttachment(owner); + declaration = { + ...DECLARATION, + supervisor: { + ...DECLARATION.supervisor, + execPath: "/opt/platform/replacement-gatewayd", + }, + }; + + await expect(runtime.attachGateway(owner, expectedProbe)).rejects.toThrow( + /authority changed during this run/, + ); + expect(runOpenshell).not.toHaveBeenCalled(); + }); + it("replaces stale registration before selecting the declared endpoint (#6576)", async () => { declareExternalSupervision(); const statuses = [1, 0, 0]; diff --git a/src/lib/onboard/gateway-host-runtime.ts b/src/lib/onboard/gateway-host-runtime.ts index 616c156fb38..a3427a999f9 100644 --- a/src/lib/onboard/gateway-host-runtime.ts +++ b/src/lib/onboard/gateway-host-runtime.ts @@ -355,6 +355,7 @@ export function createGatewayHostRuntime(deps: GatewayHostRuntimeDeps): GatewayH owner: GatewayOwner, expectedProbe: GatewayAttachmentProbe, ): Promise { + bindGatewayOwner(owner); if (!isExternallySupervised(owner) || !owner.endpoint) return; const expectedAttachment = evaluateGatewayAttachment(owner, expectedProbe); if (!expectedAttachment.ok) { diff --git a/src/lib/onboard/gateway-ownership.test.ts b/src/lib/onboard/gateway-ownership.test.ts index 0647b159713..9b5cc5e456a 100644 --- a/src/lib/onboard/gateway-ownership.test.ts +++ b/src/lib/onboard/gateway-ownership.test.ts @@ -184,7 +184,7 @@ describe("externally supervised gateway attachment", () => { expect(result.ok === false && result.message).toMatch(/will not start a competing gateway/); }); - it("fails when an unrecognized process holds the gateway port (#6576)", () => { + it("fails when an occupied gateway port has no identifiable listener process (#6576)", () => { const result = evaluateGatewayAttachment( externalOwner, probe({ listenerPids: [], listenerExecPath: null }), diff --git a/src/lib/onboard/gateway-ownership.ts b/src/lib/onboard/gateway-ownership.ts index e45c6c18418..2f05db6cba8 100644 --- a/src/lib/onboard/gateway-ownership.ts +++ b/src/lib/onboard/gateway-ownership.ts @@ -374,7 +374,7 @@ export function evaluateGatewayAttachment( ok: false, code: "unknown_listener", message: - `The gateway port is held by a process that is not a recognizable OpenShell gateway. ` + + `NemoClaw could not identify a listener process for the occupied gateway port. ` + `NemoClaw will not attach to it or replace it. Identify the process holding the port, ` + `stop only that process, and let ${supervisorName} own the gateway.`, }; From 564e08c3e2bb15e281a9acd4262da7e9a6bfe2c0 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Thu, 23 Jul 2026 13:54:32 -0700 Subject: [PATCH 08/20] fix(onboard): verify gateway supervisor scope --- .../gateway-lifecycle-authority.mdx | 4 +- src/lib/onboard/gateway-host-runtime.test.ts | 42 +++++++++++++++++++ src/lib/onboard/gateway-host-runtime.ts | 2 +- src/lib/onboard/gateway-ownership.test.ts | 35 +++++++++++++--- src/lib/onboard/gateway-ownership.ts | 28 +++++++++---- 5 files changed, 95 insertions(+), 16 deletions(-) diff --git a/docs/deployment/gateway-lifecycle-authority.mdx b/docs/deployment/gateway-lifecycle-authority.mdx index ed79cd51198..9f79ea67bc1 100644 --- a/docs/deployment/gateway-lifecycle-authority.mdx +++ b/docs/deployment/gateway-lifecycle-authority.mdx @@ -85,7 +85,7 @@ The authority may be persisted in checkpoints, diagnostics, and machine events, Only systemd supervisors are supported in version 1. -NemoClaw verifies both the listener executable and its membership in the declared unit's cgroup. +NemoClaw verifies the listener executable, the declared system or user manager scope, and membership in the declared unit's cgroup. A matching executable alone is not sufficient because the same binary can run outside the declared unit. @@ -109,7 +109,7 @@ NemoClaw validates external authority before provider, policy, sandbox, or sandb NemoClaw requires systemd to confirm that the declared unit is active. It checks that exactly one live process holds the configured port. The process identity must remain stable while NemoClaw reads `/proc` evidence and confirms the listener a second time. -The process must belong to the unit's cgroup, and its executable must match `execPath`. +The process must belong to the unit's cgroup under the declared systemd manager scope, and its executable must match `execPath`. The exact endpoint must answer its health check. After validation, NemoClaw registers the exact endpoint under the canonical gateway name and selects it as the active OpenShell gateway. diff --git a/src/lib/onboard/gateway-host-runtime.test.ts b/src/lib/onboard/gateway-host-runtime.test.ts index b5f4168ece7..d8a06e18104 100644 --- a/src/lib/onboard/gateway-host-runtime.test.ts +++ b/src/lib/onboard/gateway-host-runtime.test.ts @@ -200,6 +200,48 @@ describe("gateway host runtime attachment probe", () => { }); }); + it("rejects a same-named user unit when the system manager is declared (#6576)", async () => { + declareExternalSupervision(); + const runOpenshell = vi.fn((_args: string[]) => ({ status: 0 })); + const runtime = createGatewayHostRuntime( + createDeps({ + readProcCgroup: () => + `0::/user.slice/user-1000.slice/user@1000.service/app.slice/${DECLARATION.supervisor.serviceName}\n`, + runOpenshell, + }), + ); + const owner = runtime.getGatewayOwner(); + const probe = await runtime.probeGatewayAttachment(owner); + + expect(probe.listenerSupervisorMatch).toBe(false); + await expect(runtime.attachGateway(owner, probe)).rejects.toMatchObject({ + code: "identity_mismatch", + }); + expect(runOpenshell).not.toHaveBeenCalled(); + }); + + it("rejects a same-named system unit when the user manager is declared (#6576)", async () => { + declareExternalSupervision({ + ...DECLARATION, + supervisor: { ...DECLARATION.supervisor, kind: "systemd-user" }, + }); + const runOpenshell = vi.fn((_args: string[]) => ({ status: 0 })); + const runtime = createGatewayHostRuntime( + createDeps({ + readProcCgroup: () => `0::/system.slice/${DECLARATION.supervisor.serviceName}\n`, + runOpenshell, + }), + ); + const owner = runtime.getGatewayOwner(); + const probe = await runtime.probeGatewayAttachment(owner); + + expect(probe.listenerSupervisorMatch).toBe(false); + await expect(runtime.attachGateway(owner, probe)).rejects.toMatchObject({ + code: "identity_mismatch", + }); + expect(runOpenshell).not.toHaveBeenCalled(); + }); + it("rejects a listener whose executable differs from the arbitrary declared path (#6576)", async () => { declareExternalSupervision(); const runtime = createGatewayHostRuntime( diff --git a/src/lib/onboard/gateway-host-runtime.ts b/src/lib/onboard/gateway-host-runtime.ts index a3427a999f9..5ed1166906d 100644 --- a/src/lib/onboard/gateway-host-runtime.ts +++ b/src/lib/onboard/gateway-host-runtime.ts @@ -208,7 +208,7 @@ export function createGatewayHostRuntime(deps: GatewayHostRuntimeDeps): GatewayH if (!supervisor) return null; const cgroupText = readProcCgroup(pid); if (cgroupText === null) return null; - return cgroupBelongsToUnit(cgroupText, supervisor.serviceName); + return cgroupBelongsToUnit(cgroupText, supervisor.serviceName, supervisor.kind); } /** Read executable and cgroup evidence only while the PID names one process. */ diff --git a/src/lib/onboard/gateway-ownership.test.ts b/src/lib/onboard/gateway-ownership.test.ts index 9b5cc5e456a..0bf55bb5f6c 100644 --- a/src/lib/onboard/gateway-ownership.test.ts +++ b/src/lib/onboard/gateway-ownership.test.ts @@ -296,7 +296,7 @@ describe("cgroupBelongsToUnit", () => { const UNIT = "openshell-gateway.service"; it("matches a cgroup v2 process in the unit's system slice (#6576)", () => { - expect(cgroupBelongsToUnit(`0::/system.slice/${UNIT}\n`, UNIT)).toBe(true); + expect(cgroupBelongsToUnit(`0::/system.slice/${UNIT}\n`, UNIT, "systemd-system")).toBe(true); }); it("matches a cgroup v1 process listed under the unit (#6576)", () => { @@ -304,7 +304,13 @@ describe("cgroupBelongsToUnit", () => { "12:pids:/system.slice/openshell-gateway.service", "0:name=systemd:/system.slice/openshell-gateway.service", ].join("\n"); - expect(cgroupBelongsToUnit(v1, UNIT)).toBe(true); + expect(cgroupBelongsToUnit(v1, UNIT, "systemd-system")).toBe(true); + }); + + it("matches a system-manager unit in a custom slice (#6576)", () => { + expect( + cgroupBelongsToUnit(`0::/platform.slice/gateways.slice/${UNIT}`, UNIT, "systemd-system"), + ).toBe(true); }); it("matches a user-manager unit path (#6576)", () => { @@ -312,19 +318,38 @@ describe("cgroupBelongsToUnit", () => { cgroupBelongsToUnit( `0::/user.slice/user-1000.slice/user@1000.service/app.slice/${UNIT}`, UNIT, + "systemd-user", ), ).toBe(true); }); + it("rejects a same-named user unit for the system manager (#6576)", () => { + expect( + cgroupBelongsToUnit( + `0::/user.slice/user-1000.slice/user@1000.service/app.slice/${UNIT}`, + UNIT, + "systemd-system", + ), + ).toBe(false); + }); + + it("rejects a same-named system unit for the user manager (#6576)", () => { + expect(cgroupBelongsToUnit(`0::/system.slice/${UNIT}`, UNIT, "systemd-user")).toBe(false); + }); + it("rejects a same-binary process in a login session scope (#6576)", () => { - expect(cgroupBelongsToUnit("0::/user.slice/user-1000.slice/session-3.scope", UNIT)).toBe(false); + expect( + cgroupBelongsToUnit("0::/user.slice/user-1000.slice/session-3.scope", UNIT, "systemd-user"), + ).toBe(false); }); it("rejects a different unit that merely shares a prefix (#6576)", () => { - expect(cgroupBelongsToUnit("0::/system.slice/openshell-gateway.service.d", UNIT)).toBe(false); + expect( + cgroupBelongsToUnit("0::/system.slice/openshell-gateway.service.d", UNIT, "systemd-system"), + ).toBe(false); }); it("rejects empty or unreadable cgroup text (#6576)", () => { - expect(cgroupBelongsToUnit("", UNIT)).toBe(false); + expect(cgroupBelongsToUnit("", UNIT, "systemd-system")).toBe(false); }); }); diff --git a/src/lib/onboard/gateway-ownership.ts b/src/lib/onboard/gateway-ownership.ts index 2f05db6cba8..6d96dfd6d6f 100644 --- a/src/lib/onboard/gateway-ownership.ts +++ b/src/lib/onboard/gateway-ownership.ts @@ -264,16 +264,20 @@ export interface GatewayAttachmentProbe { } /** - * Whether a process cgroup path names the given systemd unit. + * Whether a process cgroup path names the given systemd unit under the + * declared manager scope. * * `/proc//cgroup` lists the process's cgroup path; a process managed by a - * systemd unit sits under a slice segment named for the unit (for example - * `/system.slice/openshell-gateway.service` or a `...//...` subpath). A - * same-binary process started outside the unit lands in a different cgroup - * (a login session scope, the user slice), so this distinguishes the unit's - * own process from an impostor holding the same port. + * user-manager unit sits below the `user@.service` delegation boundary. + * A system-manager unit does not, even when it uses a custom slice. Requiring + * that scope as well as the unit segment prevents a same-named unit in the + * other manager from satisfying the authority declaration. */ -export function cgroupBelongsToUnit(cgroupText: string, serviceName: string): boolean { +export function cgroupBelongsToUnit( + cgroupText: string, + serviceName: string, + supervisorKind: GatewaySupervisorDeclaration["kind"], +): boolean { const unit = serviceName.trim(); if (!unit) return false; for (const line of cgroupText.split(/\r?\n/)) { @@ -281,7 +285,15 @@ export function cgroupBelongsToUnit(cgroupText: string, serviceName: string): bo const cgroupPath = line.slice(line.lastIndexOf(":") + 1).trim(); if (!cgroupPath) continue; const segments = cgroupPath.split("/").filter(Boolean); - if (segments.includes(unit)) return true; + const unitIndex = segments.indexOf(unit); + if (unitIndex < 0) continue; + const userManagerIndex = segments.findIndex((segment) => /^user@\d+\.service$/.test(segment)); + if (supervisorKind === "systemd-system") { + if (userManagerIndex < 0 || userManagerIndex >= unitIndex) return true; + continue; + } + if (segments[0] === "user.slice" && userManagerIndex >= 0 && userManagerIndex < unitIndex) + return true; } return false; } From 82cdc33d7b90988e9d4bc56446c3c76961644c14 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Thu, 23 Jul 2026 14:47:37 -0700 Subject: [PATCH 09/20] fix(onboard): expose validated gateway authority --- .../gateway-lifecycle-authority.mdx | 15 +++- src/lib/inventory/index.ts | 48 ++++++++++++ src/lib/onboard/gateway-host-runtime.test.ts | 28 +++++++ src/lib/onboard/gateway-host-runtime.ts | 5 ++ src/lib/onboard/gateway-ownership.ts | 73 +++++++++++++++---- src/lib/state/onboard-session.test.ts | 33 +++++++++ src/lib/state/onboard-session.ts | 7 ++ src/lib/status-command-deps.ts | 2 + test/cli/debug-command.test.ts | 20 ++++- test/cli/status-root-json.test.ts | 65 ++++++++++++++++- test/helpers/gateway-authority-session.ts | 44 +++++++++++ 11 files changed, 321 insertions(+), 19 deletions(-) create mode 100644 test/helpers/gateway-authority-session.ts diff --git a/docs/deployment/gateway-lifecycle-authority.mdx b/docs/deployment/gateway-lifecycle-authority.mdx index 9f79ea67bc1..589c4521a88 100644 --- a/docs/deployment/gateway-lifecycle-authority.mdx +++ b/docs/deployment/gateway-lifecycle-authority.mdx @@ -75,7 +75,7 @@ The declaration uses the following fields. The endpoint port must match `NEMOCLAW_GATEWAY_PORT`. When the endpoint omits a port, `http` resolves to port `80` and `https` resolves to port `443` for this comparison. -Onboarding rejects a mismatch because validating one endpoint and operating a different gateway would break the authority boundary. +Onboarding rejects a mismatch before making a health request or inspecting the host listener because validating one endpoint and operating a different gateway would break the authority boundary. The declaration is versioned and secret-free. @@ -143,6 +143,19 @@ Resume resolves the current authority again and compares the complete record wit If any value changes, including the per-port gateway binding, resume fails before gateway effects and directs you to start a fresh onboarding run. A completed gateway step does not bypass listener, supervisor, identity, health, registration, or checkpoint validation. +## Inspect the selected authority + +Run `nemoclaw status` to see the management mode and redacted owner identity selected by the last onboarding run. +The JSON form includes the same secret-free fields under `gatewayAuthority`. + +```bash +nemoclaw status +nemoclaw status --json +``` + +`nemoclaw debug` also records `gatewayAuthority` in `onboard-session-summary.txt`. +The status and debug views omit the external state directory and never include credential values. + ## Keep external lifecycle paths inert External supervision prevents every NemoClaw gateway lifecycle path from affecting the supervised process. diff --git a/src/lib/inventory/index.ts b/src/lib/inventory/index.ts index 64847fe2e57..96f8f8e20d1 100644 --- a/src/lib/inventory/index.ts +++ b/src/lib/inventory/index.ts @@ -4,6 +4,7 @@ import { CLI_NAME } from "../cli/branding"; import type { GatewayInference } from "../inference/config"; import { getActiveChannelIdsFromPlan } from "../messaging/plan-validation"; +import type { GatewayOwnerDescription } from "../onboard/gateway-ownership"; import { redactFull } from "../security/redact"; import { getSandboxEntryDisplayInference, type SandboxMessagingState } from "../state/registry"; import { resolveDefaultSandboxName } from "../tunnel/service-command"; @@ -132,6 +133,8 @@ export interface ShowStatusCommandDeps { * detect the degraded state from `$?` (#3386). */ getGatewayHealth?: () => GatewayHealth; + /** Last authority durably selected by onboarding, with secret-free identity fields. */ + getGatewayAuthority?: () => GatewayOwnerDescription | null; checkMessagingBridgeHealth?: ( sandboxName: string, channels: string[], @@ -173,6 +176,7 @@ export interface StatusReport { model: string | null; } | null; gatewayHealth: GatewayHealth | null; + gatewayAuthority: GatewayOwnerDescription | null; sandboxes: StatusSandboxRow[]; services: StatusServiceRow[]; } @@ -411,6 +415,36 @@ function normalizeGatewayHealth(health: GatewayHealth | null | undefined): Gatew }; } +function normalizeGatewayAuthority( + authority: GatewayOwnerDescription | null | undefined, +): GatewayOwnerDescription | null { + if (!authority) return null; + const gatewayName = safeStatusString(authority.gatewayName); + const source = safeStatusString(authority.source); + const endpoint = safeStatusString(authority.endpoint); + const supervisor = authority.supervisor + ? { + kind: authority.supervisor.kind, + serviceName: safeStatusString(authority.supervisor.serviceName) ?? "unknown", + execPath: safeStatusString(authority.supervisor.execPath) ?? "unknown", + } + : null; + return { + gatewayName: gatewayName ?? "unknown", + gatewayPort: authority.gatewayPort, + mode: authority.mode, + source: + source === "declared" || source === "packaged-service" || source === "standalone" + ? source + : "standalone", + endpoint, + supervisor, + requiredCapabilities: authority.requiredCapabilities.map( + (capability) => safeStatusString(capability) ?? "unknown", + ), + }; +} + export function getStatusReport(deps: ShowStatusCommandDeps): StatusReport { const sandboxList = deps.listSandboxes(); const { sandboxes } = sandboxList; @@ -433,6 +467,7 @@ export function getStatusReport(deps: ShowStatusCommandDeps): StatusReport { } : null, gatewayHealth: normalizeGatewayHealth(gatewayHealth), + gatewayAuthority: normalizeGatewayAuthority(deps.getGatewayAuthority?.()), sandboxes: sandboxes.map((sandbox) => buildStatusSandboxRow(sandbox, resolvedDefault, liveInference), ), @@ -494,6 +529,19 @@ export function showStatusCommand(deps: ShowStatusCommandDeps): void { log(""); } + const gatewayAuthority = normalizeGatewayAuthority(deps.getGatewayAuthority?.()); + if (gatewayAuthority) { + const owner = gatewayAuthority.supervisor + ? `${gatewayAuthority.supervisor.kind} ${gatewayAuthority.supervisor.serviceName} (${gatewayAuthority.supervisor.execPath})` + : gatewayAuthority.source; + log(` Gateway authority: ${gatewayAuthority.mode}`); + log(` Owner: ${owner}`); + if (gatewayAuthority.endpoint) { + log(` Endpoint: ${gatewayAuthority.endpoint}`); + } + log(""); + } + // Surface gateway health between the sandbox list and the host-service // list, since the gateway sits logically between the two. When the named // gateway is unhealthy we also set process.exitCode = 1 so shell scripts diff --git a/src/lib/onboard/gateway-host-runtime.test.ts b/src/lib/onboard/gateway-host-runtime.test.ts index d8a06e18104..5261f708917 100644 --- a/src/lib/onboard/gateway-host-runtime.test.ts +++ b/src/lib/onboard/gateway-host-runtime.test.ts @@ -154,6 +154,34 @@ describe("gateway host runtime ownership", () => { }); describe("gateway host runtime attachment probe", () => { + it("rejects a mismatched endpoint port before any host or HTTP probe (#6576)", async () => { + declareExternalSupervision({ + ...DECLARATION, + endpoint: "http://127.0.0.1:9443", + }); + const checkGatewayPortAvailable = vi.fn().mockResolvedValue(OCCUPIED_PORT); + const probeGatewayHttpReady = vi.fn().mockResolvedValue(true); + const getGatewayPortListenerRawScan = vi.fn(() => ({ + pids: [SYSTEMD_GATEWAY_PID], + complete: true, + })); + const runtime = createGatewayHostRuntime( + createDeps({ + checkGatewayPortAvailable, + getGatewayPortListenerRawScan, + probeGatewayHttpReady, + }), + ); + const owner = runtime.getGatewayOwner(); + + await expect(runtime.probeGatewayAttachment(owner)).rejects.toMatchObject({ + code: "endpoint_port_mismatch", + }); + expect(checkGatewayPortAvailable).not.toHaveBeenCalled(); + expect(probeGatewayHttpReady).not.toHaveBeenCalled(); + expect(getGatewayPortListenerRawScan).not.toHaveBeenCalled(); + }); + it("attaches to a real systemd-supervised gateway listener (#6576)", async () => { declareExternalSupervision(); const runtime = createGatewayHostRuntime(createDeps()); diff --git a/src/lib/onboard/gateway-host-runtime.ts b/src/lib/onboard/gateway-host-runtime.ts index 5ed1166906d..a5a5d161a69 100644 --- a/src/lib/onboard/gateway-host-runtime.ts +++ b/src/lib/onboard/gateway-host-runtime.ts @@ -29,6 +29,7 @@ import { cgroupBelongsToUnit, describeGatewayOwnerForError, evaluateGatewayAttachment, + evaluateGatewayAttachmentConfiguration, type GatewayAttachmentProbe, type GatewayOwner, GatewayOwnershipError, @@ -249,6 +250,10 @@ export function createGatewayHostRuntime(deps: GatewayHostRuntimeDeps): GatewayH * gateway it does not own. Read-only: this runs before any effect. */ async function probeGatewayAttachment(owner: GatewayOwner): Promise { + const configuration = evaluateGatewayAttachmentConfiguration(owner, deps.gatewayPort()); + if (!configuration.ok) { + throw new GatewayOwnershipError(configuration.code, configuration.message, owner); + } const portCheck = await deps.checkGatewayPortAvailable(); const httpReady = await waitForDeclaredGatewayHttpReady(owner); const supervisorActive = isSupervisorUnitActive(owner); diff --git a/src/lib/onboard/gateway-ownership.ts b/src/lib/onboard/gateway-ownership.ts index 6d96dfd6d6f..9f8a6e5a038 100644 --- a/src/lib/onboard/gateway-ownership.ts +++ b/src/lib/onboard/gateway-ownership.ts @@ -72,6 +72,23 @@ export interface GatewayOwner { requiredCapabilities: readonly GatewayCapability[]; } +export interface GatewayOwnerSupervisorDescription extends JsonObject { + kind: GatewaySupervisorDeclaration["kind"]; + serviceName: string; + execPath: string; +} + +/** Secret-free owner shape exposed by status, diagnostics, and machine events. */ +export interface GatewayOwnerDescription extends JsonObject { + gatewayName: string; + gatewayPort: number; + mode: GatewayManagementMode; + source: GatewayOwnerSource; + endpoint: string | null; + supervisor: GatewayOwnerSupervisorDescription | null; + requiredCapabilities: string[]; +} + /** Lifecycle effects whose legality depends on who owns the gateway. */ export type GatewayLifecycleEffect = | "start" @@ -303,34 +320,27 @@ export type GatewayAttachmentResult = | { ok: false; code: GatewayOwnershipFailureCode; message: string }; /** - * Decide whether NemoClaw may attach to an externally supervised gateway. - * - * Every failure here must be raised before provider, policy, sandbox, or - * registry mutation: an ambiguous or multiply owned gateway is precisely the - * state that must not be papered over by starting another one. + * Reject declaration-only attachment failures before any host or network + * probe. Runtime evidence is meaningful only after the declaration names the + * gateway this process will actually use. */ -export function evaluateGatewayAttachment( +export function evaluateGatewayAttachmentConfiguration( owner: GatewayOwner, - probe: GatewayAttachmentProbe, + gatewayPort: number, ): GatewayAttachmentResult { if (!isExternallySupervised(owner)) { return { ok: true, owner }; } - const supervisorName = owner.supervisor?.serviceName ?? "the declared supervisor"; - - // Configuration errors are reported before any runtime observation: probing - // is only meaningful once we know the declaration describes the gateway this - // process actually operates on. const declaredPort = declaredEndpointPort(owner.endpoint); - if (declaredPort !== null && declaredPort !== probe.gatewayPort) { + if (declaredPort !== null && declaredPort !== gatewayPort) { return { ok: false, code: "endpoint_port_mismatch", message: `The declared gateway endpoint uses port ${declaredPort}, but this NemoClaw process operates ` + - `the gateway on port ${probe.gatewayPort}. Attaching would validate one gateway and then use ` + - `another. Point the declaration at port ${probe.gatewayPort}, or re-run with ` + + `the gateway on port ${gatewayPort}. Attaching would validate one gateway and then use ` + + `another. Point the declaration at port ${gatewayPort}, or re-run with ` + `NEMOCLAW_GATEWAY_PORT=${declaredPort}.`, }; } @@ -349,6 +359,29 @@ export function evaluateGatewayAttachment( }; } + return { ok: true, owner }; +} + +/** + * Decide whether NemoClaw may attach to an externally supervised gateway. + * + * Every failure here must be raised before provider, policy, sandbox, or + * registry mutation: an ambiguous or multiply owned gateway is precisely the + * state that must not be papered over by starting another one. + */ +export function evaluateGatewayAttachment( + owner: GatewayOwner, + probe: GatewayAttachmentProbe, +): GatewayAttachmentResult { + if (!isExternallySupervised(owner)) { + return { ok: true, owner }; + } + + const supervisorName = owner.supervisor?.serviceName ?? "the declared supervisor"; + + const configuration = evaluateGatewayAttachmentConfiguration(owner, probe.gatewayPort); + if (!configuration.ok) return configuration; + if (probe.listenerPids.length > 1) { return { ok: false, @@ -469,7 +502,15 @@ export function evaluateGatewayAttachment( * endpoint still goes through URL redaction so this stays safe if the contract * ever widens. */ -export function describeGatewayOwner(owner: GatewayOwner): JsonObject { +export function describeGatewayOwner(owner: { + gatewayName: string; + gatewayPort: number; + mode: GatewayManagementMode; + source: GatewayOwnerSource; + endpoint: string | null; + supervisor: GatewaySupervisorDeclaration | null; + requiredCapabilities: readonly string[]; +}): GatewayOwnerDescription { return { gatewayName: owner.gatewayName, gatewayPort: owner.gatewayPort, diff --git a/src/lib/state/onboard-session.test.ts b/src/lib/state/onboard-session.test.ts index dbcfb61b919..f31c6865bca 100644 --- a/src/lib/state/onboard-session.test.ts +++ b/src/lib/state/onboard-session.test.ts @@ -1421,6 +1421,39 @@ describe("onboard session", () => { expect(summary.resumable).toBe(false); }); + it("summarizes the checkpointed gateway authority without its state directory", () => { + const selected = session.createSession({ sandboxName: "my-assistant" }); + selected.checkpoint = { + ...selected.checkpoint!, + gatewayAuthority: { + kind: "selected", + value: { + gatewayName: "nemoclaw", + gatewayPort: 8080, + mode: "externally-supervised", + source: "declared", + endpoint: "http://127.0.0.1:8080", + stateDir: "/var/lib/openshell/private-gateway-state", + supervisor: { + kind: "systemd-system", + serviceName: "openshell-gateway.service", + execPath: "/usr/local/bin/openshell-gateway", + }, + requiredCapabilities: ["gateway.health"], + }, + }, + }; + + const summary = requireDebugSummary(session.summarizeForDebug(selected)); + + expect(summary.gatewayAuthority).toMatchObject({ + mode: "externally-supervised", + source: "declared", + supervisor: { serviceName: "openshell-gateway.service" }, + }); + expect(JSON.stringify(summary.gatewayAuthority)).not.toContain("private-gateway-state"); + }); + it("keeps debug summaries redacted when failures were sanitized", () => { session.saveSession(session.createSession({ sandboxName: "my-assistant" })); markStepFailedLegacy( diff --git a/src/lib/state/onboard-session.ts b/src/lib/state/onboard-session.ts index f182f4a206b..898c20908f6 100644 --- a/src/lib/state/onboard-session.ts +++ b/src/lib/state/onboard-session.ts @@ -19,6 +19,7 @@ import type { SandboxMessagingPlan } from "../messaging/manifest"; import { compactSandboxMessagingPlanForPersistence } from "../messaging/persistence"; import { parseSandboxMessagingPlan } from "../messaging/plan-validation"; import { NAME_MAX_LENGTH, NAME_VALID_PATTERN } from "../name-validation"; +import { describeGatewayOwner, type GatewayOwnerDescription } from "../onboard/gateway-ownership"; import { createOnboardMachineEvent, emitOnboardMachineEvent, @@ -299,6 +300,7 @@ export interface DebugSessionSummary { lastStepStarted: string | null; lastCompletedStep: string | null; failure: SessionFailure | null; + gatewayAuthority: GatewayOwnerDescription | null; machine: OnboardMachineSnapshot; steps: Record; } @@ -1689,6 +1691,10 @@ export function summarizeForDebug( session: Session | null = loadSession(), ): DebugSessionSummary | null { if (!session) return null; + const gatewayAuthority = + session.checkpoint?.gatewayAuthority.kind === "selected" + ? describeGatewayOwner(session.checkpoint.gatewayAuthority.value) + : null; return { version: session.version, sessionId: session.sessionId, @@ -1715,6 +1721,7 @@ export function summarizeForDebug( lastStepStarted: session.lastStepStarted, lastCompletedStep: session.lastCompletedStep, failure: sanitizeFailure(session.failure), + gatewayAuthority, machine: session.machine, steps: Object.fromEntries( Object.entries(session.steps).map(([name, step]) => [ diff --git a/src/lib/status-command-deps.ts b/src/lib/status-command-deps.ts index 10b0ba67d94..46a53d37fa4 100644 --- a/src/lib/status-command-deps.ts +++ b/src/lib/status-command-deps.ts @@ -24,6 +24,7 @@ import { } from "./messaging/hooks/status-runner"; import type { MessagingAgentId } from "./messaging/manifest"; import { resolveGatewayName } from "./onboard/gateway-binding"; +import { summarizeForDebug } from "./state/onboard-session"; import * as registry from "./state/registry"; import { createSystemDeps, parseSshProcesses } from "./state/sandbox-session"; import { getServiceStatuses, showStatus as showServiceStatus } from "./tunnel/services"; @@ -271,6 +272,7 @@ export function buildStatusCommandDeps(rootDir: string): ShowStatusCommandDeps { showServiceStatus, getServiceStatuses, getGatewayHealth: probeGatewayHealth, + getGatewayAuthority: () => summarizeForDebug()?.gatewayAuthority ?? null, getActiveSessionCount: sessionDeps ? (name) => { try { diff --git a/test/cli/debug-command.test.ts b/test/cli/debug-command.test.ts index 591327e042a..711e991b431 100644 --- a/test/cli/debug-command.test.ts +++ b/test/cli/debug-command.test.ts @@ -4,7 +4,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; - +import { writeExternalGatewayAuthoritySession } from "../helpers/gateway-authority-session"; import { describe, expect, test as it } from "../helpers/owned-test-resources"; import { @@ -41,6 +41,24 @@ describe("CLI debug command", () => { }, ); + it( + "debug --quick reports the selected gateway authority without its private state path", + testTimeoutOptions(30_000), + ({ resources }) => { + const env = createDebugCommandTestEnv(resources, "nemoclaw-cli-debug-authority-"); + if (!env.HOME) throw new Error("Expected debug test environment to set HOME"); + writeExternalGatewayAuthoritySession(env.HOME); + + const result = runWithEnv("debug --quick", env, 30000); + + expect(result.code).toBe(0); + expect(result.out).toContain('"gatewayAuthority"'); + expect(result.out).toContain('"mode": "externally-supervised"'); + expect(result.out).toContain('"serviceName": "openshell-gateway.service"'); + expect(result.out).not.toContain("private-gateway-state"); + }, + ); + it.skipIf(os.platform() !== "linux")( "debug --quick explains restricted dmesg instead of printing raw stderr on Linux", testTimeoutOptions(30_000), diff --git a/test/cli/status-root-json.test.ts b/test/cli/status-root-json.test.ts index fab97a8b0d2..6932efa7f6a 100644 --- a/test/cli/status-root-json.test.ts +++ b/test/cli/status-root-json.test.ts @@ -1,11 +1,12 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, it, expect } from "vitest"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { writeExternalGatewayAuthoritySession } from "../helpers/gateway-authority-session"; import { runWithEnv } from "./helpers"; describe("CLI root status JSON", () => { @@ -91,6 +92,7 @@ describe("CLI root status JSON", () => { ].join("\n"), { mode: 0o755 }, ); + writeExternalGatewayAuthoritySession(home); try { const r = runWithEnv("status --json", { @@ -116,6 +118,19 @@ describe("CLI root status JSON", () => { healthy: true, state: "healthy_named", }, + gatewayAuthority: { + gatewayName: "nemoclaw", + gatewayPort: 8080, + mode: "externally-supervised", + source: "declared", + endpoint: "http://127.0.0.1:8080/", + supervisor: { + kind: "systemd-system", + serviceName: "openshell-gateway.service", + execPath: "/usr/local/bin/openshell-gateway", + }, + requiredCapabilities: ["gateway.health", "sandbox.create"], + }, sandboxes: [ { name: sandboxName, @@ -139,11 +154,59 @@ describe("CLI root status JSON", () => { expect(r.out).not.toMatch( /Bearer|nvapi-|sk-|xoxb-|xapp-|password|api[-_]?key|dashboard-secret|should-not-render/i, ); + expect(r.out).not.toContain("private-gateway-state"); } finally { fs.rmSync(serviceDir, { recursive: true, force: true }); } }); + it("status text identifies the selected management mode and redacted owner", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-status-authority-")); + const localBin = path.join(home, "bin"); + fs.mkdirSync(localBin, { recursive: true }); + fs.mkdirSync(path.join(home, ".nemoclaw"), { recursive: true }); + fs.writeFileSync( + path.join(home, ".nemoclaw", "sandboxes.json"), + JSON.stringify({ + sandboxes: { + alpha: { + name: "alpha", + model: "configured-model", + provider: "configured-provider", + gpuEnabled: false, + policies: [], + }, + }, + defaultSandbox: "alpha", + }), + { mode: 0o600 }, + ); + fs.writeFileSync( + path.join(localBin, "openshell"), + [ + "#!/usr/bin/env bash", + 'if [ "$1" = "status" ]; then echo "Gateway: nemoclaw"; echo "Status: Connected"; exit 0; fi', + 'if [ "$1" = "gateway" ] && [ "$2" = "info" ]; then echo "Gateway: nemoclaw"; exit 0; fi', + "exit 0", + ].join("\n"), + { mode: 0o755 }, + ); + writeExternalGatewayAuthoritySession(home); + + const result = runWithEnv("status", { + HOME: home, + PATH: `${localBin}:${process.env.PATH || ""}`, + }); + + expect(result.code).toBe(0); + expect(result.out).toContain("Gateway authority: externally-supervised"); + expect(result.out).toContain( + "Owner: systemd-system openshell-gateway.service (/usr/local/bin/openshell-gateway)", + ); + expect(result.out).toContain("Endpoint: http://127.0.0.1:8080/"); + expect(result.out).not.toContain("private-gateway-state"); + }); + it("status --json reports gateway health and exits 1 when gateway is unhealthy", () => { const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-status-json-gateway-")); const localBin = path.join(home, "bin"); diff --git a/test/helpers/gateway-authority-session.ts b/test/helpers/gateway-authority-session.ts new file mode 100644 index 00000000000..2b978993a04 --- /dev/null +++ b/test/helpers/gateway-authority-session.ts @@ -0,0 +1,44 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import path from "node:path"; + +import { bindGatewayAuthorityToCheckpoint } from "../../src/lib/onboard/gateway-authority-checkpoint"; +import type { GatewayManagementDeclaration } from "../../src/lib/onboard/gateway-management"; +import { resolveGatewayOwner } from "../../src/lib/onboard/gateway-ownership"; +import { createSession } from "../../src/lib/state/onboard-session"; + +const EXTERNAL_GATEWAY_DECLARATION: GatewayManagementDeclaration = { + version: 1, + mode: "externally-supervised", + endpoint: "http://127.0.0.1:8080", + stateDir: "/var/lib/openshell/private-gateway-state", + supervisor: { + kind: "systemd-system", + serviceName: "openshell-gateway.service", + execPath: "/usr/local/bin/openshell-gateway", + }, + requiredCapabilities: ["gateway.health", "sandbox.create"], +}; + +export function writeExternalGatewayAuthoritySession(home: string): void { + const session = createSession({ + sessionId: "external-gateway-authority-session", + sandboxName: "alpha", + }); + bindGatewayAuthorityToCheckpoint( + session, + resolveGatewayOwner({ + gatewayName: "nemoclaw", + gatewayPort: 8080, + declaration: EXTERNAL_GATEWAY_DECLARATION, + hasPackagedService: false, + }), + ); + const stateDir = path.join(home, ".nemoclaw"); + fs.mkdirSync(stateDir, { recursive: true }); + fs.writeFileSync(path.join(stateDir, "onboard-session.json"), JSON.stringify(session), { + mode: 0o600, + }); +} From f8852af00779a001c9072a4443fd6f49d7bb710e Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Thu, 23 Jul 2026 14:56:05 -0700 Subject: [PATCH 10/20] test(debug): keep authority diagnostics linear --- test/cli/debug-command.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/cli/debug-command.test.ts b/test/cli/debug-command.test.ts index 711e991b431..ac15de1db3a 100644 --- a/test/cli/debug-command.test.ts +++ b/test/cli/debug-command.test.ts @@ -46,8 +46,8 @@ describe("CLI debug command", () => { testTimeoutOptions(30_000), ({ resources }) => { const env = createDebugCommandTestEnv(resources, "nemoclaw-cli-debug-authority-"); - if (!env.HOME) throw new Error("Expected debug test environment to set HOME"); - writeExternalGatewayAuthoritySession(env.HOME); + expect(env.HOME).toBeTypeOf("string"); + writeExternalGatewayAuthoritySession(env.HOME!); const result = runWithEnv("debug --quick", env, 30000); From d99ab3206aec6395f9f25b7ba413d6ff358af78e Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Thu, 23 Jul 2026 15:17:43 -0700 Subject: [PATCH 11/20] fix(onboard): scope recovery authority to target Signed-off-by: Prekshi Vyas --- src/lib/onboard/gateway-host-runtime.test.ts | 11 +++++ src/lib/onboard/gateway-host-runtime.ts | 43 ++++++++++++++------ src/lib/onboard/gateway-recovery.test.ts | 12 ++++-- src/lib/onboard/gateway-recovery.ts | 12 ++++-- 4 files changed, 59 insertions(+), 19 deletions(-) diff --git a/src/lib/onboard/gateway-host-runtime.test.ts b/src/lib/onboard/gateway-host-runtime.test.ts index 5261f708917..c7b268fc1ac 100644 --- a/src/lib/onboard/gateway-host-runtime.test.ts +++ b/src/lib/onboard/gateway-host-runtime.test.ts @@ -93,6 +93,17 @@ describe("gateway host runtime ownership", () => { ); }); + it("binds a recovery guard to its explicit gateway target (#6576)", () => { + const runtime = createGatewayHostRuntime(createDeps()); + + runtime.assertGatewayStartAllowed(false, { + gatewayName: "nemoclaw-8090", + gatewayPort: 8090, + }); + + expect(() => runtime.getGatewayOwner()).toThrow(/authority changed during this run/); + }); + it("fails closed on a malformed declaration rather than self-managing (#6576)", () => { declareExternalSupervision({ ...DECLARATION, version: 99 }); diff --git a/src/lib/onboard/gateway-host-runtime.ts b/src/lib/onboard/gateway-host-runtime.ts index a5a5d161a69..8ffdc3a68e5 100644 --- a/src/lib/onboard/gateway-host-runtime.ts +++ b/src/lib/onboard/gateway-host-runtime.ts @@ -85,7 +85,10 @@ export interface GatewayHostRuntime { * Fail before the caller can start a gateway that an external supervisor * owns. Applies to onboarding, rebuild, and recovery alike. */ - assertGatewayStartAllowed(exitOnFailure: boolean): void; + assertGatewayStartAllowed( + exitOnFailure: boolean, + target?: { gatewayName: string; gatewayPort: number }, + ): void; attachGateway(owner: GatewayOwner, expectedProbe: GatewayAttachmentProbe): Promise; bindGatewayOwner(owner: GatewayOwner): void; /** HTTPS endpoint of the gateway this process operates. */ @@ -107,6 +110,19 @@ export interface GatewayHostRuntime { export function createGatewayHostRuntime(deps: GatewayHostRuntimeDeps): GatewayHostRuntime { let boundOwner: GatewayOwner | null = null; + function resolveCurrentGatewayOwner(gatewayName: string, gatewayPort: number): GatewayOwner { + const loaded = loadGatewayManagementDeclaration(); + if (!loaded.ok) { + throw new Error(`Invalid gateway management declaration: ${loaded.reason}`); + } + return resolveGatewayOwner({ + gatewayName, + gatewayPort, + declaration: loaded.declaration, + hasPackagedService: hasOpenShellGatewayUserService(), + }); + } + /** * Resolve the one gateway lifecycle authority for this run. A malformed * declaration throws instead of degrading to self-management: a host that @@ -121,17 +137,13 @@ export function createGatewayHostRuntime(deps: GatewayHostRuntimeDeps): GatewayH * migration, not something a mutating file can do underneath a running * onboard (#6576). */ - function getGatewayOwner(): GatewayOwner { - const loaded = loadGatewayManagementDeclaration(); - if (!loaded.ok) { - throw new Error(`Invalid gateway management declaration: ${loaded.reason}`); - } - const resolved = resolveGatewayOwner({ + function getGatewayOwnerForTarget( + target: { gatewayName: string; gatewayPort: number } = { gatewayName: deps.gatewayName(), gatewayPort: deps.gatewayPort(), - declaration: loaded.declaration, - hasPackagedService: hasOpenShellGatewayUserService(), - }); + }, + ): GatewayOwner { + const resolved = resolveCurrentGatewayOwner(target.gatewayName, target.gatewayPort); if (boundOwner) { if (!sameGatewayOwner(boundOwner, resolved)) { throw new Error( @@ -146,6 +158,10 @@ export function createGatewayHostRuntime(deps: GatewayHostRuntimeDeps): GatewayH return boundOwner; } + function getGatewayOwner(): GatewayOwner { + return getGatewayOwnerForTarget(); + } + function isSupervisorUnitActive(owner: GatewayOwner): boolean | null { const supervisor = owner.supervisor; if (!supervisor) return null; @@ -306,9 +322,12 @@ export function createGatewayHostRuntime(deps: GatewayHostRuntimeDeps): GatewayH }; } - function assertGatewayStartAllowed(exitOnFailure: boolean): void { + function assertGatewayStartAllowed( + exitOnFailure: boolean, + target?: { gatewayName: string; gatewayPort: number }, + ): void { try { - assertGatewayEffectAllowed(getGatewayOwner(), "start"); + assertGatewayEffectAllowed(getGatewayOwnerForTarget(target), "start"); } catch (error) { console.error(` ${error instanceof Error ? error.message : String(error)}`); if (exitOnFailure) process.exit(1); diff --git a/src/lib/onboard/gateway-recovery.test.ts b/src/lib/onboard/gateway-recovery.test.ts index af71abc565f..558f2c1f354 100644 --- a/src/lib/onboard/gateway-recovery.test.ts +++ b/src/lib/onboard/gateway-recovery.test.ts @@ -296,10 +296,13 @@ describe("gateway lifecycle authority during recovery", () => { // The cross-port, non-default-name target is the branch that reaches a raw // `openshell gateway start` without going through startGatewayWithOptions. await expect( - startGatewayForRecovery({ gatewayName: "other", gatewayPort: 8080 }, deps), + startGatewayForRecovery({ gatewayName: "nemoclaw-8090", gatewayPort: 8090 }, deps), ).rejects.toThrow(ownershipError); - expect(deps.assertGatewayStartAllowed).toHaveBeenCalledWith(false); + expect(deps.assertGatewayStartAllowed).toHaveBeenCalledWith(false, { + gatewayName: "nemoclaw-8090", + gatewayPort: 8090, + }); expect(deps.runOpenshell).not.toHaveBeenCalled(); expect(deps.startGatewayWithOptions).not.toHaveBeenCalled(); }); @@ -309,7 +312,10 @@ describe("gateway lifecycle authority during recovery", () => { await startGatewayForRecovery({}, deps); - expect(deps.assertGatewayStartAllowed).toHaveBeenCalledWith(false); + expect(deps.assertGatewayStartAllowed).toHaveBeenCalledWith(false, { + gatewayName: "nemoclaw", + gatewayPort: 8080, + }); expect(deps.startGatewayWithOptions).toHaveBeenCalledOnce(); }); }); diff --git a/src/lib/onboard/gateway-recovery.ts b/src/lib/onboard/gateway-recovery.ts index cbc90b25fe9..b51677a9ba6 100644 --- a/src/lib/onboard/gateway-recovery.ts +++ b/src/lib/onboard/gateway-recovery.ts @@ -53,7 +53,10 @@ export type GatewayRecoveryDeps = { * supervisor owns (#6576). Optional so existing test harnesses keep working; * production wiring always supplies it. */ - assertGatewayStartAllowed?(exitOnFailure: boolean): void; + assertGatewayStartAllowed?( + exitOnFailure: boolean, + target: { gatewayName: string; gatewayPort: number }, + ): void; getGatewayClusterContainerState?(gatewayName: string): string; getGatewayStartEnv(): Record; runCaptureOpenshell(args: string[], opts?: RunCaptureOpenshellOptions): string; @@ -267,11 +270,12 @@ export async function startGatewayForRecovery( options: StartGatewayForRecoveryOptions, deps: GatewayRecoveryDeps, ): Promise { + const target = resolveGatewayRecoveryTarget(options); // Guard every recovery branch, including the cross-port / non-default-name // path below that reaches `openshell gateway start` without going through - // startGatewayWithOptions. - deps.assertGatewayStartAllowed?.(false); - const target = resolveGatewayRecoveryTarget(options); + // startGatewayWithOptions. Resolve and bind the requested target first: the + // process-global gateway can name a different port during sandbox recovery. + deps.assertGatewayStartAllowed?.(false, target); const linuxDockerDriverEnabled = ( deps.isLinuxDockerDriverGatewayEnabled ?? isLinuxDockerDriverGatewayEnabled )(); From 2539a25de431b4884bc99b82021dde996ba54c2b Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Thu, 23 Jul 2026 15:43:06 -0700 Subject: [PATCH 12/20] fix(onboard): require recovery authority guard Signed-off-by: Prekshi Vyas --- src/lib/onboard/gateway-recovery.test.ts | 1 + src/lib/onboard/gateway-recovery.ts | 7 +++---- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/lib/onboard/gateway-recovery.test.ts b/src/lib/onboard/gateway-recovery.test.ts index 558f2c1f354..6a86b96fd36 100644 --- a/src/lib/onboard/gateway-recovery.test.ts +++ b/src/lib/onboard/gateway-recovery.test.ts @@ -32,6 +32,7 @@ function makeVirtualClock(startMs = 1_000_000_000_000) { function createDeps(overrides: Partial = {}): GatewayRecoveryDeps { return { + assertGatewayStartAllowed: vi.fn(), getGatewayClusterContainerState: () => "missing", getGatewayStartEnv: () => ({ OPENSHELL_DRIVERS: "docker" }), runCaptureOpenshell: vi.fn(() => "Disconnected"), diff --git a/src/lib/onboard/gateway-recovery.ts b/src/lib/onboard/gateway-recovery.ts index b51677a9ba6..84b8d94b876 100644 --- a/src/lib/onboard/gateway-recovery.ts +++ b/src/lib/onboard/gateway-recovery.ts @@ -50,10 +50,9 @@ type GatewayStartResult = { export type GatewayRecoveryDeps = { /** * Fail closed before any recovery branch starts a gateway process an external - * supervisor owns (#6576). Optional so existing test harnesses keep working; - * production wiring always supplies it. + * supervisor owns (#6576). */ - assertGatewayStartAllowed?( + assertGatewayStartAllowed( exitOnFailure: boolean, target: { gatewayName: string; gatewayPort: number }, ): void; @@ -275,7 +274,7 @@ export async function startGatewayForRecovery( // path below that reaches `openshell gateway start` without going through // startGatewayWithOptions. Resolve and bind the requested target first: the // process-global gateway can name a different port during sandbox recovery. - deps.assertGatewayStartAllowed?.(false, target); + deps.assertGatewayStartAllowed(false, target); const linuxDockerDriverEnabled = ( deps.isLinuxDockerDriverGatewayEnabled ?? isLinuxDockerDriverGatewayEnabled )(); From 57ebaf79e0a5e25f3f07c9033a7688efe1ab5627 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Thu, 23 Jul 2026 23:06:01 -0700 Subject: [PATCH 13/20] fix(gateway): preserve external teardown authority --- .../gateway-lifecycle-authority.mdx | 11 ++ docs/reference/commands.mdx | 21 ++- .../destroy-gateway-runtime-evidence.test.ts | 16 ++ .../actions/sandbox/destroy-gateway.test.ts | 80 ++++++++++ src/lib/actions/sandbox/destroy-gateway.ts | 50 ++++-- .../run-plan-gateway-segregation.test.ts | 150 ++++++++++++++++++ src/lib/actions/uninstall/run-plan.ts | 79 +++++++-- .../gateway-teardown-authority.test.ts | 127 +++++++++++++++ src/lib/onboard/gateway-teardown-authority.ts | 116 ++++++++++++++ .../onboard-session-gateway-authority.test.ts | 41 +++++ src/lib/state/onboard-session.test.ts | 33 ---- .../gateway-port-release-lifecycle.test.ts | 49 ++++++ .../gateway-port-release-test-helpers.ts | 10 ++ src/lib/tunnel/gateway-port-release.ts | 27 +++- test/cli/debug-command.test.ts | 2 +- test/cli/status-root-json.test.ts | 2 +- 16 files changed, 754 insertions(+), 60 deletions(-) create mode 100644 src/lib/onboard/gateway-teardown-authority.test.ts create mode 100644 src/lib/onboard/gateway-teardown-authority.ts create mode 100644 src/lib/state/onboard-session-gateway-authority.test.ts diff --git a/docs/deployment/gateway-lifecycle-authority.mdx b/docs/deployment/gateway-lifecycle-authority.mdx index 589c4521a88..47416aa758a 100644 --- a/docs/deployment/gateway-lifecycle-authority.mdx +++ b/docs/deployment/gateway-lifecycle-authority.mdx @@ -165,6 +165,17 @@ The external attachment path has no standalone fallback. Recovery and rebuild guard every start branch, including non-default port recovery. The gateway state handler validates and attaches instead of starting, stopping, restarting, destroying, or replacing the gateway. +Stop, final-sandbox cleanup, and uninstall reload the declaration before gateway teardown. +When a valid checkpoint exists, they compare the current authority with that checkpoint. +The comparison uses the exact gateway name and port. +If the authority changed, teardown stops before it scans listeners or changes gateway runtime resources. + +`nemoclaw stop` does not scan or signal the externally supervised gateway. +Final-sandbox cleanup can stop local dashboard forwards and remove the modern local gateway registration. +It does not signal the gateway, use the legacy `gateway destroy` fallback, or remove its Docker volumes. +Uninstall can delete the selected sandboxes, providers, and local registration. +It keeps the externally supervised gateway process, Docker resources, and OpenShell binaries. + Registering and selecting the validated endpoint changes only the local OpenShell gateway registration used by downstream commands. It does not transfer process ownership to NemoClaw. diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index e0c8800b3ce..b4bd544baea 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -3139,11 +3139,21 @@ $$nemoclaw gc [--dry-run] [--yes|-y|--force] ### `$$nemoclaw uninstall` -Run `uninstall.sh` to remove NemoClaw sandboxes, gateway resources, related images and containers, and local state. +Run `uninstall.sh` to remove NemoClaw sandboxes, local gateway resources, related images and containers, and local state. The CLI runs the local `uninstall.sh` shipped with the installed npm package. If that local script is missing, the CLI does not auto-fetch a remote copy. It prints the versioned URL of the matching `uninstall.sh` so you can download, review, and run it manually. +When the gateway is externally supervised, uninstall preserves its process, Docker resources, and OpenShell binaries. +It still deletes the selected sandboxes and attempts to remove the modern local gateway registration. +When no sibling gateways remain, it also deletes NemoClaw provider registrations. +It does not use the legacy `gateway destroy` command for that gateway. + + +Refer to [Declare the OpenShell Gateway Lifecycle Authority](../deployment/gateway-lifecycle-authority). + + + Uninstall also stops any orphaned `openshell` host processes left behind by previous onboard or destroy cycles, including `openshell sandbox create`, `openshell ssh-proxy`, and SSH sessions spawned by OpenShell. Earlier releases only stopped `openshell forward` processes, so those orphans accumulated across runs. @@ -3152,7 +3162,8 @@ For Local Ollama setups, uninstall also stops matching Ollama auth proxy process For Hermes setups, uninstall inspects the selected gateway's managed port-forward watcher state, stops each verified watcher process and its sandbox-scoped forward, and leaves sibling gateway state untouched. If any watcher or forward cleanup cannot be confirmed, uninstall exits nonzero and preserves the selected gateway's watcher state so you can retry cleanup. -On Linux, uninstall removes `~/.local/state/nemoclaw`, which contains Docker-driver gateway SQLite data, audit logs, VM-driver state, and standalone-fallback gateway PID files. +On Linux, uninstall removes `~/.local/state/nemoclaw`, which contains NemoClaw-owned Docker-driver gateway SQLite data, audit logs, VM-driver state, and standalone-fallback gateway PID files. +This directory is separate from a declared external gateway state directory, which uninstall does not remove. | Flag | Effect | |---|---| @@ -3203,7 +3214,11 @@ Decision matrix: The preserved entries survive uninstall as inert files on disk. Reinstall NemoClaw and re-onboard the sandbox before `$$nemoclaw snapshot restore` can use them. -Preserving `sandboxes.json` does not make the recorded sandboxes recoverable on their own: uninstall removes the gateway registration, provider registrations, and Docker image those records depend on. +The preserved `sandboxes.json` file does not make the recorded sandboxes recoverable on its own. +Uninstall deletes the selected sandboxes and attempts to remove the local gateway registration. +When no sibling gateways remain, it also deletes provider registrations. +For a NemoClaw-managed gateway, it also removes the Docker image. +For an externally supervised gateway, it preserves Docker resources, but the registry still cannot recover deleted sandbox and provider resources. Uninstall warns about this at preserve time. After reinstalling, the installer reports such records as not found on their recorded gateway instead of claiming they were recovered; run `$$nemoclaw destroy` to clear a stranded record, then `$$nemoclaw onboard` to rebuild it. Pass `--destroy-user-data` at uninstall time if you prefer to purge the registry along with its dependencies. diff --git a/src/lib/actions/sandbox/destroy-gateway-runtime-evidence.test.ts b/src/lib/actions/sandbox/destroy-gateway-runtime-evidence.test.ts index 4639f645cee..6868bbc3b2e 100644 --- a/src/lib/actions/sandbox/destroy-gateway-runtime-evidence.test.ts +++ b/src/lib/actions/sandbox/destroy-gateway-runtime-evidence.test.ts @@ -9,6 +9,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; const mocks = vi.hoisted(() => ({ dockerRemoveVolumesByPrefix: vi.fn(), + resolveGatewayTeardownAuthority: vi.fn(), spawnSync: vi.fn(), stopStaleDashboardListeners: vi.fn(), })); @@ -19,6 +20,9 @@ vi.mock("node:child_process", () => ({ vi.mock("../../adapters/docker/volume", () => ({ dockerRemoveVolumesByPrefix: mocks.dockerRemoveVolumesByPrefix, })); +vi.mock("../../onboard/gateway-teardown-authority", () => ({ + resolveGatewayTeardownAuthority: mocks.resolveGatewayTeardownAuthority, +})); vi.mock("../../onboard/stale-gateway-cleanup", () => ({ stopStaleDashboardListeners: mocks.stopStaleDashboardListeners, })); @@ -31,6 +35,18 @@ describe("cleanupGatewayAfterLastSandbox runtime evidence", () => { beforeEach(() => { stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-destroy-gateway-evidence-")); vi.stubEnv("NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR", stateDir); + mocks.resolveGatewayTeardownAuthority.mockImplementation( + ({ gatewayName, gatewayPort }: { gatewayName: string; gatewayPort: number }) => ({ + gatewayName, + gatewayPort, + mode: "nemoclaw-managed", + source: "standalone", + endpoint: null, + stateDir: null, + supervisor: null, + requiredCapabilities: [], + }), + ); }); afterEach(() => { diff --git a/src/lib/actions/sandbox/destroy-gateway.test.ts b/src/lib/actions/sandbox/destroy-gateway.test.ts index ac47fe5d7fd..7f10d13bf33 100644 --- a/src/lib/actions/sandbox/destroy-gateway.test.ts +++ b/src/lib/actions/sandbox/destroy-gateway.test.ts @@ -8,6 +8,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; const mocks = vi.hoisted(() => ({ dockerRemoveVolumesByPrefix: vi.fn(), + resolveGatewayTeardownAuthority: vi.fn(), stopHostGatewayProcesses: vi.fn(), stopStaleDashboardListeners: vi.fn(), })); @@ -18,6 +19,9 @@ vi.mock("../../adapters/docker/volume", () => ({ vi.mock("../../onboard/host-gateway-process", () => ({ stopHostGatewayProcesses: mocks.stopHostGatewayProcesses, })); +vi.mock("../../onboard/gateway-teardown-authority", () => ({ + resolveGatewayTeardownAuthority: mocks.resolveGatewayTeardownAuthority, +})); vi.mock("../../onboard/stale-gateway-cleanup", () => ({ stopStaleDashboardListeners: mocks.stopStaleDashboardListeners, })); @@ -26,6 +30,18 @@ import { cleanupGatewayAfterLastSandbox } from "./destroy-gateway"; describe("cleanupGatewayAfterLastSandbox", () => { beforeEach(() => { + mocks.resolveGatewayTeardownAuthority.mockImplementation( + ({ gatewayName, gatewayPort }: { gatewayName: string; gatewayPort: number }) => ({ + gatewayName, + gatewayPort, + mode: "nemoclaw-managed", + source: "standalone", + endpoint: null, + stateDir: null, + supervisor: null, + requiredCapabilities: [], + }), + ); mocks.stopHostGatewayProcesses.mockReturnValue({ failed: [], skippedDeadPids: [], @@ -35,6 +51,70 @@ describe("cleanupGatewayAfterLastSandbox", () => { }); }); + it.each([ + "systemd-system", + "systemd-user", + ] as const)("does not stop or destroy a %s-supervised gateway during final-sandbox cleanup (#6576)", (kind) => { + mocks.resolveGatewayTeardownAuthority.mockImplementationOnce( + ({ gatewayName, gatewayPort }: { gatewayName: string; gatewayPort: number }) => ({ + gatewayName, + gatewayPort, + mode: "externally-supervised", + source: "declared", + endpoint: `http://127.0.0.1:${String(gatewayPort)}`, + stateDir: "/var/lib/openshell/gateway", + supervisor: { + kind, + serviceName: "openshell-gateway.service", + execPath: "/usr/local/bin/openshell-gateway", + }, + requiredCapabilities: [], + }), + ); + const runOpenshell = vi.fn((args: string[]) => + args[1] === "remove" + ? { status: 2, stdout: "", stderr: "unrecognized subcommand 'remove'" } + : { status: 0, stdout: "", stderr: "" }, + ); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + + cleanupGatewayAfterLastSandbox("nemoclaw", runOpenshell); + + expect(mocks.resolveGatewayTeardownAuthority).toHaveBeenCalledWith( + { gatewayName: "nemoclaw", gatewayPort: 8080 }, + { env: process.env }, + ); + expect(mocks.stopStaleDashboardListeners).toHaveBeenCalledOnce(); + expect(mocks.stopHostGatewayProcesses).not.toHaveBeenCalled(); + expect(runOpenshell).toHaveBeenCalledWith(["gateway", "remove", "nemoclaw"], { + ignoreError: true, + stdio: ["ignore", "pipe", "pipe"], + }); + expect(runOpenshell).not.toHaveBeenCalledWith( + ["gateway", "destroy", "-g", "nemoclaw"], + expect.anything(), + ); + expect(mocks.dockerRemoveVolumesByPrefix).not.toHaveBeenCalled(); + expect(warn).toHaveBeenCalledWith( + expect.stringContaining("will not use the legacy gateway destroy command"), + ); + }); + + it("fails before local cleanup when the gateway authority cannot be revalidated (#6576)", () => { + mocks.resolveGatewayTeardownAuthority.mockImplementationOnce(() => { + throw new Error("authority drift"); + }); + const runOpenshell = vi.fn(() => ({ status: 0, stdout: "", stderr: "" })); + + expect(() => cleanupGatewayAfterLastSandbox("nemoclaw", runOpenshell)).toThrow( + "authority drift", + ); + expect(runOpenshell).not.toHaveBeenCalled(); + expect(mocks.stopStaleDashboardListeners).not.toHaveBeenCalled(); + expect(mocks.stopHostGatewayProcesses).not.toHaveBeenCalled(); + expect(mocks.dockerRemoveVolumesByPrefix).not.toHaveBeenCalled(); + }); + afterEach(() => { vi.restoreAllMocks(); vi.clearAllMocks(); diff --git a/src/lib/actions/sandbox/destroy-gateway.ts b/src/lib/actions/sandbox/destroy-gateway.ts index df6921ace25..32c495fe2ce 100644 --- a/src/lib/actions/sandbox/destroy-gateway.ts +++ b/src/lib/actions/sandbox/destroy-gateway.ts @@ -11,6 +11,11 @@ import { resolveGatewayPortFromName, resolveGatewayStateDirName, } from "../../onboard/gateway-binding"; +import { isExternallySupervised } from "../../onboard/gateway-ownership"; +import { + type GatewayTeardownAuthorityResolver, + resolveGatewayTeardownAuthority, +} from "../../onboard/gateway-teardown-authority"; import { stopHostGatewayProcesses } from "../../onboard/host-gateway-process"; import { stopStaleDashboardListeners } from "../../onboard/stale-gateway-cleanup"; @@ -21,6 +26,10 @@ export type DestroyRunOpenshell = ( const DASHBOARD_FORWARD_PORT = String(DASHBOARD_PORT); +export interface CleanupGatewayDeps { + resolveGatewayTeardownAuthority?: GatewayTeardownAuthorityResolver; +} + // Compute the Docker-driver gateway state directory that belongs to // `gatewayName`. `stopHostGatewayProcesses` defaults to the bare leaf // `openshell-docker-gateway`, so without this override a destroy of a @@ -71,7 +80,17 @@ export function selectGatewayForSandboxDestroy( export function cleanupGatewayAfterLastSandbox( gatewayName: string, runOpenshell?: DestroyRunOpenshell, + deps: CleanupGatewayDeps = {}, ): void { + const perGatewayState = resolvePerGatewayState(gatewayName); + if (!perGatewayState) { + throw new Error(`Refusing cleanup for noncanonical NemoClaw gateway '${gatewayName}'.`); + } + const owner = (deps.resolveGatewayTeardownAuthority ?? resolveGatewayTeardownAuthority)( + { gatewayName, gatewayPort: perGatewayState.port }, + { env: process.env }, + ); + const externallySupervised = isExternallySupervised(owner); const openshell = runOpenshell ?? (require("../../adapters/openshell/runtime") as { runOpenshell: DestroyRunOpenshell }) @@ -86,7 +105,7 @@ export function cleanupGatewayAfterLastSandbox( // ports the live openshell tracks; this catches orphans whose openshell // record was lost across upgrades or failed onboards. stopStaleDashboardListeners(); - if (process.platform === "linux" || process.platform === "darwin") { + if (!externallySupervised && (process.platform === "linux" || process.platform === "darwin")) { // Sandbox destroy is conservative: only stop the host gateway whose PID // file we wrote during onboard. Disable the pgrep sweep so a stray // openshell-gateway under another user/project on the same host (rare but @@ -96,7 +115,6 @@ export function cleanupGatewayAfterLastSandbox( // per-port pid file rather than defaulting to the bare instance's. The // expected gateway name and port also gate `openshell gateway start` // cmdlines so a stale pid file cannot kill another gateway instance. - const perGatewayState = resolvePerGatewayState(gatewayName); const stopOptions: { openShellGatewayName?: string; openShellGatewayPort?: number; @@ -108,12 +126,10 @@ export function cleanupGatewayAfterLastSandbox( preserveRuntimeFilesOnNonMatching: true, usePgrepFallback: false, }; - if (perGatewayState) { - stopOptions.stateDir = perGatewayState.stateDir; - stopOptions.pidFile = path.join(perGatewayState.stateDir, "openshell-gateway.pid"); - stopOptions.openShellGatewayName = gatewayName; - stopOptions.openShellGatewayPort = perGatewayState.port; - } + stopOptions.stateDir = perGatewayState.stateDir; + stopOptions.pidFile = path.join(perGatewayState.stateDir, "openshell-gateway.pid"); + stopOptions.openShellGatewayName = gatewayName; + stopOptions.openShellGatewayPort = perGatewayState.port; const stopResult = stopHostGatewayProcesses({}, stopOptions); const unverifiablePids = [...new Set(stopResult.skippedNonMatchingPids)]; if (unverifiablePids.length > 0) { @@ -160,10 +176,20 @@ export function cleanupGatewayAfterLastSandbox( stdio: ["ignore", "pipe", "pipe"], }); if (removeResult.status !== 0) { - openshell(["gateway", "destroy", "-g", gatewayName], { - ignoreError: true, - stdio: ["ignore", "pipe", "pipe"], - }); + if (externallySupervised) { + console.warn( + `Could not remove local registration for externally supervised gateway '${gatewayName}'. ` + + "NemoClaw will not use the legacy gateway destroy command for an externally supervised gateway.", + ); + } else { + openshell(["gateway", "destroy", "-g", gatewayName], { + ignoreError: true, + stdio: ["ignore", "pipe", "pipe"], + }); + } + } + if (externallySupervised) { + return; } dockerRemoveVolumesByPrefix(`openshell-cluster-${gatewayName}`, { ignoreError: true, diff --git a/src/lib/actions/uninstall/run-plan-gateway-segregation.test.ts b/src/lib/actions/uninstall/run-plan-gateway-segregation.test.ts index 23f07fd9a49..2caaf05907a 100644 --- a/src/lib/actions/uninstall/run-plan-gateway-segregation.test.ts +++ b/src/lib/actions/uninstall/run-plan-gateway-segregation.test.ts @@ -21,6 +21,156 @@ afterEach(() => { }); describe("uninstall gateway-port segregation (#3053)", () => { + it.each([ + ["full", "systemd-system"], + ["full", "systemd-user"], + ["scoped", "systemd-system"], + ["scoped", "systemd-user"], + ] as const)("preserves the gateway process, Docker resources, and OpenShell binaries during %s uninstall for a %s-supervised gateway (#6576)", (scope, kind) => { + const tmpHome = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-uninstall-external-")); + try { + const stateDir = path.join(tmpHome, ".nemoclaw"); + fs.mkdirSync(stateDir, { recursive: true }); + if (scope === "scoped") { + fs.writeFileSync( + path.join(stateDir, "sandboxes.json"), + JSON.stringify({ + defaultSandbox: "alpha", + sandboxes: { + alpha: { name: "alpha", gatewayName: "nemoclaw", gatewayPort: 8080 }, + beta: { name: "beta", gatewayName: "nemoclaw-8091", gatewayPort: 8091 }, + }, + }), + ); + } + const calls: Array<{ args: string[]; command: string }> = []; + const dockerCalls: string[][] = []; + const kill = vi.fn(() => true); + + const result = runUninstallPlan( + { assumeYes: true, deleteModels: false, keepOpenShell: false }, + { + commandExists: () => true, + env: { HOME: tmpHome, LOGNAME: "tester" } as NodeJS.ProcessEnv, + existsSync: (target) => target.startsWith(tmpHome) && fs.existsSync(target), + isTty: false, + kill, + log: vi.fn(), + resolveGatewayTeardownAuthority: ({ gatewayName, gatewayPort }) => ({ + gatewayName, + gatewayPort, + mode: "externally-supervised", + source: "declared", + endpoint: `http://127.0.0.1:${String(gatewayPort)}`, + stateDir: "/var/lib/openshell/gateway", + supervisor: { + kind, + serviceName: "openshell-gateway.service", + execPath: "/usr/local/bin/openshell-gateway", + }, + requiredCapabilities: [], + }), + rmSync: fs.rmSync, + run: (command, args) => { + calls.push({ args, command }); + return ok(); + }, + runDocker: (args) => { + dockerCalls.push(args); + return ok(); + }, + }, + ); + + expect(result.exitCode).toBe(0); + const openshellCalls = calls + .filter(({ command }) => command === "openshell") + .map(({ args }) => args); + expect(openshellCalls).toContainEqual(["gateway", "remove", "nemoclaw"]); + expect(openshellCalls).not.toContainEqual(["gateway", "destroy", "-g", "nemoclaw"]); + expect(calls.some(({ args }) => args.join(" ").includes("openshell-gateway"))).toBe(false); + expect(kill).not.toHaveBeenCalled(); + expect(dockerCalls).toEqual([]); + expect( + calls.some( + ({ command, args }) => + command === "rm" && args.includes("/usr/local/bin/openshell-gateway"), + ), + ).toBe(false); + } finally { + fs.rmSync(tmpHome, { recursive: true, force: true }); + } + }); + + it("does not use legacy gateway destroy when external registration removal is unsupported (#6576)", () => { + const calls: Array<{ args: string[]; command: string }> = []; + const result = runUninstallPlan( + { assumeYes: true, deleteModels: false, keepOpenShell: true }, + { + commandExists: (command) => command === "openshell", + env: { HOME: "/home/test" } as NodeJS.ProcessEnv, + existsSync: () => false, + isTty: false, + resolveGatewayTeardownAuthority: ({ gatewayName, gatewayPort }) => ({ + gatewayName, + gatewayPort, + mode: "externally-supervised", + source: "declared", + endpoint: `http://127.0.0.1:${String(gatewayPort)}`, + stateDir: "/var/lib/openshell/gateway", + supervisor: { + kind: "systemd-system", + serviceName: "openshell-gateway.service", + execPath: "/usr/local/bin/openshell-gateway", + }, + requiredCapabilities: [], + }), + rmSync: vi.fn(), + run: (command, args) => { + calls.push({ args, command }); + if ([command, ...args].join(" ") === "openshell gateway remove nemoclaw") { + return { status: 2, stdout: "", stderr: "unrecognized subcommand 'remove'" }; + } + return ok(); + }, + runDocker: () => ok(), + }, + ); + + expect(result.exitCode).toBe(0); + const openshellCalls = calls + .filter(({ command }) => command === "openshell") + .map(({ args }) => args); + expect(openshellCalls).toContainEqual(["gateway", "remove", "nemoclaw"]); + expect(openshellCalls).not.toContainEqual(["gateway", "destroy", "-g", "nemoclaw"]); + }); + + it("fails before uninstall effects when gateway authority revalidation fails (#6576)", () => { + const run = vi.fn(() => ok()); + const runDocker = vi.fn(() => ok()); + const rmSync = vi.fn(); + + const result = runUninstallPlan( + { assumeYes: true, deleteModels: false, keepOpenShell: true }, + { + env: { HOME: "/home/test" } as NodeJS.ProcessEnv, + error: vi.fn(), + existsSync: () => false, + resolveGatewayTeardownAuthority: () => { + throw new Error("authority drift"); + }, + rmSync, + run, + runDocker, + }, + ); + + expect(result.exitCode).toBe(1); + expect(run).not.toHaveBeenCalled(); + expect(runDocker).not.toHaveBeenCalled(); + expect(rmSync).not.toHaveBeenCalled(); + }); + it("falls back to legacy gateway destroy only when gateway remove is unsupported", () => { const calls: Array<{ args: string[]; command: string }> = []; const responses = new Map([ diff --git a/src/lib/actions/uninstall/run-plan.ts b/src/lib/actions/uninstall/run-plan.ts index 88098a6e453..b503b5d46b0 100644 --- a/src/lib/actions/uninstall/run-plan.ts +++ b/src/lib/actions/uninstall/run-plan.ts @@ -27,6 +27,11 @@ import { import { buildUninstallPlan, type UninstallPlan } from "../../domain/uninstall/plan"; import { isOllamaAuthProxyCommandLine } from "../../inference/ollama/process"; import { resolveGatewayName } from "../../onboard/gateway-binding"; +import { isExternallySupervised } from "../../onboard/gateway-ownership"; +import { + type GatewayTeardownAuthorityResolver, + resolveGatewayTeardownAuthority, +} from "../../onboard/gateway-teardown-authority"; import { stopHostGatewayProcesses } from "../../onboard/host-gateway-process"; import { isModelRouterCommandLineForPort } from "../../onboard/model-router-process"; import { stopStaleDashboardListeners } from "../../onboard/stale-gateway-cleanup"; @@ -66,6 +71,7 @@ export interface UninstallRunDeps { log?: (message: string) => void; readProcessArgv?: (pid: number) => readonly string[] | null; readLine?: () => string | null; + resolveGatewayTeardownAuthority?: GatewayTeardownAuthorityResolver; rmSync?: typeof fs.rmSync; run?: (command: string, args: string[], options?: SpawnSyncOptions) => RunResult; runDocker?: (args: string[], options?: SpawnSyncOptions) => RunResult; @@ -291,6 +297,7 @@ interface UninstallRuntime { log: (message: string) => void; readProcessArgv: ((pid: number) => readonly string[] | null) | undefined; readLine: () => string | null; + resolveGatewayTeardownAuthority: GatewayTeardownAuthorityResolver; rmSync: typeof fs.rmSync; run: (command: string, args: string[], options?: SpawnSyncOptions) => RunResult; runDocker: (args: string[], options?: SpawnSyncOptions) => RunResult; @@ -320,6 +327,8 @@ function buildRuntime(deps: UninstallRunDeps): UninstallRuntime { log: deps.log ?? ((message) => console.log(message)), readProcessArgv: deps.readProcessArgv, readLine: deps.readLine ?? readLineFromStdin, + resolveGatewayTeardownAuthority: + deps.resolveGatewayTeardownAuthority ?? resolveGatewayTeardownAuthority, rmSync: deps.rmSync ?? fs.rmSync, run: deps.run ?? defaultRun, runDocker: deps.runDocker ?? defaultRunDocker, @@ -443,7 +452,11 @@ const GATEWAY_ALREADY_ABSENT = const GATEWAY_REMOVE_UNSUPPORTED = /unrecognized subcommand ['"]remove['"]|unknown command ['"]remove['"]/i; -function removeGatewayRegistration(runtime: UninstallRuntime, gatewayLabel: string): boolean { +function removeGatewayRegistration( + runtime: UninstallRuntime, + gatewayLabel: string, + allowLegacyDestroy: boolean, +): boolean { const removeResult = runtime.run("openshell", ["gateway", "remove", gatewayLabel], { env: runtime.env, }); @@ -461,6 +474,13 @@ function removeGatewayRegistration(runtime: UninstallRuntime, gatewayLabel: stri runtime.warn(gatewayDestroySkipMessage(gatewayLabel)); return false; } + if (!allowLegacyDestroy) { + runtime.warn( + `Could not remove local registration for externally supervised gateway '${gatewayLabel}'. ` + + "NemoClaw will not use the legacy gateway destroy command for an externally supervised gateway.", + ); + return false; + } // OpenShell builds before 0.0.44 exposed `gateway destroy` instead of the // current `gateway remove` command. Only fall back when the modern verb is @@ -867,6 +887,7 @@ function removeOpenShellResources( runtime: UninstallRuntime, scopedToSelectedGateway: boolean, sandboxNames: readonly string[], + externallySupervised: boolean, ): boolean { if (!runtime.commandExists("openshell")) { runtime.warn("openshell not found; skipping gateway/provider/sandbox cleanup."); @@ -896,7 +917,8 @@ function removeOpenShellResources( ) && removedSelectedResources; } removedSelectedResources = - removeGatewayRegistration(runtime, gatewayLabel) && removedSelectedResources; + removeGatewayRegistration(runtime, gatewayLabel, !externallySupervised) && + removedSelectedResources; if (!removedSelectedResources) { runtime.warn("Selected gateway cleanup was incomplete; preserving its state for retry."); return false; @@ -924,7 +946,7 @@ function removeOpenShellResources( { onSkip: providerDeleteSkipMessage(provider) }, ); } - removeGatewayRegistration(runtime, gatewayLabel); + removeGatewayRegistration(runtime, gatewayLabel, !externallySupervised); return true; } @@ -1276,6 +1298,7 @@ function executePlan( scopedToSelectedGateway: boolean, sharedRegistryMustBePreserved: boolean, sandboxNames: readonly string[], + externallySupervised: boolean, ): { ok: boolean } { let ok = true; const branding = runtimeBranding(runtime); @@ -1303,7 +1326,7 @@ function executePlan( runtime.log("Sibling gateways remain; kept shared helper services and sibling forwards."); } if (!stopHermesForwardWatchers(paths.nemoclawStateDir, runtime)) return { ok: false }; - if (!scopedToSelectedGateway) { + if (!scopedToSelectedGateway && !externallySupervised) { stopHostGatewayProcesses( { run: runtime.run, @@ -1315,6 +1338,8 @@ function executePlan( }, { logNoProcesses: true }, ); + } else if (externallySupervised) { + runtime.log("Kept the externally supervised OpenShell gateway process running."); } stopOllamaAuthProxy(paths, runtime, !scopedToSelectedGateway); stopOpenRouterRuntimeAdapter(paths, runtime, { @@ -1322,10 +1347,18 @@ function executePlan( }); stopModelRouter(paths, runtime, !scopedToSelectedGateway); } else if (step.name === "OpenShell resources") { - if (!removeOpenShellResources(options, runtime, scopedToSelectedGateway, sandboxNames)) { + if ( + !removeOpenShellResources( + options, + runtime, + scopedToSelectedGateway, + sandboxNames, + externallySupervised, + ) + ) { return { ok: false }; } - if (scopedToSelectedGateway) { + if (scopedToSelectedGateway && !externallySupervised) { stopHostGatewayProcesses( { run: runtime.run, @@ -1344,6 +1377,8 @@ function executePlan( stateDir: paths.selectedGatewayLocalStateDir, }, ); + } else if (scopedToSelectedGateway && externallySupervised) { + runtime.log("Kept the externally supervised OpenShell gateway process running."); } } else if (step.name === "NemoClaw CLI") { if (scopedToSelectedGateway) { @@ -1352,7 +1387,11 @@ function executePlan( removeNemoclawCli(paths, runtime); } } else if (step.name === "Docker resources") { - if (dockerIsAvailable(runtime)) { + if (externallySupervised) { + runtime.log( + "Kept Docker containers, images, and volumes used by the externally supervised gateway.", + ); + } else if (dockerIsAvailable(runtime)) { removeDockerContainers( runtime, scopedToSelectedGateway @@ -1377,8 +1416,13 @@ function executePlan( removeManagedSwap(paths, runtime, scopedToSelectedGateway); if (!scopedToSelectedGateway) { for (const pattern of paths.runtimeTempGlobs) removeGlob(pattern, runtime); - if (options.keepOpenShell) runtime.log("Keeping OpenShell binaries as requested."); - else + if (options.keepOpenShell || externallySupervised) { + runtime.log( + externallySupervised + ? "Keeping OpenShell binaries used by the externally supervised gateway." + : "Keeping OpenShell binaries as requested.", + ); + } else for (const target of paths.openshellInstallPaths) removeFileWithOptionalSudo(target, runtime); } else { @@ -1464,6 +1508,22 @@ export function runUninstallPlan( } const resolvedOptions = { ...options, gatewayName: expectedGatewayName }; const { paths, plan } = buildRunPlan(resolvedOptions, { ...deps, env: runtime.env }); + let externallySupervised: boolean; + try { + externallySupervised = isExternallySupervised( + runtime.resolveGatewayTeardownAuthority( + { gatewayName: expectedGatewayName, gatewayPort: GATEWAY_PORT }, + { env: runtime.env }, + ), + ); + } catch (error) { + runtime.error( + `Refusing gateway teardown because lifecycle authority could not be revalidated: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + return { exitCode: 1, plan }; + } const gatewayInspection = inspectOtherGatewayEnvironments(paths, runtime); const { otherGatewayEnvironmentsRemain: scopedToSelectedGateway } = gatewayInspection; let sandboxNames: string[] = []; @@ -1489,6 +1549,7 @@ export function runUninstallPlan( scopedToSelectedGateway, gatewayInspection.sharedRegistryMustBePreserved, sandboxNames, + externallySupervised, ); if (ok) { printBye(runtime); diff --git a/src/lib/onboard/gateway-teardown-authority.test.ts b/src/lib/onboard/gateway-teardown-authority.test.ts new file mode 100644 index 00000000000..5f4a218db0d --- /dev/null +++ b/src/lib/onboard/gateway-teardown-authority.test.ts @@ -0,0 +1,127 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { createSession } from "../state/onboard-session"; +import { bindGatewayAuthorityToCheckpoint } from "./gateway-authority-checkpoint"; +import type { GatewayManagementDeclaration } from "./gateway-management"; +import { type GatewayOwner, resolveGatewayOwner } from "./gateway-ownership"; +import { resolveGatewayTeardownAuthority } from "./gateway-teardown-authority"; + +const target = { gatewayName: "nemoclaw", gatewayPort: 8080 }; + +function declaration( + kind: "systemd-system" | "systemd-user" = "systemd-system", +): GatewayManagementDeclaration { + return { + version: 1, + mode: "externally-supervised", + endpoint: "http://127.0.0.1:8080", + stateDir: "/var/lib/openshell/gateway", + supervisor: { + kind, + serviceName: "openshell-gateway.service", + execPath: "/usr/local/bin/openshell-gateway", + }, + requiredCapabilities: ["gateway.health"], + }; +} + +function owner(currentDeclaration: GatewayManagementDeclaration | null): GatewayOwner { + return resolveGatewayOwner({ + ...target, + declaration: currentDeclaration, + hasPackagedService: false, + }); +} + +function checkpointSession(recordedOwner: GatewayOwner) { + const session = createSession(); + bindGatewayAuthorityToCheckpoint(session, recordedOwner); + return session; +} + +describe("resolveGatewayTeardownAuthority", () => { + it.each([ + "systemd-system", + "systemd-user", + ] as const)("returns the exact recorded %s authority when the declaration still matches (#6576)", (kind) => { + const currentDeclaration = declaration(kind); + const recordedOwner = owner(currentDeclaration); + + expect( + resolveGatewayTeardownAuthority(target, { + hasPackagedService: () => false, + loadDeclaration: () => ({ + ok: true, + declaration: currentDeclaration, + source: "profile", + }), + loadSession: () => checkpointSession(recordedOwner), + }), + ).toEqual(recordedOwner); + }); + + it("uses the current external declaration when no checkpoint exists (#6576)", () => { + const currentDeclaration = declaration(); + + expect( + resolveGatewayTeardownAuthority(target, { + hasPackagedService: () => false, + loadDeclaration: () => ({ + ok: true, + declaration: currentDeclaration, + source: "profile", + }), + loadSession: () => null, + }).mode, + ).toBe("externally-supervised"); + }); + + it("fails closed when a recorded external authority is removed (#6576)", () => { + const recordedOwner = owner(declaration()); + + expect(() => + resolveGatewayTeardownAuthority(target, { + hasPackagedService: () => false, + loadDeclaration: () => ({ ok: true, declaration: null, source: null }), + loadSession: () => checkpointSession(recordedOwner), + }), + ).toThrow(/authority changed since onboarding.*teardown will not perform gateway effects/); + }); + + it("fails closed when the recorded authority targets another gateway (#6576)", () => { + const recordedOwner = resolveGatewayOwner({ + gatewayName: "nemoclaw-8081", + gatewayPort: 8081, + declaration: null, + hasPackagedService: false, + }); + + expect(() => + resolveGatewayTeardownAuthority(target, { + hasPackagedService: () => false, + loadDeclaration: () => ({ ok: true, declaration: null, source: null }), + loadSession: () => checkpointSession(recordedOwner), + }), + ).toThrow(/recorded authority targets 'nemoclaw-8081@8081'/); + }); + + it("rejects a noncanonical gateway target before loading authority (#6576)", () => { + let loaded = false; + + expect(() => + resolveGatewayTeardownAuthority( + { gatewayName: "other", gatewayPort: 8080 }, + { + loadDeclaration: () => { + loaded = true; + return { ok: true, declaration: null, source: null }; + }, + }, + ), + ).toThrow(/noncanonical target/); + expect(loaded).toBe(false); + }); +}); diff --git a/src/lib/onboard/gateway-teardown-authority.ts b/src/lib/onboard/gateway-teardown-authority.ts new file mode 100644 index 00000000000..20a3ad8b793 --- /dev/null +++ b/src/lib/onboard/gateway-teardown-authority.ts @@ -0,0 +1,116 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Exact-target gateway authority resolution for destructive teardown paths. + * + * Onboarding binds authority before gateway effects. Stop, final-sandbox + * cleanup, and uninstall run in separate processes, so they must reload that + * authority before they scan listeners, signal processes, or remove runtime + * resources (#6576). + */ + +import fs from "node:fs"; +import path from "node:path"; + +import { normalizeSession, type Session } from "../state/onboard-session"; +import { nemoclawStateRoot, resolveHome } from "../state/state-root"; +import { hasOpenShellGatewayUserService } from "./docker-driver-gateway-service"; +import { gatewayOwnerFromCheckpoint } from "./gateway-authority-checkpoint"; +import { + loadGatewayManagementDeclaration, + type GatewayManagementLoadResult, +} from "./gateway-management"; +import { + describeGatewayOwnerForError, + type GatewayOwner, + resolveGatewayOwner, + sameGatewayOwner, +} from "./gateway-ownership"; +import { resolveGatewayName } from "./gateway-binding"; + +export interface GatewayTeardownTarget { + gatewayName: string; + gatewayPort: number; +} + +export interface GatewayTeardownAuthorityDeps { + env?: NodeJS.ProcessEnv; + hasPackagedService?: () => boolean; + loadDeclaration?: (env: NodeJS.ProcessEnv) => GatewayManagementLoadResult; + loadSession?: (target: GatewayTeardownTarget, env: NodeJS.ProcessEnv) => Session | null; +} + +export type GatewayTeardownAuthorityResolver = ( + target: GatewayTeardownTarget, + deps?: GatewayTeardownAuthorityDeps, +) => GatewayOwner; + +function loadTargetSession(target: GatewayTeardownTarget, env: NodeJS.ProcessEnv): Session | null { + const sessionFile = path.join( + nemoclawStateRoot(resolveHome(env), target.gatewayPort), + "onboard-session.json", + ); + try { + if (!fs.existsSync(sessionFile)) return null; + return normalizeSession(JSON.parse(fs.readFileSync(sessionFile, "utf-8"))); + } catch { + // Preserve loadSession() compatibility for legacy or interrupted state. + // A valid selected authority still remains binding when it can be read. + return null; + } +} + +/** + * Resolve the current owner and revalidate a selected checkpoint for the exact + * gateway before a teardown effect. A declaration or recorded-owner change is + * an explicit migration, never permission to switch owners during cleanup. + */ +export function resolveGatewayTeardownAuthority( + target: GatewayTeardownTarget, + deps: GatewayTeardownAuthorityDeps = {}, +): GatewayOwner { + if (resolveGatewayName(target.gatewayPort) !== target.gatewayName) { + throw new Error( + `Refusing gateway teardown for noncanonical target '${target.gatewayName}@${String(target.gatewayPort)}'.`, + ); + } + + const env = deps.env ?? process.env; + const loaded = deps.loadDeclaration + ? deps.loadDeclaration(env) + : loadGatewayManagementDeclaration({ env }); + if (!loaded.ok) { + throw new Error(`Invalid gateway management declaration: ${loaded.reason}`); + } + const resolved = resolveGatewayOwner({ + ...target, + declaration: loaded.declaration, + hasPackagedService: deps.hasPackagedService?.() ?? hasOpenShellGatewayUserService(), + }); + + const session = (deps.loadSession ?? loadTargetSession)(target, env); + const recordedDecision = session?.checkpoint?.gatewayAuthority; + if (!recordedDecision || recordedDecision.kind === "unset") return resolved; + if (recordedDecision.kind === "declined") { + throw new Error( + `Refusing gateway teardown for '${target.gatewayName}': the onboarding checkpoint contains an invalid declined gateway authority.`, + ); + } + + const recorded = gatewayOwnerFromCheckpoint(recordedDecision.value); + if (recorded.gatewayName !== target.gatewayName || recorded.gatewayPort !== target.gatewayPort) { + throw new Error( + `Refusing gateway teardown for '${target.gatewayName}@${String(target.gatewayPort)}': ` + + `the recorded authority targets '${recorded.gatewayName}@${String(recorded.gatewayPort)}'.`, + ); + } + if (!sameGatewayOwner(recorded, resolved)) { + throw new Error( + "Gateway lifecycle authority changed since onboarding " + + `(${describeGatewayOwnerForError(recorded)} -> ${describeGatewayOwnerForError(resolved)}). ` + + "Changing authority requires a fresh onboarding run; teardown will not perform gateway effects.", + ); + } + return recorded; +} diff --git a/src/lib/state/onboard-session-gateway-authority.test.ts b/src/lib/state/onboard-session-gateway-authority.test.ts new file mode 100644 index 00000000000..bef637b4763 --- /dev/null +++ b/src/lib/state/onboard-session-gateway-authority.test.ts @@ -0,0 +1,41 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { createSession, summarizeForDebug } from "./onboard-session"; + +describe("onboard session gateway authority", () => { + it("summarizes the checkpointed gateway authority without its state directory (#6576)", () => { + const selected = createSession({ sandboxName: "my-assistant" }); + selected.checkpoint = { + ...selected.checkpoint!, + gatewayAuthority: { + kind: "selected", + value: { + gatewayName: "nemoclaw", + gatewayPort: 8080, + mode: "externally-supervised", + source: "declared", + endpoint: "http://127.0.0.1:8080", + stateDir: "/var/lib/openshell/private-gateway-state", + supervisor: { + kind: "systemd-system", + serviceName: "openshell-gateway.service", + execPath: "/usr/local/bin/openshell-gateway", + }, + requiredCapabilities: ["gateway.health"], + }, + }, + }; + + const summary = summarizeForDebug(selected); + + expect(summary?.gatewayAuthority).toMatchObject({ + mode: "externally-supervised", + source: "declared", + supervisor: { serviceName: "openshell-gateway.service" }, + }); + expect(JSON.stringify(summary?.gatewayAuthority)).not.toContain("private-gateway-state"); + }); +}); diff --git a/src/lib/state/onboard-session.test.ts b/src/lib/state/onboard-session.test.ts index 52747de4dd7..0de6c8cb5f7 100644 --- a/src/lib/state/onboard-session.test.ts +++ b/src/lib/state/onboard-session.test.ts @@ -1439,39 +1439,6 @@ describe("onboard session", () => { expect(summary.resumable).toBe(false); }); - it("summarizes the checkpointed gateway authority without its state directory", () => { - const selected = session.createSession({ sandboxName: "my-assistant" }); - selected.checkpoint = { - ...selected.checkpoint!, - gatewayAuthority: { - kind: "selected", - value: { - gatewayName: "nemoclaw", - gatewayPort: 8080, - mode: "externally-supervised", - source: "declared", - endpoint: "http://127.0.0.1:8080", - stateDir: "/var/lib/openshell/private-gateway-state", - supervisor: { - kind: "systemd-system", - serviceName: "openshell-gateway.service", - execPath: "/usr/local/bin/openshell-gateway", - }, - requiredCapabilities: ["gateway.health"], - }, - }, - }; - - const summary = requireDebugSummary(session.summarizeForDebug(selected)); - - expect(summary.gatewayAuthority).toMatchObject({ - mode: "externally-supervised", - source: "declared", - supervisor: { serviceName: "openshell-gateway.service" }, - }); - expect(JSON.stringify(summary.gatewayAuthority)).not.toContain("private-gateway-state"); - }); - it("keeps debug summaries redacted when failures were sanitized", () => { session.saveSession(session.createSession({ sandboxName: "my-assistant" })); markStepFailedLegacy( diff --git a/src/lib/tunnel/gateway-port-release-lifecycle.test.ts b/src/lib/tunnel/gateway-port-release-lifecycle.test.ts index e2068aabbd7..68231c3a24d 100644 --- a/src/lib/tunnel/gateway-port-release-lifecycle.test.ts +++ b/src/lib/tunnel/gateway-port-release-lifecycle.test.ts @@ -5,6 +5,7 @@ import path from "node:path"; import { describe, expect, it, vi } from "vitest"; import { DEFAULT_GATEWAY_PORT } from "../core/ports"; +import type { GatewaySupervisorKind } from "../onboard/gateway-management"; import type { HostGatewayProcessDeps } from "../onboard/host-gateway-process"; import { releaseManagedGatewayPort } from "./gateway-port-release"; import { @@ -16,6 +17,54 @@ import { } from "./gateway-port-release-test-helpers"; describe("releaseManagedGatewayPort lifecycle (#5968)", () => { + it.each([ + "systemd-system", + "systemd-user", + ] satisfies GatewaySupervisorKind[])("does not scan or signal a %s-supervised gateway during stop (#6576)", (kind) => { + const run = vi.fn(() => ok("123\n")); + const stop = stopSpy(emptyStopResult({ stopped: [123] })); + const log = vi.fn(); + + const result = releaseManagedGatewayPort( + { sandboxName: "alpha" }, + { + ...baseDeps(), + log, + run, + stopHostGatewayProcesses: stop.fn, + getSandbox: () => ({ gatewayPort: DEFAULT_GATEWAY_PORT }), + resolveGatewayTeardownAuthority: ({ gatewayName, gatewayPort }) => ({ + gatewayName, + gatewayPort, + mode: "externally-supervised", + source: "declared", + endpoint: `http://127.0.0.1:${String(gatewayPort)}`, + stateDir: "/var/lib/openshell/gateway", + supervisor: { + kind, + serviceName: "openshell-gateway.service", + execPath: "/usr/local/bin/openshell-gateway", + }, + requiredCapabilities: [], + }), + }, + ); + + expect(result).toEqual({ + port: DEFAULT_GATEWAY_PORT, + released: false, + stopped: [], + remaining: [], + scanned: false, + skipped: true, + }); + expect(run).not.toHaveBeenCalled(); + expect(stop.fn).not.toHaveBeenCalled(); + expect(log).toHaveBeenCalledWith( + expect.stringContaining("Keeping externally supervised OpenShell gateway"), + ); + }); + it("stops lsof-discovered gateways, then reports the port released", () => { const lsof = lsofResponder(ok("111\n222\n"), ok("")); const stop = stopSpy(emptyStopResult({ stopped: [111, 222] })); diff --git a/src/lib/tunnel/gateway-port-release-test-helpers.ts b/src/lib/tunnel/gateway-port-release-test-helpers.ts index 035d49b8ee4..d68c0f62e1e 100644 --- a/src/lib/tunnel/gateway-port-release-test-helpers.ts +++ b/src/lib/tunnel/gateway-port-release-test-helpers.ts @@ -96,5 +96,15 @@ export function baseDeps(): ReleaseGatewayPortDeps { probePortFree: () => true, log: () => {}, warn: () => {}, + resolveGatewayTeardownAuthority: ({ gatewayName, gatewayPort }) => ({ + gatewayName, + gatewayPort, + mode: "nemoclaw-managed", + source: "standalone", + endpoint: null, + stateDir: null, + supervisor: null, + requiredCapabilities: [], + }), }; } diff --git a/src/lib/tunnel/gateway-port-release.ts b/src/lib/tunnel/gateway-port-release.ts index d51fe772a81..aa1c964e0e2 100644 --- a/src/lib/tunnel/gateway-port-release.ts +++ b/src/lib/tunnel/gateway-port-release.ts @@ -5,7 +5,12 @@ import os from "node:os"; -import type { SandboxGatewayBinding } from "../onboard/gateway-binding"; +import { resolveGatewayName, type SandboxGatewayBinding } from "../onboard/gateway-binding"; +import { + resolveGatewayTeardownAuthority, + type GatewayTeardownAuthorityResolver, +} from "../onboard/gateway-teardown-authority"; +import { isExternallySupervised } from "../onboard/gateway-ownership"; import { type HostGatewayProcessDeps, type StopHostGatewayResult, @@ -31,6 +36,7 @@ export interface ReleaseGatewayPortDeps extends Partial now?: () => number; sleep?: (ms: number) => void; stopHostGatewayProcesses?: typeof stopHostGatewayProcesses; + resolveGatewayTeardownAuthority?: GatewayTeardownAuthorityResolver; getSandbox?: (name: string) => SandboxGatewayBinding | null; probePortFree?: (port: number) => boolean; } @@ -70,6 +76,8 @@ export function releaseManagedGatewayPort( depsOverrides.commandExists ?? ((command: string) => defaultGatewayReleaseCommandExists(command, env)); const stop = depsOverrides.stopHostGatewayProcesses ?? stopHostGatewayProcesses; + const resolveAuthority = + depsOverrides.resolveGatewayTeardownAuthority ?? resolveGatewayTeardownAuthority; const getSandbox = depsOverrides.getSandbox ?? getRegisteredSandbox; const probePortFree = depsOverrides.probePortFree ?? defaultProbePortFree; @@ -90,6 +98,23 @@ export function releaseManagedGatewayPort( }; } + const gatewayName = resolveGatewayName(port); + const owner = resolveAuthority({ gatewayName, gatewayPort: port }, { env }); + if (isExternallySupervised(owner)) { + log( + `Keeping externally supervised OpenShell gateway '${gatewayName}' running; ` + + "NemoClaw stopped only the selected sandbox.", + ); + return { + port, + released: false, + stopped: [], + remaining: [], + scanned: false, + skipped: true, + }; + } + const stateDir = resolveGatewayReleaseStateDir(port, env, homeDir); let lsofPids: number[] = []; let scanned = false; diff --git a/test/cli/debug-command.test.ts b/test/cli/debug-command.test.ts index ac15de1db3a..9e2a6991889 100644 --- a/test/cli/debug-command.test.ts +++ b/test/cli/debug-command.test.ts @@ -42,7 +42,7 @@ describe("CLI debug command", () => { ); it( - "debug --quick reports the selected gateway authority without its private state path", + "debug --quick reports the selected gateway authority without its private state path (#6576)", testTimeoutOptions(30_000), ({ resources }) => { const env = createDebugCommandTestEnv(resources, "nemoclaw-cli-debug-authority-"); diff --git a/test/cli/status-root-json.test.ts b/test/cli/status-root-json.test.ts index 6932efa7f6a..be2f429010e 100644 --- a/test/cli/status-root-json.test.ts +++ b/test/cli/status-root-json.test.ts @@ -160,7 +160,7 @@ describe("CLI root status JSON", () => { } }); - it("status text identifies the selected management mode and redacted owner", () => { + it("status text identifies the selected management mode and redacted owner (#6576)", () => { const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-status-authority-")); const localBin = path.join(home, "bin"); fs.mkdirSync(localBin, { recursive: true }); From aff3e3ebae1b05a3e19674ccc10ac4c5a1dcf0f5 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Thu, 23 Jul 2026 23:18:57 -0700 Subject: [PATCH 14/20] test(gateway): keep teardown coverage linear --- .../run-plan-gateway-segregation.test.ts | 38 +++++++++++-------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/src/lib/actions/uninstall/run-plan-gateway-segregation.test.ts b/src/lib/actions/uninstall/run-plan-gateway-segregation.test.ts index 2caaf05907a..2af9ba3dc6f 100644 --- a/src/lib/actions/uninstall/run-plan-gateway-segregation.test.ts +++ b/src/lib/actions/uninstall/run-plan-gateway-segregation.test.ts @@ -31,18 +31,21 @@ describe("uninstall gateway-port segregation (#3053)", () => { try { const stateDir = path.join(tmpHome, ".nemoclaw"); fs.mkdirSync(stateDir, { recursive: true }); - if (scope === "scoped") { - fs.writeFileSync( - path.join(stateDir, "sandboxes.json"), - JSON.stringify({ - defaultSandbox: "alpha", - sandboxes: { - alpha: { name: "alpha", gatewayName: "nemoclaw", gatewayPort: 8080 }, - beta: { name: "beta", gatewayName: "nemoclaw-8091", gatewayPort: 8091 }, - }, - }), - ); - } + const prepareScope = { + full: () => undefined, + scoped: () => + fs.writeFileSync( + path.join(stateDir, "sandboxes.json"), + JSON.stringify({ + defaultSandbox: "alpha", + sandboxes: { + alpha: { name: "alpha", gatewayName: "nemoclaw", gatewayPort: 8080 }, + beta: { name: "beta", gatewayName: "nemoclaw-8091", gatewayPort: 8091 }, + }, + }), + ), + } as const; + prepareScope[scope](); const calls: Array<{ args: string[]; command: string }> = []; const dockerCalls: string[][] = []; const kill = vi.fn(() => true); @@ -104,6 +107,12 @@ describe("uninstall gateway-port segregation (#3053)", () => { it("does not use legacy gateway destroy when external registration removal is unsupported (#6576)", () => { const calls: Array<{ args: string[]; command: string }> = []; + const responses = new Map([ + [ + "openshell gateway remove nemoclaw", + { status: 2, stdout: "", stderr: "unrecognized subcommand 'remove'" }, + ], + ]); const result = runUninstallPlan( { assumeYes: true, deleteModels: false, keepOpenShell: true }, { @@ -128,10 +137,7 @@ describe("uninstall gateway-port segregation (#3053)", () => { rmSync: vi.fn(), run: (command, args) => { calls.push({ args, command }); - if ([command, ...args].join(" ") === "openshell gateway remove nemoclaw") { - return { status: 2, stdout: "", stderr: "unrecognized subcommand 'remove'" }; - } - return ok(); + return responses.get([command, ...args].join(" ")) ?? ok(); }, runDocker: () => ok(), }, From 6236527652279ef2aaaa227b070fe0a0efaaeb0a Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Fri, 24 Jul 2026 00:12:21 -0700 Subject: [PATCH 15/20] docs(gateway): clarify external authority contract --- docs/deployment/gateway-lifecycle-authority.mdx | 15 +++++++++++++-- docs/reference/commands.mdx | 3 ++- src/lib/onboard/gateway-host-runtime.ts | 2 +- src/lib/onboard/gateway-management.ts | 4 ++-- 4 files changed, 18 insertions(+), 6 deletions(-) diff --git a/docs/deployment/gateway-lifecycle-authority.mdx b/docs/deployment/gateway-lifecycle-authority.mdx index 47416aa758a..89b6d862f2f 100644 --- a/docs/deployment/gateway-lifecycle-authority.mdx +++ b/docs/deployment/gateway-lifecycle-authority.mdx @@ -68,11 +68,15 @@ The declaration uses the following fields. |---|---| | `version` | Contract version. Only `1` is supported. | | `mode` | `nemoclaw-managed` or `externally-supervised`. | -| `endpoint` | Exact bare origin for an externally supervised gateway. Use `http` or `https`, a numeric IPv4 or IPv6 loopback address, and the gateway port that this NemoClaw process uses. Credentials, DNS names such as `localhost`, paths, query strings, and fragments are rejected. | -| `stateDir` | Absolute external gateway state directory. For HTTPS, it contains the client trust bundle. | +| `endpoint` | Exact bare origin for an externally supervised gateway. Use `http` or `https`, host `127.0.0.1` or `[::1]`, and the gateway port that this NemoClaw process uses. Credentials, DNS names such as `localhost`, paths, query strings, and fragments are rejected. | +| `stateDir` | Absolute external gateway state directory. Keep it outside NemoClaw-owned state paths such as `~/.local/state/nemoclaw`; uninstall removes those owned paths and does not otherwise target the declared external directory. For HTTPS, it contains the client trust bundle. | | `supervisor` | Required for `externally-supervised` and rejected for `nemoclaw-managed`. `kind` is `systemd-system` or `systemd-user`. `serviceName` names one `.service` unit, and `execPath` is the absolute gateway executable path. | | `requiredCapabilities` | Capabilities onboarding needs. Supported values are `gateway.health`, `sandbox.create`, `sandbox.exec`, and `gpu.passthrough`. | +`requiredCapabilities` is checked against the capabilities implemented by this NemoClaw build. +OpenShell does not expose capability discovery, so this field does not probe or attest capabilities on the running gateway. +The listener, supervisor, identity, and health checks below validate the live gateway separately. + The endpoint port must match `NEMOCLAW_GATEWAY_PORT`. When the endpoint omits a port, `http` resolves to port `80` and `https` resolves to port `443` for this comparison. Onboarding rejects a mismatch before making a health request or inspecting the host listener because validating one endpoint and operating a different gateway would break the authority boundary. @@ -182,10 +186,17 @@ It does not transfer process ownership to NemoClaw. ## Fix a failure Resolve an external gateway failure through the declared supervisor. +For `systemd-system`, inspect the declared service with: ```bash systemctl status openshell-gateway.service ``` +For `systemd-user`, use the user manager: + +```bash +systemctl --user status openshell-gateway.service +``` + Bring the supervised gateway up, leave exactly one verified process holding the configured port, and rerun `nemoclaw onboard`. To hand the lifecycle back to NemoClaw, remove the platform gateway service and declare `nemoclaw-managed`, or remove the declaration. diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 544b21a64f3..eae5e02b602 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -3164,7 +3164,8 @@ For Hermes setups, uninstall inspects the selected gateway's managed port-forwar If any watcher or forward cleanup cannot be confirmed, uninstall exits nonzero and preserves the selected gateway's watcher state so you can retry cleanup. On Linux, uninstall removes `~/.local/state/nemoclaw`, which contains NemoClaw-owned Docker-driver gateway SQLite data, audit logs, VM-driver state, and standalone-fallback gateway PID files. -This directory is separate from a declared external gateway state directory, which uninstall does not remove. +Keep a declared external gateway state directory outside that NemoClaw-owned path. +Uninstall does not otherwise target the declared external directory. | Flag | Effect | |---|---| diff --git a/src/lib/onboard/gateway-host-runtime.ts b/src/lib/onboard/gateway-host-runtime.ts index 8ffdc3a68e5..ddb5c799fb1 100644 --- a/src/lib/onboard/gateway-host-runtime.ts +++ b/src/lib/onboard/gateway-host-runtime.ts @@ -91,7 +91,7 @@ export interface GatewayHostRuntime { ): void; attachGateway(owner: GatewayOwner, expectedProbe: GatewayAttachmentProbe): Promise; bindGatewayOwner(owner: GatewayOwner): void; - /** HTTPS endpoint of the gateway this process operates. */ + /** Local endpoint of the gateway this process operates. */ getGatewayLocalEndpoint(): string; getGatewayOwner(): GatewayOwner; resetGatewayOwnerBinding(): void; diff --git a/src/lib/onboard/gateway-management.ts b/src/lib/onboard/gateway-management.ts index 5400fbf54f5..703cc266656 100644 --- a/src/lib/onboard/gateway-management.ts +++ b/src/lib/onboard/gateway-management.ts @@ -40,7 +40,6 @@ export const GATEWAY_MANAGEMENT_ENV_VAR = "NEMOCLAW_GATEWAY_MANAGEMENT"; export type GatewayManagementMode = "nemoclaw-managed" | "externally-supervised"; -/** How the external supervisor runs the gateway, and how to recognize it. */ /** * Supervisor kinds NemoClaw can authoritatively bind a listening PID to. Only * systemd is supported in v1: an opaque "external" supervisor offers no way to @@ -52,6 +51,7 @@ export const SUPPORTED_GATEWAY_SUPERVISOR_KINDS = ["systemd-system", "systemd-us export type GatewaySupervisorKind = (typeof SUPPORTED_GATEWAY_SUPERVISOR_KINDS)[number]; +/** How the external supervisor runs the gateway, and how to recognize it. */ export interface GatewaySupervisorDeclaration { kind: GatewaySupervisorKind; /** Unit (or equivalent) name the platform supervisor manages. */ @@ -65,7 +65,7 @@ export interface GatewayManagementDeclaration { mode: GatewayManagementMode; /** Required only when an external supervisor owns the gateway. */ endpoint: string | null; - /** External gateway state root containing the client TLS bundle. */ + /** External gateway state root; for HTTPS, contains the client TLS bundle. */ stateDir: string | null; /** Required for `externally-supervised`; must be absent for `nemoclaw-managed`. */ supervisor: GatewaySupervisorDeclaration | null; From f757878a7c8f17d2990476aeb538da9d1a54c5fd Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Fri, 24 Jul 2026 00:58:01 -0700 Subject: [PATCH 16/20] fix(credentials): revalidate gateway authority --- src/commands/credentials.test.ts | 40 +++++++++++++- src/commands/credentials/reset.ts | 8 ++- src/lib/actions/credentials-add.ts | 7 ++- src/lib/credentials/command-support.ts | 30 +++++++++++ .../gateway-teardown-authority.test.ts | 19 ++++++- src/lib/onboard/gateway-teardown-authority.ts | 52 +++++++++++++------ 6 files changed, 133 insertions(+), 23 deletions(-) diff --git a/src/commands/credentials.test.ts b/src/commands/credentials.test.ts index ff45d0e3836..6113c36df2b 100644 --- a/src/commands/credentials.test.ts +++ b/src/commands/credentials.test.ts @@ -5,10 +5,11 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; const mocks = vi.hoisted(() => ({ prompt: vi.fn().mockResolvedValue("yes"), - recoverNamedGatewayRuntime: vi.fn().mockResolvedValue({ recovered: true }), + recoverNamedGatewayRuntime: vi.fn().mockResolvedValue({ recovered: true, attempted: false }), runOpenshellProviderCommand: vi.fn(), recordExtraProvider: vi.fn(), forgetExtraProvider: vi.fn(), + resolveGatewayCredentialMutationAuthority: vi.fn(), })); vi.mock("../lib/credentials/store", () => ({ @@ -21,7 +22,11 @@ vi.mock("../lib/actions/global", () => ({ recordExtraProvider: mocks.recordExtraProvider, forgetExtraProvider: mocks.forgetExtraProvider, })); +vi.mock("../lib/onboard/gateway-teardown-authority", () => ({ + resolveGatewayCredentialMutationAuthority: mocks.resolveGatewayCredentialMutationAuthority, +})); +import { runCredentialsAddAction } from "../lib/actions/credentials-add"; import CredentialsCommand from "./credentials"; import CredentialsListCommand from "./credentials/list"; import CredentialsResetCommand from "./credentials/reset"; @@ -31,8 +36,10 @@ const rootDir = process.cwd(); describe("credentials oclif adapter source coverage", () => { beforeEach(() => { vi.clearAllMocks(); - mocks.recoverNamedGatewayRuntime.mockResolvedValue({ recovered: true }); + mocks.recoverNamedGatewayRuntime.mockResolvedValue({ recovered: true, attempted: false }); mocks.runOpenshellProviderCommand.mockReturnValue({ status: 0, stdout: "nvidia-prod\n" }); + mocks.resolveGatewayCredentialMutationAuthority.mockReturnValue({}); + process.exitCode = undefined; }); it("prints top-level credentials usage", async () => { @@ -56,6 +63,7 @@ describe("credentials oclif adapter source coverage", () => { await CredentialsListCommand.run([], rootDir); expect(mocks.recoverNamedGatewayRuntime).toHaveBeenCalledWith(); + expect(mocks.resolveGatewayCredentialMutationAuthority).not.toHaveBeenCalled(); expect(mocks.runOpenshellProviderCommand).toHaveBeenCalledWith( ["provider", "list", "--names"], { @@ -90,4 +98,32 @@ describe("credentials oclif adapter source coverage", () => { log.mockRestore(); expect(output).toContain("Removed provider 'nvidia-prod'"); }); + + it("rejects add and reset before provider mutation when the gateway is healthy but authority changed since onboarding (#6576)", async () => { + mocks.resolveGatewayCredentialMutationAuthority.mockImplementation(() => { + throw new Error( + "Gateway lifecycle authority changed since onboarding; provider credential mutation will not perform gateway effects.", + ); + }); + const error = vi.spyOn(console, "error").mockImplementation(() => {}); + + const add = await runCredentialsAddAction({ + provider: "custom-provider", + type: "custom", + credentials: [], + configPairs: [], + fromExisting: true, + }); + await CredentialsResetCommand.run(["nvidia-prod", "--yes"], rootDir); + + expect(add.exitCode).toBe(1); + expect(add.failureLines.join("\n")).toContain( + "gateway lifecycle authority could not be revalidated", + ); + expect(mocks.resolveGatewayCredentialMutationAuthority).toHaveBeenCalledTimes(2); + expect(mocks.runOpenshellProviderCommand).not.toHaveBeenCalled(); + expect(error.mock.calls.flat().join("\n")).toContain( + "gateway lifecycle authority could not be revalidated", + ); + }); }); diff --git a/src/commands/credentials/reset.ts b/src/commands/credentials/reset.ts index 73fe5489a55..9fb6fc10319 100644 --- a/src/commands/credentials/reset.ts +++ b/src/commands/credentials/reset.ts @@ -7,7 +7,10 @@ import { OPENSHELL_OPERATION_TIMEOUT_MS } from "../../lib/adapters/openshell/tim import { CLI_NAME } from "../../lib/cli/branding"; import { yesFlag } from "../../lib/cli/common-flags"; import { NemoClawCommand } from "../../lib/cli/nemoclaw-oclif-command"; -import { isBridgeProviderName, recoverGatewayOrExit } from "../../lib/credentials/command-support"; +import { + isBridgeProviderName, + recoverGatewayForCredentialMutationOrExit, +} from "../../lib/credentials/command-support"; import { prompt as askPrompt, KNOWN_CREDENTIAL_ENV_KEYS } from "../../lib/credentials/store"; import { deleteProviderWithRecovery, @@ -65,7 +68,8 @@ export default class CredentialsResetCommand extends NemoClawCommand { } } - if (!(await recoverGatewayOrExit("reach", (lines) => this.failWithLines(lines)))) return; + if (!(await recoverGatewayForCredentialMutationOrExit((lines) => this.failWithLines(lines)))) + return; // `provider delete` trips on FailedPrecondition when the provider is still // attached to a sandbox (e.g. `-brave-search` after onboard). The diff --git a/src/lib/actions/credentials-add.ts b/src/lib/actions/credentials-add.ts index dffb95ad74c..9c45c702f68 100644 --- a/src/lib/actions/credentials-add.ts +++ b/src/lib/actions/credentials-add.ts @@ -6,7 +6,10 @@ import path from "node:path"; import { OPENSHELL_OPERATION_TIMEOUT_MS } from "../adapters/openshell/timeouts"; import { CLI_NAME } from "../cli/branding"; -import { isBridgeProviderName, recoverGatewayOrExit } from "../credentials/command-support"; +import { + isBridgeProviderName, + recoverGatewayForCredentialMutationOrExit, +} from "../credentials/command-support"; import { redact } from "../security/redact"; import { SECRET_PATTERNS } from "../security/secret-patterns"; import { ROOT } from "../state/paths"; @@ -157,7 +160,7 @@ export async function runCredentialsAddAction( } const recoveryFailureLines: string[] = []; - const recovered = await recoverGatewayOrExit("reach", (lines) => { + const recovered = await recoverGatewayForCredentialMutationOrExit((lines) => { recoveryFailureLines.push(...lines); }); if (!recovered) { diff --git a/src/lib/credentials/command-support.ts b/src/lib/credentials/command-support.ts index a20ed2eca3f..afbaf52d41b 100644 --- a/src/lib/credentials/command-support.ts +++ b/src/lib/credentials/command-support.ts @@ -3,6 +3,9 @@ import { recoverNamedGatewayRuntime } from "../actions/global"; import { CLI_DISPLAY_NAME, CLI_NAME } from "../cli/branding"; +import { GATEWAY_PORT } from "../core/ports"; +import { resolveGatewayName } from "../onboard/gateway-binding"; +import { resolveGatewayCredentialMutationAuthority } from "../onboard/gateway-teardown-authority"; export { isBridgeProviderName } from "./provider-list"; @@ -30,6 +33,15 @@ export function credentialsGatewayRecoveryFailureLines(kind: "query" | "reach"): ]; } +export function credentialsGatewayAuthorityFailureLines(error: unknown): string[] { + const detail = error instanceof Error ? error.message : String(error); + return [ + " Refusing to change provider credentials because the gateway lifecycle authority could not be revalidated.", + ` ${detail}`, + ` Run '${CLI_NAME} onboard' to bind the current gateway authority before retrying.`, + ]; +} + export async function recoverGatewayOrExit( kind: "query" | "reach", reportFailure: (lines: readonly string[]) => void = (lines) => @@ -41,3 +53,21 @@ export async function recoverGatewayOrExit( reportFailure(credentialsGatewayRecoveryFailureLines(kind)); return false; } + +export async function recoverGatewayForCredentialMutationOrExit( + reportFailure: (lines: readonly string[]) => void = (lines) => + lines.forEach((line) => console.error(line)), +): Promise { + if (!(await recoverGatewayOrExit("reach", reportFailure))) return false; + + try { + resolveGatewayCredentialMutationAuthority({ + gatewayName: resolveGatewayName(GATEWAY_PORT), + gatewayPort: GATEWAY_PORT, + }); + return true; + } catch (error) { + reportFailure(credentialsGatewayAuthorityFailureLines(error)); + return false; + } +} diff --git a/src/lib/onboard/gateway-teardown-authority.test.ts b/src/lib/onboard/gateway-teardown-authority.test.ts index 5f4a218db0d..8118fc10f79 100644 --- a/src/lib/onboard/gateway-teardown-authority.test.ts +++ b/src/lib/onboard/gateway-teardown-authority.test.ts @@ -7,7 +7,10 @@ import { createSession } from "../state/onboard-session"; import { bindGatewayAuthorityToCheckpoint } from "./gateway-authority-checkpoint"; import type { GatewayManagementDeclaration } from "./gateway-management"; import { type GatewayOwner, resolveGatewayOwner } from "./gateway-ownership"; -import { resolveGatewayTeardownAuthority } from "./gateway-teardown-authority"; +import { + resolveGatewayCredentialMutationAuthority, + resolveGatewayTeardownAuthority, +} from "./gateway-teardown-authority"; const target = { gatewayName: "nemoclaw", gatewayPort: 8080 }; @@ -91,6 +94,20 @@ describe("resolveGatewayTeardownAuthority", () => { ).toThrow(/authority changed since onboarding.*teardown will not perform gateway effects/); }); + it("fails closed before credential mutation when authority changed since onboarding (#6576)", () => { + const recordedOwner = owner(declaration()); + + expect(() => + resolveGatewayCredentialMutationAuthority(target, { + hasPackagedService: () => false, + loadDeclaration: () => ({ ok: true, declaration: null, source: null }), + loadSession: () => checkpointSession(recordedOwner), + }), + ).toThrow( + /authority changed since onboarding.*provider credential mutation will not perform gateway effects/, + ); + }); + it("fails closed when the recorded authority targets another gateway (#6576)", () => { const recordedOwner = resolveGatewayOwner({ gatewayName: "nemoclaw-8081", diff --git a/src/lib/onboard/gateway-teardown-authority.ts b/src/lib/onboard/gateway-teardown-authority.ts index 20a3ad8b793..4e9dce61eb9 100644 --- a/src/lib/onboard/gateway-teardown-authority.ts +++ b/src/lib/onboard/gateway-teardown-authority.ts @@ -2,12 +2,12 @@ // SPDX-License-Identifier: Apache-2.0 /** - * Exact-target gateway authority resolution for destructive teardown paths. + * Exact-target gateway authority resolution for teardown and provider credential mutations. * - * Onboarding binds authority before gateway effects. Stop, final-sandbox - * cleanup, and uninstall run in separate processes, so they must reload that - * authority before they scan listeners, signal processes, or remove runtime - * resources (#6576). + * Onboarding binds authority before gateway effects. Credentials add and reset, + * stop, final-sandbox cleanup, and uninstall can run after onboarding exits. + * They must reload that authority before they mutate providers, scan listeners, + * signal processes, or remove runtime resources (#6576). */ import fs from "node:fs"; @@ -17,9 +17,10 @@ import { normalizeSession, type Session } from "../state/onboard-session"; import { nemoclawStateRoot, resolveHome } from "../state/state-root"; import { hasOpenShellGatewayUserService } from "./docker-driver-gateway-service"; import { gatewayOwnerFromCheckpoint } from "./gateway-authority-checkpoint"; +import { resolveGatewayName } from "./gateway-binding"; import { - loadGatewayManagementDeclaration, type GatewayManagementLoadResult, + loadGatewayManagementDeclaration, } from "./gateway-management"; import { describeGatewayOwnerForError, @@ -27,7 +28,6 @@ import { resolveGatewayOwner, sameGatewayOwner, } from "./gateway-ownership"; -import { resolveGatewayName } from "./gateway-binding"; export interface GatewayTeardownTarget { gatewayName: string; @@ -46,6 +46,8 @@ export type GatewayTeardownAuthorityResolver = ( deps?: GatewayTeardownAuthorityDeps, ) => GatewayOwner; +type GatewayAuthorityEffect = "credential mutation" | "teardown"; + function loadTargetSession(target: GatewayTeardownTarget, env: NodeJS.ProcessEnv): Session | null { const sessionFile = path.join( nemoclawStateRoot(resolveHome(env), target.gatewayPort), @@ -62,17 +64,20 @@ function loadTargetSession(target: GatewayTeardownTarget, env: NodeJS.ProcessEnv } /** - * Resolve the current owner and revalidate a selected checkpoint for the exact - * gateway before a teardown effect. A declaration or recorded-owner change is - * an explicit migration, never permission to switch owners during cleanup. + * Resolve the current owner and revalidate checkpointed authority for the exact + * gateway before teardown or provider credential mutation. A declaration or + * recorded-owner change is an explicit migration. It does not permit the + * operation to use another owner. */ -export function resolveGatewayTeardownAuthority( +function resolveGatewayEffectAuthority( target: GatewayTeardownTarget, - deps: GatewayTeardownAuthorityDeps = {}, + effect: GatewayAuthorityEffect, + deps: GatewayTeardownAuthorityDeps, ): GatewayOwner { + const operation = effect === "teardown" ? "gateway teardown" : "provider credential mutation"; if (resolveGatewayName(target.gatewayPort) !== target.gatewayName) { throw new Error( - `Refusing gateway teardown for noncanonical target '${target.gatewayName}@${String(target.gatewayPort)}'.`, + `Refusing ${operation} for noncanonical target '${target.gatewayName}@${String(target.gatewayPort)}'.`, ); } @@ -94,14 +99,14 @@ export function resolveGatewayTeardownAuthority( if (!recordedDecision || recordedDecision.kind === "unset") return resolved; if (recordedDecision.kind === "declined") { throw new Error( - `Refusing gateway teardown for '${target.gatewayName}': the onboarding checkpoint contains an invalid declined gateway authority.`, + `Refusing ${operation} for '${target.gatewayName}': the onboarding checkpoint contains an invalid declined gateway authority.`, ); } const recorded = gatewayOwnerFromCheckpoint(recordedDecision.value); if (recorded.gatewayName !== target.gatewayName || recorded.gatewayPort !== target.gatewayPort) { throw new Error( - `Refusing gateway teardown for '${target.gatewayName}@${String(target.gatewayPort)}': ` + + `Refusing ${operation} for '${target.gatewayName}@${String(target.gatewayPort)}': ` + `the recorded authority targets '${recorded.gatewayName}@${String(recorded.gatewayPort)}'.`, ); } @@ -109,8 +114,23 @@ export function resolveGatewayTeardownAuthority( throw new Error( "Gateway lifecycle authority changed since onboarding " + `(${describeGatewayOwnerForError(recorded)} -> ${describeGatewayOwnerForError(resolved)}). ` + - "Changing authority requires a fresh onboarding run; teardown will not perform gateway effects.", + `Changing authority requires a fresh onboarding run; ${operation} will not perform gateway effects.`, ); } return recorded; } + +export function resolveGatewayTeardownAuthority( + target: GatewayTeardownTarget, + deps: GatewayTeardownAuthorityDeps = {}, +): GatewayOwner { + return resolveGatewayEffectAuthority(target, "teardown", deps); +} + +/** Revalidate the exact checkpointed authority before a provider credential mutation. */ +export function resolveGatewayCredentialMutationAuthority( + target: GatewayTeardownTarget, + deps: GatewayTeardownAuthorityDeps = {}, +): GatewayOwner { + return resolveGatewayEffectAuthority(target, "credential mutation", deps); +} From 5c11a275b86d398bc10cd45ae5b6570d61ed9f0b Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Fri, 24 Jul 2026 01:47:40 -0700 Subject: [PATCH 17/20] ci: retry E2E after runner loss Signed-off-by: Prekshi Vyas From 659882219b03c98f6bc04dd0d4cb05d2791c0d23 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Fri, 24 Jul 2026 02:03:38 -0700 Subject: [PATCH 18/20] test(onboard): isolate GPU recreate DNS probe Signed-off-by: Prekshi Vyas --- src/lib/onboard/docker-gpu-patch-recreate.test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/lib/onboard/docker-gpu-patch-recreate.test.ts b/src/lib/onboard/docker-gpu-patch-recreate.test.ts index ee15aeba57d..bd4f3889618 100644 --- a/src/lib/onboard/docker-gpu-patch-recreate.test.ts +++ b/src/lib/onboard/docker-gpu-patch-recreate.test.ts @@ -36,6 +36,7 @@ describe("Docker GPU recreate orchestration", () => { runOpenshell, sleep: vi.fn(), now: () => new Date("2026-05-12T00:00:00Z"), + detectSandboxFallbackDns: vi.fn(() => null), readDir: vi.fn(() => null), readFile: vi.fn(() => null), }, @@ -110,6 +111,7 @@ describe("Docker GPU recreate orchestration", () => { runOpenshell, sleep: vi.fn(), now: () => new Date("2026-05-12T00:00:00Z"), + detectSandboxFallbackDns: vi.fn(() => null), }, ); From 1bd90ae6725e27b9a4fb5ef4ce093987f237fa01 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Fri, 24 Jul 2026 03:30:53 -0700 Subject: [PATCH 19/20] ci: retry E2E after runner hang Signed-off-by: Prekshi Vyas From bcbbef70c7646a4bc5ec41f1051e9eb9d73c73c1 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Fri, 24 Jul 2026 04:39:54 -0700 Subject: [PATCH 20/20] ci: retry E2E after canceled resume run Signed-off-by: Prekshi Vyas