diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 56755ce447e..c13de1db7d5 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -119,6 +119,12 @@ const DIM = USE_COLOR ? "\x1b[2m" : ""; const RESET = USE_COLOR ? "\x1b[0m" : ""; let OPENSHELL_BIN = null; const GATEWAY_NAME = "nemoclaw"; +const GATEWAY_BOOTSTRAP_SECRET_NAMES = [ + "openshell-server-tls", + "openshell-server-client-ca", + "openshell-client-tls", + "openshell-ssh-handshake", +]; const BACK_TO_SELECTION = "__NEMOCLAW_BACK_TO_SELECTION__"; /** @@ -2227,6 +2233,182 @@ function destroyGateway() { ); } +function getGatewayClusterContainerState() { + const containerName = getGatewayClusterContainerName(); + const state = runCapture( + `docker inspect --type container --format '{{.State.Status}}{{if .State.Health}} {{.State.Health.Status}}{{end}}' ${shellQuote(containerName)} 2>/dev/null`, + { ignoreError: true }, + ) + .trim() + .toLowerCase(); + return state || "missing"; +} + +function getGatewayHealthWaitConfig(_startStatus = 0, containerState = "") { + const isArm64 = process.arch === "arm64"; + const standardCount = envInt("NEMOCLAW_HEALTH_POLL_COUNT", isArm64 ? 30 : 12); + const standardInterval = envInt("NEMOCLAW_HEALTH_POLL_INTERVAL", isArm64 ? 10 : 5); + const extendedCount = envInt("NEMOCLAW_GATEWAY_START_POLL_COUNT", standardCount); + const extendedInterval = envInt("NEMOCLAW_GATEWAY_START_POLL_INTERVAL", standardInterval); + const normalizedState = String(containerState || "") + .trim() + .toLowerCase(); + const normalizedContainerState = normalizedState || "missing"; + const useExtendedWait = normalizedContainerState !== "missing"; + + return { + count: useExtendedWait ? extendedCount : standardCount, + interval: useExtendedWait ? extendedInterval : standardInterval, + extended: useExtendedWait, + containerState: normalizedContainerState, + }; +} + +function getGatewayClusterContainerName() { + return `openshell-cluster-${GATEWAY_NAME}`; +} + +function getGatewayLocalEndpoint() { + return `https://127.0.0.1:${GATEWAY_PORT}`; +} + +function getGatewayBootstrapRepairPlan(missingSecrets = []) { + const allowed = new Set(GATEWAY_BOOTSTRAP_SECRET_NAMES); + const normalized = [...new Set((missingSecrets || []).map((name) => String(name).trim()).filter(Boolean))] + .filter((name) => allowed.has(name)); + const missing = new Set(normalized); + const needsClientBundle = + missing.has("openshell-server-client-ca") || missing.has("openshell-client-tls"); + + return { + missingSecrets: normalized, + needsRepair: normalized.length > 0, + needsServerTls: missing.has("openshell-server-tls"), + needsClientBundle, + needsHandshake: missing.has("openshell-ssh-handshake"), + }; +} + +function buildGatewayBootstrapSecretsScript(missingSecrets = []) { + const plan = getGatewayBootstrapRepairPlan(missingSecrets); + if (!plan.needsRepair) return "exit 0"; + + return ` +set -eu +export KUBECONFIG=/etc/rancher/k3s/k3s.yaml +kubectl get namespace openshell >/dev/null 2>&1 +kubectl -n openshell get statefulset/openshell >/dev/null 2>&1 +TMPDIR="$(mktemp -d)" +cleanup() { + rm -rf "$TMPDIR" +} +trap cleanup EXIT +if ${plan.needsServerTls ? "true" : "false"}; then + cat >"$TMPDIR/server-ext.cnf" <<'EOF' +subjectAltName=DNS:openshell,DNS:openshell.openshell,DNS:openshell.openshell.svc,DNS:openshell.openshell.svc.cluster.local,DNS:localhost,IP:127.0.0.1 +extendedKeyUsage=serverAuth +EOF + openssl req -nodes -newkey rsa:2048 -keyout "$TMPDIR/server.key" -out "$TMPDIR/server.csr" -subj "/CN=openshell.openshell.svc.cluster.local" >/dev/null 2>&1 + openssl x509 -req -in "$TMPDIR/server.csr" -signkey "$TMPDIR/server.key" -out "$TMPDIR/server.crt" -days 3650 -sha256 -extfile "$TMPDIR/server-ext.cnf" >/dev/null 2>&1 + kubectl create secret tls -n openshell openshell-server-tls --cert="$TMPDIR/server.crt" --key="$TMPDIR/server.key" --dry-run=client -o yaml | kubectl apply -f - +fi +if ${plan.needsClientBundle ? "true" : "false"}; then + cat >"$TMPDIR/client-ext.cnf" <<'EOF' +extendedKeyUsage=clientAuth +EOF + openssl req -x509 -nodes -newkey rsa:2048 -keyout "$TMPDIR/client-ca.key" -out "$TMPDIR/client-ca.crt" -subj "/CN=openshell-client-ca" -days 3650 >/dev/null 2>&1 + openssl req -nodes -newkey rsa:2048 -keyout "$TMPDIR/client.key" -out "$TMPDIR/client.csr" -subj "/CN=openshell-client" >/dev/null 2>&1 + openssl x509 -req -in "$TMPDIR/client.csr" -CA "$TMPDIR/client-ca.crt" -CAkey "$TMPDIR/client-ca.key" -CAcreateserial -out "$TMPDIR/client.crt" -days 3650 -sha256 -extfile "$TMPDIR/client-ext.cnf" >/dev/null 2>&1 + kubectl create secret generic -n openshell openshell-server-client-ca --from-file=ca.crt="$TMPDIR/client-ca.crt" --dry-run=client -o yaml | kubectl apply -f - + kubectl create secret generic -n openshell openshell-client-tls --from-file=tls.crt="$TMPDIR/client.crt" --from-file=tls.key="$TMPDIR/client.key" --from-file=ca.crt="$TMPDIR/client-ca.crt" --dry-run=client -o yaml | kubectl apply -f - +fi +if ${plan.needsHandshake ? "true" : "false"}; then + kubectl create secret generic -n openshell openshell-ssh-handshake --from-literal=secret="$(openssl rand -hex 32)" --dry-run=client -o yaml | kubectl apply -f - +fi +`; +} + +function runGatewayClusterCapture(script, opts = {}) { + const containerName = getGatewayClusterContainerName(); + return runCapture( + `docker exec ${shellQuote(containerName)} sh -lc ${shellQuote(script)}`, + opts, + ); +} + +function runGatewayCluster(script, opts = {}) { + const containerName = getGatewayClusterContainerName(); + return run( + `docker exec ${shellQuote(containerName)} sh -lc ${shellQuote(script)}`, + opts, + ); +} + +function listMissingGatewayBootstrapSecrets() { + const output = runGatewayClusterCapture( + ` +set -eu +export KUBECONFIG=/etc/rancher/k3s/k3s.yaml +kubectl get namespace openshell >/dev/null 2>&1 || exit 0 +kubectl -n openshell get statefulset/openshell >/dev/null 2>&1 || exit 0 +for name in ${GATEWAY_BOOTSTRAP_SECRET_NAMES.map((name) => shellQuote(name)).join(" ")}; do + kubectl -n openshell get secret "$name" >/dev/null 2>&1 || printf '%s\\n' "$name" +done +`, + { ignoreError: true }, + ); + return output + .split("\n") + .map((line) => line.trim()) + .filter(Boolean); +} + +function gatewayClusterHealthcheckPassed() { + const result = runGatewayCluster("/usr/local/bin/cluster-healthcheck.sh", { + ignoreError: true, + suppressOutput: true, + }); + return result.status === 0; +} + +function repairGatewayBootstrapSecrets() { + const missingSecrets = listMissingGatewayBootstrapSecrets(); + const plan = getGatewayBootstrapRepairPlan(missingSecrets); + if (!plan.needsRepair) return { repaired: false, missingSecrets }; + + console.log( + ` OpenShell bootstrap secrets missing: ${plan.missingSecrets.join(", ")}. Repairing...`, + ); + const repairResult = runGatewayCluster(buildGatewayBootstrapSecretsScript(plan.missingSecrets), { + ignoreError: true, + suppressOutput: true, + }); + const remainingSecrets = listMissingGatewayBootstrapSecrets(); + if (repairResult.status === 0 && remainingSecrets.length === 0) { + console.log(" ✓ OpenShell bootstrap secrets created"); + return { repaired: true, missingSecrets: remainingSecrets }; + } + return { repaired: false, missingSecrets: remainingSecrets }; +} + +function attachGatewayMetadataIfNeeded({ forceRefresh = false } = {}) { + const gwInfo = runCaptureOpenshell(["gateway", "info", "-g", GATEWAY_NAME], { ignoreError: true }); + // runCaptureOpenshell may return stale-but-present gateway metadata. When + // hasStaleGateway(gwInfo) is truthy we skip runOpenshell unless a repair + // flow explicitly forces a refresh after recreating bootstrap secrets. + if (!forceRefresh && hasStaleGateway(gwInfo)) return true; + + const addResult = runOpenshell( + ["gateway", "add", "--local", "--name", GATEWAY_NAME, getGatewayLocalEndpoint()], + { ignoreError: true, suppressOutput: true }, + ); + if (addResult.status === 0) { + console.log(" ✓ Gateway metadata reattached"); + return true; + } + return false; +} + async function ensureNamedCredential(envName, label, helpUrl = null) { let key = getCredential(envName); if (key) { @@ -2723,17 +2905,28 @@ async function startGatewayWithOptions(_gpu, { exitOnFailure = true } = {}) { } } console.log(" Waiting for gateway health..."); + const healthWait = getGatewayHealthWaitConfig( + startResult.status, + getGatewayClusterContainerState(), + ); + if (healthWait.extended) { + console.log( + ` Gateway container is still ${healthWait.containerState}; allowing up to ${ + healthWait.count * healthWait.interval + }s for first-time startup.`, + ); + } - // ARM64 (e.g. Raspberry Pi) needs more time: k3s takes 90-180s to init - const isArm64 = process.arch === "arm64"; - // After openshell gateway start returns (container HEALTHY at Layer 1), - // poll application-layer connectivity (Layer 2: gRPC, TLS, port mapping). - // 60s default gives enough buffer for gRPC init and TLS handshake. (#1830) - const healthPollCount = envInt("NEMOCLAW_HEALTH_POLL_COUNT", isArm64 ? 30 : 12); - const healthPollInterval = envInt("NEMOCLAW_HEALTH_POLL_INTERVAL", isArm64 ? 10 : 5); + const healthPollCount = healthWait.count; + const healthPollInterval = healthWait.interval; for (let i = 0; i < healthPollCount; i++) { - // Ensure the gateway is selected before each probe (non-TTY environments - // like ARM64 may not have it selected automatically) + const repairResult = repairGatewayBootstrapSecrets(); + if (repairResult.repaired) { + attachGatewayMetadataIfNeeded({ forceRefresh: true }); + } else if (gatewayClusterHealthcheckPassed()) { + attachGatewayMetadataIfNeeded(); + } + // Ensure the gateway remains selected before each probe. runCaptureOpenshell(["gateway", "select", GATEWAY_NAME], { ignoreError: true }); const status = runCaptureOpenshell(["status"], { ignoreError: true }); const namedInfo = runCaptureOpenshell(["gateway", "info", "-g", GATEWAY_NAME], { @@ -2857,9 +3050,23 @@ async function recoverGatewayRuntime() { } runOpenshell(["gateway", "select", GATEWAY_NAME], { ignoreError: true }); - const recoveryPollCount = envInt("NEMOCLAW_HEALTH_POLL_COUNT", 10); - const recoveryPollInterval = envInt("NEMOCLAW_HEALTH_POLL_INTERVAL", 2); + const recoveryWait = getGatewayHealthWaitConfig( + startResult.status, + getGatewayClusterContainerState(), + ); + const recoveryPollCount = recoveryWait.extended + ? recoveryWait.count + : envInt("NEMOCLAW_HEALTH_POLL_COUNT", 10); + const recoveryPollInterval = recoveryWait.extended + ? recoveryWait.interval + : envInt("NEMOCLAW_HEALTH_POLL_INTERVAL", 2); for (let i = 0; i < recoveryPollCount; i++) { + const repairResult = repairGatewayBootstrapSecrets(); + if (repairResult.repaired) { + attachGatewayMetadataIfNeeded({ forceRefresh: true }); + } else if (gatewayClusterHealthcheckPassed()) { + attachGatewayMetadataIfNeeded(); + } status = runCaptureOpenshell(["status"], { ignoreError: true }); if (status.includes("Connected") && isSelectedGateway(status)) { process.env.OPENSHELL_GATEWAY = GATEWAY_NAME; @@ -6324,6 +6531,7 @@ async function onboard(opts = {}) { module.exports = { buildProviderArgs, + buildGatewayBootstrapSecretsScript, buildSandboxConfigSyncScript, compactText, copyBuildContextDir, @@ -6333,7 +6541,11 @@ module.exports = { ensureValidatedBraveSearchCredential, formatEnvAssignment, getFutureShellPathHint, + getGatewayBootstrapRepairPlan, + getGatewayLocalEndpoint, getGatewayStartEnv, + getGatewayClusterContainerState, + getGatewayHealthWaitConfig, getGatewayReuseState, getNavigationChoice, getSandboxInferenceConfig, diff --git a/test/gateway-start-wait.test.ts b/test/gateway-start-wait.test.ts new file mode 100644 index 00000000000..74fae8b46e4 --- /dev/null +++ b/test/gateway-start-wait.test.ts @@ -0,0 +1,171 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, describe, expect, it } from "vitest"; +import { createRequire } from "node:module"; + +const require = createRequire(import.meta.url); +const ORIGINAL_ENV = { ...process.env }; +const ONBOARD_MODULE = require.resolve("../dist/lib/onboard.js"); +const PORTS_MODULE = require.resolve("../dist/lib/ports.js"); + +function loadOnboard() { + delete require.cache[ONBOARD_MODULE]; + delete require.cache[PORTS_MODULE]; + return require("../dist/lib/onboard"); +} + +afterEach(() => { + process.env = { ...ORIGINAL_ENV }; + delete require.cache[ONBOARD_MODULE]; + delete require.cache[PORTS_MODULE]; +}); + +describe("gateway startup wait config", () => { + it("extends the health wait when gateway start exits non-zero but the container is still starting", () => { + const { getGatewayHealthWaitConfig } = loadOnboard(); + process.env.NEMOCLAW_HEALTH_POLL_COUNT = "5"; + process.env.NEMOCLAW_HEALTH_POLL_INTERVAL = "2"; + process.env.NEMOCLAW_GATEWAY_START_POLL_COUNT = "60"; + process.env.NEMOCLAW_GATEWAY_START_POLL_INTERVAL = "5"; + + expect(getGatewayHealthWaitConfig(1, "starting")).toEqual({ + count: 60, + interval: 5, + extended: true, + containerState: "starting", + }); + }); + + it("treats a running container without a health state as a slow-start case", () => { + const { getGatewayHealthWaitConfig } = loadOnboard(); + process.env.NEMOCLAW_GATEWAY_START_POLL_COUNT = "12"; + process.env.NEMOCLAW_GATEWAY_START_POLL_INTERVAL = "4"; + + expect(getGatewayHealthWaitConfig(1, "running")).toEqual({ + count: 12, + interval: 4, + extended: true, + containerState: "running", + }); + }); + + it("extends the wait for other live container states such as created or unhealthy", () => { + const { getGatewayHealthWaitConfig } = loadOnboard(); + process.env.NEMOCLAW_GATEWAY_START_POLL_COUNT = "9"; + process.env.NEMOCLAW_GATEWAY_START_POLL_INTERVAL = "6"; + + expect(getGatewayHealthWaitConfig(1, "created")).toEqual({ + count: 9, + interval: 6, + extended: true, + containerState: "created", + }); + expect(getGatewayHealthWaitConfig(1, "running unhealthy")).toEqual({ + count: 9, + interval: 6, + extended: true, + containerState: "running unhealthy", + }); + }); + + it("still uses the extended wait when start exits non-zero before container metadata appears", () => { + const { getGatewayHealthWaitConfig } = loadOnboard(); + process.env.NEMOCLAW_GATEWAY_START_POLL_COUNT = "7"; + process.env.NEMOCLAW_GATEWAY_START_POLL_INTERVAL = "3"; + process.env.NEMOCLAW_HEALTH_POLL_COUNT = "4"; + process.env.NEMOCLAW_HEALTH_POLL_INTERVAL = "1"; + + expect(getGatewayHealthWaitConfig(1, "missing")).toEqual({ + count: 4, + interval: 1, + extended: false, + containerState: "missing", + }); + }); + + it("uses the short wait for missing containers regardless of start exit code", () => { + const { getGatewayHealthWaitConfig } = loadOnboard(); + process.env.NEMOCLAW_HEALTH_POLL_COUNT = "7"; + process.env.NEMOCLAW_HEALTH_POLL_INTERVAL = "3"; + + expect(getGatewayHealthWaitConfig(0, "missing")).toEqual({ + count: 7, + interval: 3, + extended: false, + containerState: "missing", + }); + }); + + it("extends the wait when the container is still live even if gateway start exited zero", () => { + const { getGatewayHealthWaitConfig } = loadOnboard(); + process.env.NEMOCLAW_GATEWAY_START_POLL_COUNT = "8"; + process.env.NEMOCLAW_GATEWAY_START_POLL_INTERVAL = "6"; + + expect(getGatewayHealthWaitConfig(0, "running")).toEqual({ + count: 8, + interval: 6, + extended: true, + containerState: "running", + }); + }); +}); + +describe("gateway bootstrap secret repair", () => { + it("uses the configured gateway port for local metadata reattachment", () => { + process.env.NEMOCLAW_GATEWAY_PORT = "9443"; + const { getGatewayLocalEndpoint } = loadOnboard(); + + expect(getGatewayLocalEndpoint()).toBe("https://127.0.0.1:9443"); + }); + + it("repairs the client CA and client TLS secrets together", () => { + const { getGatewayBootstrapRepairPlan } = loadOnboard(); + expect( + getGatewayBootstrapRepairPlan(["openshell-client-tls"]), + ).toEqual({ + missingSecrets: ["openshell-client-tls"], + needsRepair: true, + needsServerTls: false, + needsClientBundle: true, + needsHandshake: false, + }); + }); + + it("ignores unknown secret names when planning repairs", () => { + const { getGatewayBootstrapRepairPlan } = loadOnboard(); + + expect( + getGatewayBootstrapRepairPlan(["openshell-client-tls", "noise", " openshell-server-tls ", ""]), + ).toEqual({ + missingSecrets: ["openshell-client-tls", "openshell-server-tls"], + needsRepair: true, + needsServerTls: true, + needsClientBundle: true, + needsHandshake: false, + }); + }); + + it("emits a script that creates all missing bootstrap secrets", () => { + const { buildGatewayBootstrapSecretsScript } = loadOnboard(); + const script = buildGatewayBootstrapSecretsScript([ + "openshell-server-tls", + "openshell-server-client-ca", + "openshell-client-tls", + "openshell-ssh-handshake", + ]); + + expect(script).toContain("openshell-server-tls"); + expect(script).toContain("openshell-server-client-ca"); + expect(script).toContain("openshell-client-tls"); + expect(script).toContain("openshell-ssh-handshake"); + expect(script).toContain('CN=openshell-client-ca'); + expect(script).toContain('CN=openshell-client'); + expect(script).toContain("subjectAltName=DNS:openshell"); + }); + + it("skips secret generation when nothing is missing", () => { + const { buildGatewayBootstrapSecretsScript } = loadOnboard(); + expect(buildGatewayBootstrapSecretsScript([]).trim()).toBe("exit 0"); + }); +});