diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 3db19bbd12e..5dbb26affc0 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -22,6 +22,10 @@ function envInt(name, fallback) { /** Inference timeout (seconds) for local providers (Ollama, vLLM, NIM). */ const LOCAL_INFERENCE_TIMEOUT_SECS = envInt("NEMOCLAW_LOCAL_INFERENCE_TIMEOUT", 180); + +/** Strip ANSI escape sequences before printing process output to the terminal. + * Covers CSI (color, erase, cursor), OSC, and C1 two-byte escapes per ECMA-48. */ +const ANSI_RE = /\x1B(?:\[[0-?]*[ -/]*[@-~]|\][^\x07]*(?:\x07|\x1B\\)|[@-_])/g; const { ROOT, SCRIPTS, redact, run, runCapture, shellQuote } = require("./runner"); const { stageOptimizedSandboxBuildContext } = require("./sandbox-build-context"); const { DASHBOARD_PORT, GATEWAY_PORT, VLLM_PORT, OLLAMA_PORT } = require("./ports"); @@ -683,8 +687,8 @@ function upsertProvider(name, type, credentialEnv, baseUrl, env = {}) { const result = runOpenshell(args, runOpts); if (result.status !== 0) { const output = - compactText(`${result.stderr || ""}`) || - compactText(`${result.stdout || ""}`) || + compactText(redact(`${result.stderr || ""}`)) || + compactText(redact(`${result.stdout || ""}`)) || `Failed to ${action} provider '${name}'.`; return { ok: false, status: result.status || 1, message: output }; } @@ -2187,9 +2191,13 @@ async function startGatewayWithOptions(_gpu, { exitOnFailure = true } = {}) { }, ); if (startResult.status !== 0) { - const output = compactText(String(startResult.output || "")); - if (output) { - console.log(` Gateway start returned before healthy: ${output.slice(0, 240)}`); + const lines = String(redact(startResult.output || "")) + .split("\n") + .map((l) => compactText(l.replace(ANSI_RE, ""))) + .filter(Boolean) + .map((l) => ` ${l}`); + if (lines.length > 0) { + console.log(` Gateway start returned before healthy:\n${lines.join("\n")}`); } } console.log(" Waiting for gateway health..."); @@ -2229,6 +2237,25 @@ async function startGatewayWithOptions(_gpu, { exitOnFailure = true } = {}) { console.error(` Gateway failed to start after ${retries + 1} attempts.`); console.error(" Gateway state preserved for diagnostics."); console.error(""); + try { + const logs = redact( + runCaptureOpenshell(["doctor", "logs", "--name", GATEWAY_NAME], { + ignoreError: true, + }), + ); + if (logs) { + console.error(" Gateway logs:"); + for (const line of String(logs) + .split("\n") + .map((l) => l.replace(/\r/g, "").replace(ANSI_RE, "")) + .filter(Boolean)) { + console.error(` ${line}`); + } + console.error(""); + } + } catch { + // doctor logs unavailable — fall through to manual instructions + } console.error(" Troubleshooting:"); console.error(" openshell doctor logs --name nemoclaw"); console.error(" openshell doctor check"); @@ -3532,7 +3559,7 @@ async function setupInference( break; } const message = - compactText(`${applyResult.stderr || ""} ${applyResult.stdout || ""}`) || + compactText(redact(`${applyResult.stderr || ""} ${applyResult.stdout || ""}`)) || `Failed to configure inference provider '${provider}'.`; console.error(` ${message}`); if (isNonInteractive()) { @@ -5355,6 +5382,7 @@ module.exports = { repairRecordedSandbox, recoverGatewayRuntime, resolveDashboardForwardTarget, + startGateway, buildAuthenticatedDashboardUrl, getDashboardAccessInfo, getDashboardForwardPort, diff --git a/test/onboard.test.ts b/test/onboard.test.ts index 726e610535e..c407a5fd881 100644 --- a/test/onboard.test.ts +++ b/test/onboard.test.ts @@ -937,6 +937,125 @@ describe("onboard helpers", () => { expect(getGatewayReuseState("", "")).toBe("missing"); }); + it("prints doctor logs automatically when gateway fails to start (#1605)", () => { + const repoRoot = path.join(import.meta.dirname, ".."); + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-gateway-diag-")); + const fakeBin = path.join(tmpDir, "bin"); + const scriptPath = path.join(tmpDir, "gateway-diag.cjs"); + const onboardPath = JSON.stringify(path.join(repoRoot, "dist", "lib", "onboard.js")); + + fs.mkdirSync(fakeBin, { recursive: true }); + // Fake openshell: + // gateway start — emits ANSI color codes + \r\n (mirrors real gateway output), exits 1 + // doctor logs — emits ANSI sequences, an OOMKilled message, and a fake nvapi- credential + // to exercise ANSI stripping and redaction in the doctor-log path + fs.writeFileSync( + path.join(fakeBin, "openshell"), + `#!/usr/bin/env bash +if [[ "$*" == *"doctor"*"logs"* ]]; then + printf "\\033[31mERROR\\033[0m k3s cluster crashed: OOMKilled\\r\\n" + printf " Container nemoclaw_k3s ran out of memory\\r\\n" + printf " Gateway auth token: nvapi-fakecredential-9999\\r\\n" + exit 0 +fi +if [[ "$*" == *"gateway"*"start"* ]]; then + printf "\\033[33mDeploying\\033[0m gateway nemoclaw...\\r\\n" + printf "\\r\\nWaiting for gateway health...\\r\\n" + exit 1 +fi +exit 1 +`, + { mode: 0o755 }, + ); + + // Script runs in a child process: patching p-retry to be immediate avoids the + // 10 s + 30 s minTimeout delays, and NEMOCLAW_HEALTH_POLL_COUNT=0 skips the + // health-poll loop so the function throws "Gateway failed to start" on the + // first attempt. With exitOnFailure:true the catch block should auto-print + // doctor logs to stderr and then call process.exit(1). + const script = ` +const mod = require("module"); +const origLoad = mod._load; +mod._load = function(req, parent, isMain) { + if (req === "p-retry") { + return async (fn, opts) => { + try { + return await fn({ attemptNumber: 1, retriesLeft: 0 }); + } catch (e) { + if (opts && opts.onFailedAttempt) { + opts.onFailedAttempt(Object.assign(e, { attemptNumber: 1, retriesLeft: 0 })); + } + throw e; + } + }; + } + return origLoad.call(this, req, parent, isMain); +}; +const { startGateway } = require(${onboardPath}); +startGateway(null).catch(() => {}); +`; + fs.writeFileSync(scriptPath, script); + + const nodeExec = process.execPath; + const result = spawnSync(nodeExec, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: { + ...process.env, + HOME: tmpDir, + PATH: `${fakeBin}:${process.env.PATH || ""}`, + NEMOCLAW_HEALTH_POLL_COUNT: "0", + NEMOCLAW_NON_INTERACTIVE: "1", + }, + }); + + // The process exits 1 because startGateway calls process.exit(1) on failure. + assert.equal(result.status, 1, `unexpected exit code; stderr:\n${result.stderr}`); + + // Fix 3: doctor logs are auto-printed to stderr. + assert.ok( + result.stderr.includes("Gateway logs:"), + `expected "Gateway logs:" header in stderr:\n${result.stderr}`, + ); + assert.ok( + result.stderr.includes("OOMKilled"), + `expected doctor log output in stderr:\n${result.stderr}`, + ); + + // ANSI sequences must be stripped from both stdout (gateway start output) and + // stderr (doctor logs). A raw \x1b in the output means the regex failed. + assert.ok( + !result.stdout.includes("\x1b"), + `unexpected ANSI escape in stdout:\n${result.stdout}`, + ); + assert.ok( + !result.stderr.includes("\x1b"), + `unexpected ANSI escape in stderr:\n${result.stderr}`, + ); + + // Credentials in doctor logs must be redacted, never printed verbatim. + assert.ok( + !result.stderr.includes("nvapi-fakecredential-9999"), + `credential leaked verbatim in stderr:\n${result.stderr}`, + ); + + // Fix 2: the \r\n -> \naiting rendering artifact must not appear. + assert.ok( + !result.stdout.includes("\naiting"), + `\\naiting artifact present in stdout:\n${result.stdout}`, + ); + + // Fix 1: gateway start output is printed per-line under the header, not as + // one collapsed blob. "Deploying" and "Waiting" must appear on separate lines. + const gatewayLines = result.stdout + .split("\n") + .filter((l) => l.includes("Deploying") || l.includes("Waiting")); + assert.ok( + gatewayLines.length >= 2, + `expected "Deploying" and "Waiting" on separate lines in stdout:\n${result.stdout}`, + ); + }); + it("classifies sandbox reuse states from openshell outputs", () => { expect( getSandboxStateFromOutputs(