diff --git a/biome.json b/biome.json index efa7abf98ac..f76afcad86e 100644 --- a/biome.json +++ b/biome.json @@ -143,6 +143,26 @@ } } }, + { + "includes": [ + "src/lib/actions/sandbox/doctor.ts", + "src/lib/actions/sandbox/doctor-messaging.ts", + "src/lib/actions/sandbox/doctor-report.ts", + "src/lib/actions/sandbox/doctor-system-checks.ts" + ], + "linter": { + "rules": { + "complexity": { + "noExcessiveCognitiveComplexity": { + "level": "error", + "options": { + "maxAllowedComplexity": 10 + } + } + } + } + } + }, { "includes": ["nemoclaw/src/**/*.ts"], "linter": { diff --git a/src/lib/actions/sandbox/doctor-flow.test.ts b/src/lib/actions/sandbox/doctor-flow.test.ts index 2e9cbbb42da..86f20420a23 100644 --- a/src/lib/actions/sandbox/doctor-flow.test.ts +++ b/src/lib/actions/sandbox/doctor-flow.test.ts @@ -4,6 +4,7 @@ import { createRequire } from "node:module"; import { afterEach, beforeEach, describe, expect, it, type MockInstance, vi } from "vitest"; +import { testTimeoutOptions } from "../../../../test/helpers/timeouts"; type RunSandboxDoctor = typeof import("./doctor")["runSandboxDoctor"]; @@ -11,10 +12,21 @@ const requireDist = createRequire(import.meta.url); const doctorModulePath = "./doctor.js"; function createDoctorHarness(): { + buildToolScopeChecksSpy: MockInstance; + captureOpenShellSpy: MockInstance; captureHostCommandSpy: MockInstance; + configuredMessagingChannelsSpy: MockInstance; + executeSandboxCommandForVerificationSpy: MockInstance; getSandboxSpy: MockInstance; + getNamedGatewayLifecycleStateSpy: MockInstance; + healthProbeSpy: MockInstance; + inspectMutableConfigPermsSpy: MockInstance; + loadAgentSpy: MockInstance; + probeSandboxInferenceGatewayHealthSpy: MockInstance; logSpy: MockInstance; + recoverNamedGatewayRuntimeSpy: MockInstance; repairMutableConfigPermsSpy: MockInstance; + resolveOpenShellSpy: MockInstance; runSandboxDoctor: RunSandboxDoctor; } { delete require.cache[requireDist.resolve(doctorModulePath)]; @@ -50,27 +62,43 @@ function createDoctorHarness(): { gatewayPort: 19080, messaging: undefined, }); - vi.spyOn(registry, "getConfiguredMessagingChannelsFromEntry").mockReturnValue([]); + const configuredMessagingChannelsSpy = vi + .spyOn(registry, "getConfiguredMessagingChannelsFromEntry") + .mockReturnValue([]); vi.spyOn(registry, "getDisabledMessagingChannelsFromEntry").mockReturnValue([]); - vi.spyOn(resolve, "resolveOpenshell").mockReturnValue("/usr/bin/openshell"); + const resolveOpenShellSpy = vi + .spyOn(resolve, "resolveOpenshell") + .mockReturnValue("/usr/bin/openshell"); vi.spyOn(gatewayBinding, "resolveSandboxGatewayName").mockReturnValue("nemoclaw-19080"); vi.spyOn(gatewayBinding, "resolveGatewayName").mockReturnValue("nemoclaw-19080"); vi.spyOn(dockerDriverPlatform, "isLinuxDockerDriverGatewayEnabled").mockReturnValue(true); - vi.spyOn(gatewayRuntime, "recoverNamedGatewayRuntime").mockResolvedValue({ - before: { state: "healthy_named", status: "Status: Connected", gatewayInfo: "" }, - after: { state: "healthy_named", status: "Status: Connected", gatewayInfo: "" }, - recovered: false, - }); - vi.spyOn(runtime, "captureOpenshell").mockImplementation((args: unknown) => { - const argv = Array.isArray(args) ? args : []; - if (argv[0] === "sandbox" && argv[1] === "list") { - return { status: 0, output: "alpha Ready" }; - } - if (argv[0] === "inference" && argv[1] === "get") { - return { status: 0, output: "Provider: ollama-local\nModel: live-model\n" }; - } - return { status: 0, output: "" }; - }); + const recoverNamedGatewayRuntimeSpy = vi + .spyOn(gatewayRuntime, "recoverNamedGatewayRuntime") + .mockResolvedValue({ + before: { state: "healthy_named", status: "Status: Connected", gatewayInfo: "" }, + after: { state: "healthy_named", status: "Status: Connected", gatewayInfo: "" }, + recovered: false, + }); + const getNamedGatewayLifecycleStateSpy = vi + .spyOn(gatewayRuntime, "getNamedGatewayLifecycleState") + .mockReturnValue({ + state: "healthy_named", + status: "Status: Connected", + gatewayInfo: "Gateway: nemoclaw-19080", + activeGateway: "nemoclaw-19080", + }); + const captureOpenShellSpy = vi + .spyOn(runtime, "captureOpenshell") + .mockImplementation((args: unknown) => { + const argv = Array.isArray(args) ? args : []; + if (argv[0] === "sandbox" && argv[1] === "list") { + return { status: 0, output: "alpha Ready" }; + } + if (argv[0] === "inference" && argv[1] === "get") { + return { status: 0, output: "Provider: ollama-local\nModel: live-model\n" }; + } + return { status: 0, output: "" }; + }); const captureHostCommandSpy = vi .spyOn(doctorHostCommand, "captureHostCommand") .mockImplementation((command: unknown) => { @@ -80,19 +108,21 @@ function createDoctorHarness(): { } return { status: 0, stdout: "", stderr: "" }; }); - vi.spyOn(health, "probeProviderHealth").mockReturnValue({ + const healthProbeSpy = vi.spyOn(health, "probeProviderHealth").mockReturnValue({ ok: true, probed: true, providerLabel: "Ollama", endpoint: "http://127.0.0.1:11434/v1/chat/completions", detail: "healthy", }); - vi.spyOn(processRecovery, "probeSandboxInferenceGatewayHealth").mockResolvedValue({ - ok: false, - endpoint: "http://127.0.0.1:19000/v1/chat/completions", - detail: "gateway refused connection", - }); - vi.spyOn(agentDefs, "loadAgent").mockReturnValue({ + const probeSandboxInferenceGatewayHealthSpy = vi + .spyOn(processRecovery, "probeSandboxInferenceGatewayHealth") + .mockResolvedValue({ + ok: false, + endpoint: "http://127.0.0.1:19000/v1/chat/completions", + detail: "gateway refused connection", + }); + const loadAgentSpy = vi.spyOn(agentDefs, "loadAgent").mockReturnValue({ name: "openclaw", configPaths: { dir: "/sandbox/.openclaw", configFile: "openclaw.json", format: "json" }, }); @@ -107,17 +137,19 @@ function createDoctorHarness(): { mode: "temporarily_unlocked", detail: "temporarily unlocked for maintenance", }); - vi.spyOn(shields, "inspectMutableConfigPerms").mockReturnValue({ - applies: true, - ok: true, - dirMode: "2770", - dirOwner: "sandbox:sandbox", - fileMode: "660", - fileOwner: "sandbox:sandbox", - configDir: "/sandbox/.openclaw", - configFile: "openclaw.json", - issues: [], - }); + const inspectMutableConfigPermsSpy = vi + .spyOn(shields, "inspectMutableConfigPerms") + .mockReturnValue({ + applies: true, + ok: true, + dirMode: "2770", + dirOwner: "sandbox:sandbox", + fileMode: "660", + fileOwner: "sandbox:sandbox", + configDir: "/sandbox/.openclaw", + configFile: "openclaw.json", + issues: [], + }); const repairMutableConfigPermsSpy = vi .spyOn(shields, "repairMutableConfigPerms") .mockReturnValue({ @@ -127,27 +159,42 @@ function createDoctorHarness(): { }); vi.spyOn(statusCommandDeps, "buildStatusCommandDeps").mockReturnValue({}); vi.spyOn(tunnelServices, "readCloudflaredState").mockReturnValue({ kind: "running", pid: 1234 }); - vi.spyOn(sandboxVerificationExec, "executeSandboxCommandForVerification").mockReturnValue({ - status: 0, - stdout: "ok", - stderr: "", - }); - vi.spyOn(doctorToolScope, "buildToolScopeChecks").mockReturnValue([ - { - group: "Sandbox", - label: "Tool scope approvals", - status: "ok", - detail: "no pending approvals", - }, - ]); + const executeSandboxCommandForVerificationSpy = vi + .spyOn(sandboxVerificationExec, "executeSandboxCommandForVerification") + .mockReturnValue({ + status: 0, + stdout: "ok", + stderr: "", + }); + const buildToolScopeChecksSpy = vi + .spyOn(doctorToolScope, "buildToolScopeChecks") + .mockReturnValue([ + { + group: "Sandbox", + label: "Tool scope approvals", + status: "ok", + detail: "no pending approvals", + }, + ]); logSpy.mockClear(); return { + buildToolScopeChecksSpy, + captureOpenShellSpy, captureHostCommandSpy, + configuredMessagingChannelsSpy, + executeSandboxCommandForVerificationSpy, getSandboxSpy, + getNamedGatewayLifecycleStateSpy, + healthProbeSpy, + inspectMutableConfigPermsSpy, + loadAgentSpy, + probeSandboxInferenceGatewayHealthSpy, logSpy, + recoverNamedGatewayRuntimeSpy, repairMutableConfigPermsSpy, + resolveOpenShellSpy, runSandboxDoctor: requireDist(doctorModulePath).runSandboxDoctor, }; } @@ -166,50 +213,242 @@ describe("runSandboxDoctor flow", () => { delete require.cache[requireDist.resolve(doctorModulePath)]; }); - it("builds a JSON report with host, gateway, sandbox, inference, messaging, and local-service checks", async () => { + it( + "builds a JSON report with host, gateway, sandbox, inference, messaging, and local-service checks", + testTimeoutOptions(30_000), + async () => { + const harness = createDoctorHarness(); + + const report = await harness.runSandboxDoctor("alpha", ["--json"], { quietJson: true }); + + expect(report).toMatchObject({ + schemaVersion: 1, + sandbox: "alpha", + status: "fail", + }); + expect(report?.checks).toEqual( + expect.arrayContaining([ + expect.objectContaining({ group: "Host", label: "Docker daemon", status: "ok" }), + expect.objectContaining({ group: "Gateway", label: "OpenShell status", status: "ok" }), + expect.objectContaining({ group: "Sandbox", label: "Live sandbox", status: "ok" }), + expect.objectContaining({ group: "Inference", label: "Provider health", status: "ok" }), + expect.objectContaining({ + group: "Inference", + label: "Provider health (gateway)", + status: "fail", + }), + expect.objectContaining({ group: "Messaging", label: "Channels", status: "info" }), + expect.objectContaining({ group: "Local services", label: "Ollama", status: "ok" }), + expect.objectContaining({ + group: "Local services", + label: "cloudflared", + status: "ok", + }), + ]), + ); + expect(exitSpy).not.toHaveBeenCalled(); + expect(harness.logSpy).not.toHaveBeenCalled(); + }, + ); + + it("rejects mutating --fix when JSON output was requested", async () => { + const harness = createDoctorHarness(); + + await expect(harness.runSandboxDoctor("alpha", ["--json", "--fix"])).rejects.toThrow( + "process.exit(1)", + ); + + expect(exitSpy).toHaveBeenCalledWith(1); + expect(harness.getSandboxSpy).not.toHaveBeenCalled(); + expect(harness.captureHostCommandSpy).not.toHaveBeenCalled(); + expect(harness.repairMutableConfigPermsSpy).not.toHaveBeenCalled(); + }); + + it("does not run live or tool-scope probes when OpenShell is unavailable", async () => { const harness = createDoctorHarness(); + harness.resolveOpenShellSpy.mockReturnValue(null); - const report = await harness.runSandboxDoctor("alpha", ["--json"], { quietJson: true }); + await harness.runSandboxDoctor("alpha", ["--json"], { quietJson: true }); - expect(report).toMatchObject({ - schemaVersion: 1, - sandbox: "alpha", - status: "fail", + expect(harness.recoverNamedGatewayRuntimeSpy).not.toHaveBeenCalled(); + expect(harness.captureOpenShellSpy).not.toHaveBeenCalled(); + expect(harness.buildToolScopeChecksSpy).not.toHaveBeenCalled(); + expect(harness.probeSandboxInferenceGatewayHealthSpy).not.toHaveBeenCalled(); + }); + + it("does not run live or tool-scope probes when the named gateway is disconnected", async () => { + const harness = createDoctorHarness(); + harness.configuredMessagingChannelsSpy.mockReturnValue(["telegram"]); + harness.getNamedGatewayLifecycleStateSpy.mockReturnValue({ + state: "missing_named", + status: "Status: Disconnected", + gatewayInfo: "", + activeGateway: null, }); + + const report = await harness.runSandboxDoctor("alpha", ["--json"], { quietJson: true }); + + expect(harness.captureOpenShellSpy).not.toHaveBeenCalled(); + expect(harness.buildToolScopeChecksSpy).not.toHaveBeenCalled(); + expect(harness.probeSandboxInferenceGatewayHealthSpy).not.toHaveBeenCalled(); + expect(harness.executeSandboxCommandForVerificationSpy).not.toHaveBeenCalled(); expect(report?.checks).toEqual( expect.arrayContaining([ - expect.objectContaining({ group: "Host", label: "Docker daemon", status: "ok" }), - expect.objectContaining({ group: "Gateway", label: "OpenShell status", status: "ok" }), - expect.objectContaining({ group: "Sandbox", label: "Live sandbox", status: "ok" }), - expect.objectContaining({ group: "Inference", label: "Provider health", status: "ok" }), expect.objectContaining({ group: "Inference", label: "Provider health (gateway)", - status: "fail", + status: "info", + detail: "skipped because the sandbox is not reachable through its named gateway", }), - expect.objectContaining({ group: "Messaging", label: "Channels", status: "info" }), - expect.objectContaining({ group: "Local services", label: "Ollama", status: "ok" }), expect.objectContaining({ - group: "Local services", - label: "cloudflared", - status: "ok", + group: "Messaging", + label: "Runtime channel registry", + status: "info", + detail: "skipped because the sandbox is not reachable through its named gateway", }), ]), ); - expect(exitSpy).not.toHaveBeenCalled(); - expect(harness.logSpy).not.toHaveBeenCalled(); }); - it("rejects mutating --fix when JSON output was requested", async () => { + it("keeps JSON gateway diagnostics read-only", async () => { const harness = createDoctorHarness(); - await expect(harness.runSandboxDoctor("alpha", ["--json", "--fix"])).rejects.toThrow( - "process.exit(1)", + await harness.runSandboxDoctor("alpha", ["--json"], { quietJson: true }); + + expect(harness.getNamedGatewayLifecycleStateSpy).toHaveBeenCalledWith("nemoclaw-19080"); + expect(harness.recoverNamedGatewayRuntimeSpy).not.toHaveBeenCalled(); + }); + + it("runs live probes only after plain doctor recovers the named gateway", async () => { + const harness = createDoctorHarness(); + harness.configuredMessagingChannelsSpy.mockReturnValue(["telegram"]); + harness.recoverNamedGatewayRuntimeSpy.mockResolvedValue({ + before: { + state: "missing_named", + status: "Status: Disconnected", + gatewayInfo: "", + }, + after: { + state: "healthy_named", + status: "Status: Connected", + gatewayInfo: "Gateway: nemoclaw-19080", + }, + recovered: true, + }); + harness.probeSandboxInferenceGatewayHealthSpy.mockResolvedValue({ + ok: true, + endpoint: "http://127.0.0.1:19000/v1/chat/completions", + detail: "healthy", + }); + + await harness.runSandboxDoctor("alpha"); + + expect(harness.recoverNamedGatewayRuntimeSpy).toHaveBeenCalledWith({ + gatewayName: "nemoclaw-19080", + }); + expect(harness.captureOpenShellSpy).toHaveBeenCalledWith( + ["sandbox", "list"], + expect.any(Object), + ); + expect(harness.probeSandboxInferenceGatewayHealthSpy).toHaveBeenCalledWith("alpha"); + expect(harness.executeSandboxCommandForVerificationSpy).toHaveBeenCalled(); + expect(harness.buildToolScopeChecksSpy).toHaveBeenCalledWith( + "alpha", + "nemoclaw", + false, + expect.any(Object), ); + expect(harness.recoverNamedGatewayRuntimeSpy.mock.invocationCallOrder[0]).toBeLessThan( + harness.captureOpenShellSpy.mock.invocationCallOrder[0], + ); + }); + + it("does not enable repairs for plain or JSON diagnostics", async () => { + const harness = createDoctorHarness(); + harness.inspectMutableConfigPermsSpy.mockReturnValue({ + applies: true, + ok: false, + dirMode: "700", + dirOwner: "sandbox:sandbox", + fileMode: "600", + fileOwner: "sandbox:sandbox", + configDir: "/sandbox/.openclaw", + configFile: "openclaw.json", + issues: ["directory mode is 700"], + }); + const processRecovery = requireDist("./process-recovery.js"); + vi.mocked(processRecovery.probeSandboxInferenceGatewayHealth).mockResolvedValue({ + ok: true, + endpoint: "http://127.0.0.1:19000/v1/chat/completions", + detail: "healthy", + }); + + await harness.runSandboxDoctor("alpha"); + await harness.runSandboxDoctor("alpha", ["--json"], { quietJson: true }); - expect(exitSpy).toHaveBeenCalledWith(1); - expect(harness.getSandboxSpy).not.toHaveBeenCalled(); - expect(harness.captureHostCommandSpy).not.toHaveBeenCalled(); expect(harness.repairMutableConfigPermsSpy).not.toHaveBeenCalled(); + expect(harness.buildToolScopeChecksSpy).toHaveBeenCalledTimes(2); + expect(harness.buildToolScopeChecksSpy.mock.calls.map((call) => call[2])).toEqual([ + false, + false, + ]); + }); + + it("skips OpenClaw tool-scope checks for other agents", async () => { + const harness = createDoctorHarness(); + harness.getSandboxSpy.mockReturnValue({ + name: "alpha", + agent: "hermes", + model: "registry-model", + provider: "ollama-local", + openshellDriver: "docker", + gatewayName: "nemoclaw-19080", + gatewayPort: 19080, + }); + + await harness.runSandboxDoctor("alpha", ["--json"], { quietJson: true }); + + expect(harness.buildToolScopeChecksSpy).not.toHaveBeenCalled(); + }); + + it("appends the local gateway result without mutating provider health", async () => { + const harness = createDoctorHarness(); + const providerHealth = { + ok: true, + probed: true, + providerLabel: "Ollama", + endpoint: "http://127.0.0.1:11434/v1/chat/completions", + detail: "healthy", + }; + harness.healthProbeSpy.mockReturnValue(providerHealth); + + const report = await harness.runSandboxDoctor("alpha", ["--json"], { quietJson: true }); + + expect(providerHealth).not.toHaveProperty("subprobes"); + expect(report?.checks).toContainEqual( + expect.objectContaining({ + group: "Inference", + label: "Provider health (gateway)", + }), + ); + }); + + it("reports agent definition failures instead of hiding the runtime channel check", async () => { + const harness = createDoctorHarness(); + harness.configuredMessagingChannelsSpy.mockReturnValue(["telegram"]); + harness.loadAgentSpy.mockImplementation(() => { + throw new Error("agent definition is invalid"); + }); + + const report = await harness.runSandboxDoctor("alpha", ["--json"], { quietJson: true }); + + expect(report?.checks).toContainEqual( + expect.objectContaining({ + group: "Messaging", + label: "Runtime channel registry", + status: "warn", + detail: "unable to resolve agent config paths: agent definition is invalid", + }), + ); }); }); diff --git a/src/lib/actions/sandbox/doctor-messaging.ts b/src/lib/actions/sandbox/doctor-messaging.ts new file mode 100644 index 00000000000..e40eb841e83 --- /dev/null +++ b/src/lib/actions/sandbox/doctor-messaging.ts @@ -0,0 +1,280 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { loadAgent } from "../../agent/defs"; +import { compareChannelSets, probeChannelRuntimeStatus } from "../../channel-runtime-status"; +import { CLI_NAME } from "../../cli/branding"; +import { + collectBuiltInMessagingChannelDiagnostics, + type MessagingChannelDiagnosticSpec, +} from "../../messaging/diagnostics"; +import { executeSandboxCommandForVerification } from "../../onboard/sandbox-verification-exec"; +import { ROOT } from "../../runner"; +import type { SandboxEntry } from "../../state/registry"; +import * as registry from "../../state/registry"; +import { buildStatusCommandDeps } from "../../status-command-deps"; +import type { DoctorCheck } from "./doctor-report"; + +const CHANNEL_STATUS_DIAGNOSTICS = collectBuiltInMessagingChannelDiagnostics(); + +function runtimeProbeUnavailableCheck(sandboxName: string, detail: string): DoctorCheck { + return { + group: "Messaging", + label: "Runtime channel registry", + status: "warn", + detail, + hint: + `start the sandbox and rerun \`${CLI_NAME} ${sandboxName} doctor\`, ` + + `or rebuild with \`${CLI_NAME} ${sandboxName} rebuild\` if the config file is missing`, + }; +} + +function runtimeVisibilityCheck( + sandboxName: string, + enabledChannels: string[], + visibleChannels: string[], + configDir: string, + configFile: string, +): DoctorCheck | null { + const { missing } = compareChannelSets(enabledChannels, visibleChannels); + if (missing.length === 0) return null; + return { + group: "Messaging", + label: "Runtime channel registry", + status: "warn", + detail: `not visible to OpenClaw runtime: ${missing.join(", ")}`, + hint: + `the OpenClaw dashboard "Channels" panel will show "No channels found" for ` + + `${missing.join(", ")}; inspect \`${configDir}/${configFile}\` ` + + `and the gateway log with \`${CLI_NAME} ${sandboxName} logs\`, then re-run ` + + `\`${CLI_NAME} ${sandboxName} rebuild\` if the channels block needs to be regenerated`, + }; +} + +function runtimeConfigCheck( + sandboxName: string, + enabledChannels: string[], + configuredChannels: string[], + configDir: string, + configFile: string, +): DoctorCheck | null { + const { missing } = compareChannelSets(enabledChannels, configuredChannels); + if (missing.length === 0) return null; + return { + group: "Messaging", + label: "Runtime channel registry", + status: "warn", + detail: `missing from sandbox config: ${missing.join(", ")}`, + hint: + `\`${configDir}/${configFile}\` is missing the channel block ` + + `for ${missing.join(", ")}; re-run \`${CLI_NAME} ${sandboxName} rebuild\` so the config is regenerated`, + }; +} + +function runtimeLogUnavailableCheck(sandboxName: string, enabledChannels: string[]): DoctorCheck { + return { + group: "Messaging", + label: "Runtime channel registry", + status: "warn", + detail: `${enabledChannels.join(", ")} present in config; gateway log unavailable, runtime startup not confirmed`, + hint: + `start the sandbox and rerun \`${CLI_NAME} ${sandboxName} doctor\`, or inspect ` + + `the gateway log with \`${CLI_NAME} ${sandboxName} logs\``, + }; +} + +function healthyRuntimeCheck(enabledChannels: string[]): DoctorCheck { + return { + group: "Messaging", + label: "Runtime channel registry", + status: "ok", + detail: `${enabledChannels.join(", ")} acknowledged by OpenClaw runtime`, + }; +} + +function unreachableRuntimeCheck(sandboxName: string): DoctorCheck { + return { + group: "Messaging", + label: "Runtime channel registry", + status: "info", + detail: "skipped because the sandbox is not reachable through its named gateway", + hint: `fix the gateway and live sandbox checks, then rerun \`${CLI_NAME} ${sandboxName} doctor\``, + }; +} + +/** + * Compare the registry's enabled channels with the runtime's config and log + * evidence. A null result means the probe does not apply, so the caller omits + * the line instead of rendering a no-op diagnostic. + */ +function channelRuntimeDoctorCheck( + sandboxName: string, + enabledChannels: string[], + sb: SandboxEntry, +): DoctorCheck | null { + if (enabledChannels.length === 0) return null; + let agent: ReturnType; + try { + agent = loadAgent(sb.agent || "openclaw"); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + return runtimeProbeUnavailableCheck( + sandboxName, + `unable to resolve agent config paths: ${detail}`, + ); + } + if (agent.configPaths.format !== "json") return null; + const configFilePath = `${agent.configPaths.dir}/${agent.configPaths.configFile}`; + const runtime = probeChannelRuntimeStatus({ + configFilePath, + executeSandboxCommand: (script: string) => + executeSandboxCommandForVerification(sandboxName, script), + }); + if (!runtime.ok) return runtimeProbeUnavailableCheck(sandboxName, runtime.detail); + if (runtime.logProbeOk) { + return ( + runtimeVisibilityCheck( + sandboxName, + enabledChannels, + runtime.visibleChannels, + agent.configPaths.dir, + agent.configPaths.configFile, + ) ?? healthyRuntimeCheck(enabledChannels) + ); + } + return ( + runtimeConfigCheck( + sandboxName, + enabledChannels, + runtime.configuredChannels, + agent.configPaths.dir, + agent.configPaths.configFile, + ) ?? runtimeLogUnavailableCheck(sandboxName, enabledChannels) + ); +} + +function getChannelStatusDiagnostic(channelName: string): MessagingChannelDiagnosticSpec | null { + return ( + CHANNEL_STATUS_DIAGNOSTICS.find((diagnostic) => diagnostic.channelId === channelName) ?? null + ); +} + +function formatDiagnosticTemplate( + template: string, + values: Readonly>, +): string { + let result = template; + for (const [key, value] of Object.entries(values)) { + result = result.replaceAll(`{${key}}`, value); + } + return result; +} + +function formatMessagingOverlapDoctorDetail(overlap: { + readonly channel: string; + readonly sandboxes: readonly [string, string]; + readonly message?: string; +}): string { + const detail = overlap.message + ? formatDiagnosticTemplate(overlap.message, { + channel: overlap.channel, + first: overlap.sandboxes[0], + second: overlap.sandboxes[1], + }) + : `'${overlap.sandboxes[0]}' and '${overlap.sandboxes[1]}' overlap`; + return `${overlap.channel}: ${detail}`; +} + +function configuredChannelsCheck(sandboxName: string, sb: SandboxEntry): DoctorCheck { + const registeredChannels = registry.getConfiguredMessagingChannelsFromEntry(sb); + const disabledChannels = new Set(registry.getDisabledMessagingChannelsFromEntry(sb)); + const channels = registeredChannels.filter((channel: string) => !disabledChannels.has(channel)); + const pausedChannels = registeredChannels.filter((channel: string) => + disabledChannels.has(channel), + ); + if (registeredChannels.length === 0) { + return { + group: "Messaging", + label: "Channels", + status: "info", + detail: "no messaging channels registered", + }; + } + if (channels.length === 0) { + return { + group: "Messaging", + label: "Channels", + status: "info", + detail: `all messaging channels paused (${pausedChannels.join(", ")})`, + hint: `run \`${CLI_NAME} ${sandboxName} channels start \` to re-enable one`, + }; + } + + const statusDeps = buildStatusCommandDeps(ROOT); + const degraded = statusDeps.checkMessagingBridgeHealth?.(sandboxName, channels, sb.agent) || []; + const overlaps = (statusDeps.findMessagingOverlaps?.() ?? []).filter( + (overlap) => channels.includes(overlap.channel) && overlap.sandboxes.includes(sandboxName), + ); + const pausedSuffix = + pausedChannels.length > 0 ? `; paused channels skipped: ${pausedChannels.join(", ")}` : ""; + const warnings = [ + ...degraded.map( + (item: { channel: string; conflicts: number }) => + `${item.channel}: ${item.conflicts} conflict(s)`, + ), + ...overlaps.map(formatMessagingOverlapDoctorDetail), + ]; + if (warnings.length > 0) { + return { + group: "Messaging", + label: "Channels", + status: "warn", + detail: warnings.join("; ") + pausedSuffix, + hint: `run \`${CLI_NAME} ${sandboxName} logs --follow\` for enabled bridge details`, + }; + } + + const diagnostic = channels + .map(getChannelStatusDiagnostic) + .find((candidate) => candidate?.doctorWhenNoHealthSignals); + if (!diagnostic?.doctorWhenNoHealthSignals) { + return { + group: "Messaging", + label: "Channels", + status: "ok", + detail: `${channels.join(", ")} enabled; no recent conflict signatures${pausedSuffix}`, + }; + } + const context = { + channel: diagnostic.channelId, + channels: channels.join(", "), + cli: CLI_NAME, + pausedSuffix, + sandbox: sandboxName, + }; + return { + group: "Messaging", + label: "Channels", + status: "info", + detail: formatDiagnosticTemplate(diagnostic.doctorWhenNoHealthSignals.detail, context), + hint: formatDiagnosticTemplate(diagnostic.doctorWhenNoHealthSignals.hint, context), + }; +} + +export function collectMessagingDoctorChecks( + sandboxName: string, + sb: SandboxEntry, + sandboxReachable: boolean, +): DoctorCheck[] { + const checks = [configuredChannelsCheck(sandboxName, sb)]; + const registered = registry.getConfiguredMessagingChannelsFromEntry(sb); + const disabled = new Set(registry.getDisabledMessagingChannelsFromEntry(sb)); + const enabled = registered.filter((channel: string) => !disabled.has(channel)); + const runtimeCheck = sandboxReachable + ? channelRuntimeDoctorCheck(sandboxName, enabled, sb) + : enabled.length > 0 + ? unreachableRuntimeCheck(sandboxName) + : null; + if (runtimeCheck) checks.push(runtimeCheck); + return checks; +} diff --git a/src/lib/actions/sandbox/doctor-report.test.ts b/src/lib/actions/sandbox/doctor-report.test.ts new file mode 100644 index 00000000000..00f2c3d30d2 --- /dev/null +++ b/src/lib/actions/sandbox/doctor-report.test.ts @@ -0,0 +1,50 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, describe, expect, it, vi } from "vitest"; +import { buildDoctorReport, type DoctorCheck, renderDoctorReport } from "./doctor-report"; + +function check(status: DoctorCheck["status"], group = "Host"): DoctorCheck { + return { group, label: `${status} check`, status, detail: `${status} detail` }; +} + +describe("doctor reports", () => { + afterEach(() => vi.restoreAllMocks()); + + it.each([ + { checks: [], status: "ok", failed: 0, warnings: 0 }, + { checks: [check("ok"), check("info")], status: "ok", failed: 0, warnings: 0 }, + { checks: [check("warn"), check("info")], status: "warn", failed: 0, warnings: 1 }, + { checks: [check("warn"), check("fail")], status: "fail", failed: 1, warnings: 1 }, + ] as const)("summarizes $status reports", ({ checks, status, failed, warnings }) => { + expect(buildDoctorReport("alpha", [...checks])).toMatchObject({ + schemaVersion: 1, + sandbox: "alpha", + status, + failed, + warnings, + }); + }); + + it("renders the machine-readable report and returns a failing exit code", () => { + const logSpy = vi.spyOn(console, "log").mockImplementation(() => undefined); + const report = buildDoctorReport("alpha", [check("fail")]); + + expect(renderDoctorReport(report, true)).toBe(1); + expect(JSON.parse(String(logSpy.mock.calls[0]?.[0]))).toEqual(report); + }); + + it("renders preferred groups first, preserves extra-group order, and includes hints", () => { + const lines: string[] = []; + vi.spyOn(console, "log").mockImplementation((line = "") => lines.push(String(line))); + const custom = { ...check("info", "Custom"), hint: "inspect the custom probe" }; + const report = buildDoctorReport("alpha", [custom, check("warn", "Messaging"), check("ok")]); + + expect(renderDoctorReport(report, false)).toBe(0); + const output = lines.join("\n"); + expect(output.indexOf("Host:")).toBeLessThan(output.indexOf("Messaging:")); + expect(output.indexOf("Messaging:")).toBeLessThan(output.indexOf("Custom:")); + expect(output).toContain("hint: inspect the custom probe"); + expect(output).toContain("healthy with 1 warning(s)"); + }); +}); diff --git a/src/lib/actions/sandbox/doctor-report.ts b/src/lib/actions/sandbox/doctor-report.ts new file mode 100644 index 00000000000..5996bd7d0da --- /dev/null +++ b/src/lib/actions/sandbox/doctor-report.ts @@ -0,0 +1,112 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { CLI_DISPLAY_NAME } from "../../cli/branding"; +import { B, D, G, R, RD, YW } from "../../cli/terminal-style"; + +export type DoctorStatus = "ok" | "warn" | "fail" | "info"; +type DoctorReportStatus = Exclude; + +export type DoctorCheck = { + group: string; + label: string; + status: DoctorStatus; + detail: string; + hint?: string; +}; + +export type DoctorReport = { + schemaVersion: 1; + sandbox: string; + status: DoctorReportStatus; + failed: number; + warnings: number; + checks: DoctorCheck[]; +}; + +function summarizeChecks(checks: DoctorCheck[]): { + status: DoctorReportStatus; + failed: number; + warned: number; +} { + const failed = checks.filter((check) => check.status === "fail").length; + const warned = checks.filter((check) => check.status === "warn").length; + if (failed > 0) return { status: "fail", failed, warned }; + if (warned > 0) return { status: "warn", failed, warned }; + return { status: "ok", failed, warned }; +} + +export function buildDoctorReport(sandboxName: string, checks: DoctorCheck[]): DoctorReport { + const summary = summarizeChecks(checks); + return { + schemaVersion: 1, + sandbox: sandboxName, + status: summary.status, + failed: summary.failed, + warnings: summary.warned, + checks, + }; +} + +function statusLabel(status: DoctorStatus): string { + switch (status) { + case "ok": + return `${G}[ok]${R}`; + case "warn": + return `${YW}[warn]${R}`; + case "fail": + return `${RD}[fail]${R}`; + case "info": + return `${D}[info]${R}`; + } +} + +function orderedGroups(report: DoctorReport): string[] { + const preferred = ["Host", "Gateway", "Sandbox", "Inference", "Messaging", "Local services"]; + const remaining = report.checks + .map((check) => check.group) + .filter((group, index, all) => !preferred.includes(group) && all.indexOf(group) === index); + return [...preferred, ...remaining]; +} + +function renderCheckGroups(report: DoctorReport): void { + for (const group of orderedGroups(report)) { + const checks = report.checks.filter((check) => check.group === group); + if (checks.length === 0) continue; + console.log(""); + console.log(` ${G}${group}:${R}`); + for (const check of checks) { + console.log(` ${statusLabel(check.status)} ${check.label}: ${check.detail}`); + if (check.hint) console.log(` ${D}hint: ${check.hint}${R}`); + } + } +} + +function renderSummary(report: DoctorReport): void { + if (report.status === "ok") { + console.log(` Summary: ${G}healthy${R}`); + return; + } + if (report.status === "warn") { + console.log(` Summary: ${YW}healthy with ${report.warnings} warning(s)${R}`); + return; + } + console.log( + ` Summary: ${RD}attention needed${R} (${report.failed} failed, ${report.warnings} warning(s))`, + ); +} + +export function renderDoctorReport(report: DoctorReport, asJson: boolean): number { + if (asJson) { + console.log(JSON.stringify(report, null, 2)); + return report.failed > 0 ? 1 : 0; + } + + console.log(""); + console.log(` ${B}${CLI_DISPLAY_NAME} doctor:${R} ${report.sandbox}`); + renderCheckGroups(report); + console.log(""); + renderSummary(report); + console.log(""); + return report.failed > 0 ? 1 : 0; +} diff --git a/src/lib/actions/sandbox/doctor-system-checks.test.ts b/src/lib/actions/sandbox/doctor-system-checks.test.ts new file mode 100644 index 00000000000..704d73adf82 --- /dev/null +++ b/src/lib/actions/sandbox/doctor-system-checks.test.ts @@ -0,0 +1,34 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createRequire } from "node:module"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +const requireDist = createRequire(import.meta.url); +const modulePath = "./doctor-system-checks.js"; + +describe("doctor system checks", () => { + afterEach(() => { + vi.restoreAllMocks(); + delete requireDist.cache[requireDist.resolve(modulePath)]; + }); + + it("validates Docker mappings against the sandbox gateway port exactly", () => { + const hostCommand = requireDist("./doctor-host-command.js"); + const captureSpy = vi + .spyOn(hostCommand, "captureHostCommand") + .mockReturnValueOnce({ status: 0, stdout: "true\thealthy\timage", stderr: "" }) + .mockReturnValueOnce({ status: 0, stdout: "0.0.0.0:19080", stderr: "" }); + const { dockerInspectGateway } = requireDist(modulePath); + + expect(dockerInspectGateway("gateway", {}, 19080)[1]).toMatchObject({ status: "ok" }); + + captureSpy + .mockReturnValueOnce({ status: 0, stdout: "true\thealthy\timage", stderr: "" }) + .mockReturnValueOnce({ status: 0, stdout: "0.0.0.0:190800", stderr: "" }); + expect(dockerInspectGateway("gateway", {}, 19080)[1]).toMatchObject({ + status: "warn", + hint: "expected host port 19080 for this sandbox gateway", + }); + }); +}); diff --git a/src/lib/actions/sandbox/doctor-system-checks.ts b/src/lib/actions/sandbox/doctor-system-checks.ts new file mode 100644 index 00000000000..1d3fb564e1a --- /dev/null +++ b/src/lib/actions/sandbox/doctor-system-checks.ts @@ -0,0 +1,200 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import path from "node:path"; +import { buildValidatedCurlCommandArgs } from "../../adapters/http/curl-args"; +import { stripAnsi } from "../../adapters/openshell/client"; +import { CLI_NAME } from "../../cli/branding"; +import { GATEWAY_PORT, OLLAMA_PORT } from "../../core/ports"; +import { isLinuxDockerDriverGatewayEnabled } from "../../onboard/docker-driver-platform"; +import type { SandboxEntry } from "../../state/registry"; +import { readCloudflaredState } from "../../tunnel/services"; +import { + buildGatewayInspectFailureChecks, + type GatewayInspectOptions, +} from "./doctor-gateway-fallback"; +import { captureHostCommand } from "./doctor-host-command"; +import type { DoctorCheck } from "./doctor-report"; + +export function oneLine(value = ""): string { + return String(value).replace(/\s+/g, " ").trim(); +} + +function gatewayContainerCheck( + containerName: string, + output: string, + options: GatewayInspectOptions, +): DoctorCheck { + const [runningRaw, healthRaw, imageRaw] = output.trim().split("\t"); + const running = runningRaw === "true"; + const health = healthRaw || "none"; + const image = imageRaw || "unknown"; + const healthy = health === "healthy" || health === "none"; + return { + group: "Gateway", + label: "Docker container", + status: running && healthy ? "ok" : "fail", + detail: `${containerName} ${running ? "running" : "stopped"} (${health}; ${image})`, + hint: running + ? undefined + : `restart the gateway with \`openshell gateway start --name ${options.gatewayName ?? "nemoclaw"}\``, + }; +} + +function gatewayPortCheck(containerName: string, expectedHostPort: number): DoctorCheck { + const port = captureHostCommand("docker", ["port", containerName, "30051/tcp"], 5000); + if (port.status !== 0 || !port.stdout.trim()) { + return { + group: "Gateway", + label: "Port mapping", + status: "fail", + detail: "30051/tcp is not published on the host", + hint: "gateway traffic will not reach OpenShell until the container is recreated with a host port", + }; + } + const mapping = oneLine(port.stdout); + const expected = new RegExp(`:${expectedHostPort}(?:\\s|$)`).test(mapping); + return { + group: "Gateway", + label: "Port mapping", + status: expected ? "ok" : "warn", + detail: mapping, + hint: expected ? undefined : `expected host port ${expectedHostPort} for this sandbox gateway`, + }; +} + +export function dockerInspectGateway( + containerName: string, + options: GatewayInspectOptions = {}, + expectedHostPort = GATEWAY_PORT, +): DoctorCheck[] { + const inspect = captureHostCommand( + "docker", + [ + "inspect", + "--format", + "{{.State.Running}}\t{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}\t{{.Config.Image}}", + containerName, + ], + 5000, + ); + if (inspect.status !== 0) { + return buildGatewayInspectFailureChecks(containerName, options); + } + return [ + gatewayContainerCheck(containerName, inspect.stdout, options), + gatewayPortCheck(containerName, expectedHostPort), + ]; +} + +export function findSandboxListLine(output: string, sandboxName: string): string | null { + const lines = stripAnsi(output).split(/\r?\n/); + return ( + lines.find((line: string) => { + const columns = line.trim().split(/\s+/); + return columns.includes(sandboxName); + }) || null + ); +} + +export function inferSandboxReadyFromLine(line: string | null): boolean | null { + if (!line) return null; + if (/\bReady\b/i.test(line)) return true; + if (/\b(Failed|Error|CrashLoopBackOff|ImagePullBackOff|Unknown|Evicted)\b/i.test(line)) { + return false; + } + return null; +} + +function stoppedCloudflaredCheck(): DoctorCheck { + return { + group: "Local services", + label: "cloudflared", + status: "info", + detail: "stopped", + hint: `no cloudflared process; run \`${CLI_NAME} tunnel start\` to start it`, + }; +} + +function staleCloudflaredPidFileCheck(): DoctorCheck { + return { + group: "Local services", + label: "cloudflared", + status: "warn", + detail: "stale PID file", + hint: `no cloudflared process (stored PID is invalid); run \`${CLI_NAME} tunnel start\` to restart it`, + }; +} + +function staleCloudflaredPidCheck(pid: number): DoctorCheck { + return { + group: "Local services", + label: "cloudflared", + status: "warn", + detail: `stale PID ${pid}`, + hint: `no cloudflared process (PID ${pid} is dead or not cloudflared); run \`${CLI_NAME} tunnel start\` to restart it`, + }; +} + +export function cloudflaredDoctorCheck(sandboxName: string): DoctorCheck { + const state = readCloudflaredState(path.join("/tmp", `nemoclaw-services-${sandboxName}`)); + switch (state.kind) { + case "stopped": + return stoppedCloudflaredCheck(); + case "stale-pid-file": + return staleCloudflaredPidFileCheck(); + case "stale-pid-process": + return staleCloudflaredPidCheck(state.pid); + case "running": + return { + group: "Local services", + label: "cloudflared", + status: "ok", + detail: `running (PID ${state.pid})`, + }; + } +} + +export function ollamaDoctorCheck(currentProvider: string): DoctorCheck { + const endpoint = `http://127.0.0.1:${OLLAMA_PORT}/api/tags`; + const result = captureHostCommand( + "curl", + buildValidatedCurlCommandArgs(["-sS", "--connect-timeout", "2", "--max-time", "4", endpoint]), + 6000, + ); + const required = currentProvider === "ollama-local"; + if (result.status !== 0) { + return { + group: "Local services", + label: "Ollama", + status: required ? "fail" : "info", + detail: `not reachable at ${endpoint}`, + hint: required ? "start Ollama or change the sandbox inference provider" : undefined, + }; + } + + let modelCount = "unknown model count"; + try { + const parsed = JSON.parse(result.stdout); + if (Array.isArray(parsed.models)) modelCount = `${parsed.models.length} model(s)`; + } catch { + /* keep generic detail */ + } + return { + group: "Local services", + label: "Ollama", + status: "ok", + detail: `reachable at ${endpoint} (${modelCount})`, + }; +} + +/** + * The legacy k3s gateway container only exists for the Kubernetes driver. + * Prefer the recorded driver and use platform detection for older entries. + */ +export function shouldInspectLegacyGatewayContainer(sb: SandboxEntry | null | undefined): boolean { + const driver = sb?.openshellDriver; + if (driver === "docker" || driver === "vm") return false; + if (driver === "kubernetes") return true; + return !isLinuxDockerDriverGatewayEnabled(); +} diff --git a/src/lib/actions/sandbox/doctor.ts b/src/lib/actions/sandbox/doctor.ts index e27302f3821..eb00a6f2f12 100644 --- a/src/lib/actions/sandbox/doctor.ts +++ b/src/lib/actions/sandbox/doctor.ts @@ -3,25 +3,19 @@ import fs from "node:fs"; import path from "node:path"; -import { buildValidatedCurlCommandArgs } from "../../adapters/http/curl-args"; import { stripAnsi } from "../../adapters/openshell/client"; import { resolveOpenshell } from "../../adapters/openshell/resolve"; import { captureOpenshell } from "../../adapters/openshell/runtime"; import { OPENSHELL_PROBE_TIMEOUT_MS } from "../../adapters/openshell/timeouts"; -import { loadAgent } from "../../agent/defs"; import * as agentRuntime from "../../agent/runtime"; -import { compareChannelSets, probeChannelRuntimeStatus } from "../../channel-runtime-status"; -import { CLI_DISPLAY_NAME, CLI_NAME } from "../../cli/branding"; -import { B, D, G, R, RD, YW } from "../../cli/terminal-style"; -import { GATEWAY_PORT, OLLAMA_PORT } from "../../core/ports"; -import { recoverNamedGatewayRuntime } from "../../gateway-runtime-action"; +import { CLI_NAME } from "../../cli/branding"; +import { GATEWAY_PORT } from "../../core/ports"; +import { + getNamedGatewayLifecycleState, + recoverNamedGatewayRuntime, +} from "../../gateway-runtime-action"; import { parseGatewayInference } from "../../inference/config"; import { type ProviderHealthStatus, probeProviderHealth } from "../../inference/health"; -import { - collectBuiltInMessagingChannelDiagnostics, - type MessagingChannelDiagnosticSpec, -} from "../../messaging/diagnostics"; -import { isLinuxDockerDriverGatewayEnabled } from "../../onboard/docker-driver-platform"; import { resolveGatewayName, resolveSandboxGatewayName } from "../../onboard/gateway-binding"; import { executeSandboxCommandForVerification } from "../../onboard/sandbox-verification-exec"; import { ROOT } from "../../runner"; @@ -30,38 +24,30 @@ import * as sandboxVersion from "../../sandbox/version"; import * as shields from "../../shields"; import type { SandboxEntry } from "../../state/registry"; import * as registry from "../../state/registry"; -import { buildStatusCommandDeps } from "../../status-command-deps"; -import { readCloudflaredState } from "../../tunnel/services"; import { runSandboxAutoPairApprovalPass, wrapSandboxShellScript } from "./auto-pair-approval"; import { buildConfigPermsCheck } from "./doctor-config-perms"; -import { - buildGatewayInspectFailureChecks, - type GatewayInspectOptions, -} from "./doctor-gateway-fallback"; import { captureHostCommand } from "./doctor-host-command"; +import { collectMessagingDoctorChecks } from "./doctor-messaging"; +import { + buildDoctorReport, + type DoctorCheck, + type DoctorReport, + type DoctorStatus, + renderDoctorReport, +} from "./doctor-report"; +import { + cloudflaredDoctorCheck, + dockerInspectGateway, + findSandboxListLine, + inferSandboxReadyFromLine, + ollamaDoctorCheck, + oneLine, + shouldInspectLegacyGatewayContainer, +} from "./doctor-system-checks"; import { buildToolScopeChecks } from "./doctor-tool-scope"; import { probeSandboxInferenceGatewayHealth } from "./process-recovery"; -const CHANNEL_STATUS_DIAGNOSTICS = collectBuiltInMessagingChannelDiagnostics(); - -type DoctorStatus = "ok" | "warn" | "fail" | "info"; - -export type DoctorCheck = { - group: string; - label: string; - status: DoctorStatus; - detail: string; - hint?: string; -}; - -export type DoctorReport = { - schemaVersion: 1; - sandbox: string; - status: DoctorStatus; - failed: number; - warnings: number; - checks: DoctorCheck[]; -}; +export type { DoctorCheck, DoctorReport } from "./doctor-report"; function pushInferenceHealthCheck(checks: DoctorCheck[], probe: ProviderHealthStatus): void { const label = probe.probeLabel ? `Provider health (${probe.probeLabel})` : "Provider health"; @@ -78,807 +64,468 @@ function pushInferenceHealthCheck(checks: DoctorCheck[], probe: ProviderHealthSt }); } -function oneLine(value = ""): string { - return String(value).replace(/\s+/g, " ").trim(); -} +type RunSandboxDoctorOptions = { + quietJson?: boolean; +}; -function doctorSummary(checks: DoctorCheck[]): { - status: DoctorStatus; - failed: number; - warned: number; -} { - const failed = checks.filter((check) => check.status === "fail").length; - const warned = checks.filter((check) => check.status === "warn").length; - if (failed > 0) return { status: "fail", failed, warned }; - if (warned > 0) return { status: "warn", failed, warned }; - return { status: "ok", failed, warned }; -} +type DoctorIntent = { + asJson: boolean; + wantsFix: boolean; +}; + +type GatewayProbe = { + checks: DoctorCheck[]; + connected: boolean; +}; -function doctorStatusLabel(status: DoctorStatus): string { - switch (status) { - case "ok": - return `${G}[ok]${R}`; - case "warn": - return `${YW}[warn]${R}`; - case "fail": - return `${RD}[fail]${R}`; - case "info": - return `${D}[info]${R}`; - default: - return `[${status}]`; +type SandboxProbe = { + checks: DoctorCheck[]; + reachable: boolean; +}; + +type InferenceRoute = { + model: string; + provider: string; +}; + +function parseDoctorIntent(sandboxName: string, args: string[]): DoctorIntent | null { + const asJson = args.includes("--json"); + const wantsFix = args.includes("--fix"); + const helpRequested = args.includes("--help") || args.includes("-h"); + const unknown = args.filter((arg) => !["--json", "--fix", "--help", "-h"].includes(arg)); + if (helpRequested) { + console.log(` Usage: ${CLI_NAME} doctor [--json] [--fix]`); + console.log( + ` --fix Restore the mutable OpenClaw config permission contract if it was tightened,`, + ); + console.log(` and approve pending allowlisted dashboard/CLI tool-scope upgrades`); + return null; } + if (unknown.length > 0) { + console.error( + ` Unknown doctor argument${unknown.length === 1 ? "" : "s"}: ${unknown.join(" ")}`, + ); + console.error(` Usage: ${CLI_NAME} doctor [--json] [--fix]`); + process.exit(1); + } + // `--fix` mutates sandbox permissions; `--json` is the machine-readable + // readiness-gate path. Refuse the combination so automation consuming JSON + // can never trigger a silent repair (the JSON report has no dedicated + // repair-intent field). Run `doctor --json` to detect, then `doctor --fix` + // to repair. + if (wantsFix && asJson) { + console.error(` ${CLI_NAME} doctor: --fix cannot be combined with --json`); + console.error( + ` Run \`${CLI_NAME} ${sandboxName} doctor --json\` to detect, then \`${CLI_NAME} ${sandboxName} doctor --fix\` to repair`, + ); + process.exit(1); + } + return { asJson, wantsFix }; } -function buildDoctorReport(sandboxName: string, checks: DoctorCheck[]): DoctorReport { - const summary = doctorSummary(checks); +function cliBuildCheck(): DoctorCheck { + const exists = fs.existsSync(path.join(ROOT, "dist", "nemoclaw.js")); return { - schemaVersion: 1, - sandbox: sandboxName, - status: summary.status, - failed: summary.failed, - warnings: summary.warned, - checks, + group: "Host", + label: "CLI build", + status: exists ? "ok" : "fail", + detail: exists ? "dist/nemoclaw.js present" : "dist/nemoclaw.js missing", + hint: exists ? undefined : "run `npm run build:cli`", }; } -function doctorReportExitCode(report: DoctorReport): number { - return report.failed > 0 ? 1 : 0; +function collectHostChecks(): { + checks: DoctorCheck[]; + openshellBin: ReturnType; +} { + const cli = cliBuildCheck(); + const dockerInfo = captureHostCommand("docker", ["info", "--format", "{{.ServerVersion}}"], 8000); + const openshellBin = resolveOpenshell(); + return { + checks: [ + cli, + { + group: "Host", + label: "Docker daemon", + status: dockerInfo.status === 0 ? "ok" : "fail", + detail: + dockerInfo.status === 0 + ? `server ${dockerInfo.stdout.trim() || "unknown"}` + : oneLine(dockerInfo.stderr || dockerInfo.error?.message || "docker info failed"), + hint: + dockerInfo.status === 0 + ? undefined + : "start Docker and verify your user can access the daemon", + }, + { + group: "Host", + label: "OpenShell CLI", + status: openshellBin ? "ok" : "fail", + detail: openshellBin || "not found on PATH", + hint: openshellBin ? undefined : "install OpenShell before using sandbox commands", + }, + ], + openshellBin, + }; } -function renderDoctorReport(report: DoctorReport, asJson: boolean): number { - if (asJson) { - console.log(JSON.stringify(report, null, 2)); - return doctorReportExitCode(report); - } - - console.log(""); - console.log(` ${B}${CLI_DISPLAY_NAME} doctor:${R} ${report.sandbox}`); - const groupOrder = ["Host", "Gateway", "Sandbox", "Inference", "Messaging", "Local services"]; - const orderedGroups = [ - ...groupOrder, - ...report.checks - .map((check) => check.group) - .filter((group, index, all) => !groupOrder.includes(group) && all.indexOf(group) === index), - ]; - for (const group of orderedGroups) { - const groupChecks = report.checks.filter((check) => check.group === group); - if (groupChecks.length === 0) continue; - console.log(""); - console.log(` ${G}${group}:${R}`); - for (const check of groupChecks) { - console.log(` ${doctorStatusLabel(check.status)} ${check.label}: ${check.detail}`); - if (check.hint) { - console.log(` ${D}hint: ${check.hint}${R}`); - } - } - } - - console.log(""); - if (report.status === "ok") { - console.log(` Summary: ${G}healthy${R}`); - } else if (report.status === "warn") { - console.log(` Summary: ${YW}healthy with ${report.warnings} warning(s)${R}`); - } else { - console.log( - ` Summary: ${RD}attention needed${R} (${report.failed} failed, ${report.warnings} warning(s))`, +async function collectGatewayChecks( + gatewayName: string, + sb: SandboxEntry | null | undefined, + openshellBin: ReturnType, + recoverGateway: boolean, +): Promise { + const checks: DoctorCheck[] = []; + const gateway = openshellBin + ? await probeOpenShellGateway(gatewayName, recoverGateway) + : { check: null, connected: false }; + if (gateway.check) checks.push(gateway.check); + if (shouldInspectLegacyGatewayContainer(sb)) { + checks.push( + ...dockerInspectGateway( + `openshell-cluster-${gatewayName}`, + { + namedGatewayConnected: gateway.connected, + gatewayName, + }, + sb?.gatewayPort ?? GATEWAY_PORT, + ), ); } - console.log(""); - return doctorReportExitCode(report); + return { checks, connected: gateway.connected }; } -function dockerInspectGateway( - containerName: string, - options: GatewayInspectOptions = {}, -): DoctorCheck[] { - const checks: DoctorCheck[] = []; - const inspect = captureHostCommand( - "docker", - [ - "inspect", - "--format", - "{{.State.Running}}\t{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}\t{{.Config.Image}}", - containerName, - ], - 5000, - ); - if (inspect.status !== 0) { - return buildGatewayInspectFailureChecks(containerName, options); - } - - const [runningRaw, healthRaw, imageRaw] = inspect.stdout.trim().split("\t"); - const running = runningRaw === "true"; - const health = healthRaw || "none"; - const image = imageRaw || "unknown"; - const healthOk = health === "healthy" || health === "none"; - checks.push({ - group: "Gateway", - label: "Docker container", - status: running && healthOk ? "ok" : "fail", - detail: `${containerName} ${running ? "running" : "stopped"} (${health}; ${image})`, - hint: running - ? undefined - : `restart the gateway with \`openshell gateway start --name ${options.gatewayName ?? "nemoclaw"}\``, - }); +async function gatewayLifecycle(gatewayName: string, recoverGateway: boolean) { + if (!recoverGateway) return getNamedGatewayLifecycleState(gatewayName); + const recovery = await recoverNamedGatewayRuntime({ gatewayName }); + return recovery.after || recovery.before; +} - const port = captureHostCommand("docker", ["port", containerName, "30051/tcp"], 5000); - if (port.status === 0 && port.stdout.trim()) { - const mapping = oneLine(port.stdout); - checks.push({ - group: "Gateway", - label: "Port mapping", - status: mapping.includes(`:${GATEWAY_PORT}`) ? "ok" : "warn", - detail: mapping, - hint: mapping.includes(`:${GATEWAY_PORT}`) - ? undefined - : `expected host port ${GATEWAY_PORT} from NEMOCLAW_GATEWAY_PORT`, - }); - } else { - checks.push({ +async function probeOpenShellGateway( + gatewayName: string, + recoverGateway: boolean, +): Promise<{ + check: DoctorCheck; + connected: boolean; +}> { + const lifecycle = await gatewayLifecycle(gatewayName, recoverGateway); + const cleanStatus = stripAnsi(lifecycle?.status || ""); + const connected = lifecycle?.state === "healthy_named"; + return { + connected, + check: { group: "Gateway", - label: "Port mapping", - status: "fail", - detail: "30051/tcp is not published on the host", - hint: "gateway traffic will not reach OpenShell until the container is recreated with a host port", - }); - } - return checks; + label: "OpenShell status", + status: connected ? "ok" : "fail", + detail: connected + ? `connected to ${gatewayName}` + : oneLine(cleanStatus || lifecycle?.gatewayInfo || `not connected to ${gatewayName}`), + hint: connected ? undefined : `run \`openshell gateway select ${gatewayName}\` and retry`, + }, + }; } -function findSandboxListLine(output: string, sandboxName: string): string | null { - const lines = stripAnsi(output).split(/\r?\n/); - return ( - lines.find((line: string) => { - const columns = line.trim().split(/\s+/); - return columns.includes(sandboxName); - }) || null - ); +function liveSandboxDetail( + sandboxName: string, + present: boolean, + ready: boolean | null, + line: string | null, +): string { + if (!present) return `${sandboxName} not present in live OpenShell sandbox list`; + if (ready) return `${sandboxName} present (Ready)`; + return `${sandboxName} present${line ? ` (${oneLine(line)})` : ""}`; } -function inferSandboxReadyFromLine(line: string | null): boolean | null { - if (!line) return null; - if (/\bReady\b/i.test(line)) return true; - if (/\b(Failed|Error|CrashLoopBackOff|ImagePullBackOff|Unknown|Evicted)\b/i.test(line)) { - return false; +function liveSandboxHint( + sandboxName: string, + present: boolean, + ready: boolean | null, +): string | undefined { + if (!present) { + return `run \`${CLI_NAME} ${sandboxName} status\` or recreate with \`${CLI_NAME} onboard\``; } - return null; + if (ready) return undefined; + return `run \`${CLI_NAME} ${sandboxName} status\` or \`${CLI_NAME} ${sandboxName} logs --follow\``; } -function stoppedCloudflaredCheck(): DoctorCheck { +function liveSandboxCheck(sandboxName: string): SandboxProbe { + const list = captureOpenshell(["sandbox", "list"], { + ignoreError: true, + timeout: OPENSHELL_PROBE_TIMEOUT_MS, + }); + const liveNames = parseLiveSandboxNames(list.output || ""); + const present = list.status === 0 && liveNames.has(sandboxName); + const line = findSandboxListLine(list.output || "", sandboxName); + const ready = inferSandboxReadyFromLine(line); + const reachable = present && ready === true; return { - group: "Local services", - label: "cloudflared", - status: "info", - detail: "stopped", - hint: `no cloudflared process; run \`${CLI_NAME} tunnel start\` to start it`, + reachable, + checks: [ + { + group: "Sandbox", + label: "Live sandbox", + status: reachable ? "ok" : "fail", + detail: liveSandboxDetail(sandboxName, present, ready, line), + hint: liveSandboxHint(sandboxName, present, ready), + }, + ], }; } -function staleCloudflaredPidFileCheck(): DoctorCheck { +function collectSandboxReadinessChecks( + sandboxName: string, + openshellBin: ReturnType, + openshellConnected: boolean, +): SandboxProbe { + if (openshellBin && openshellConnected) return liveSandboxCheck(sandboxName); + if (!openshellBin) return { checks: [], reachable: false }; return { - group: "Local services", - label: "cloudflared", - status: "warn", - detail: "stale PID file", - hint: `no cloudflared process (stored PID is invalid); run \`${CLI_NAME} tunnel start\` to restart it`, + reachable: false, + checks: [ + { + group: "Sandbox", + label: "Live sandbox", + status: "fail", + detail: "skipped because the nemoclaw gateway is not connected", + hint: "fix the gateway check above before trusting sandbox readiness", + }, + ], }; } -function staleCloudflaredPidCheck(pid: number): DoctorCheck { +function resolveInferenceRoute( + sb: SandboxEntry | null | undefined, + openshellBin: ReturnType, + openshellConnected: boolean, +): InferenceRoute { + const live = + openshellBin && openshellConnected + ? parseGatewayInference( + captureOpenshell(["inference", "get"], { + ignoreError: true, + timeout: OPENSHELL_PROBE_TIMEOUT_MS, + }).output, + ) + : null; return { - group: "Local services", - label: "cloudflared", - status: "warn", - detail: `stale PID ${pid}`, - hint: `no cloudflared process (PID ${pid} is dead or not cloudflared); run \`${CLI_NAME} tunnel start\` to restart it`, + model: live?.model || sb?.model || "unknown", + provider: live?.provider || sb?.provider || "unknown", }; } -function cloudflaredDoctorCheck(sandboxName: string): DoctorCheck { - const state = readCloudflaredState(path.join("/tmp", `nemoclaw-services-${sandboxName}`)); - switch (state.kind) { - case "stopped": - return stoppedCloudflaredCheck(); - case "stale-pid-file": - return staleCloudflaredPidFileCheck(); - case "stale-pid-process": - return staleCloudflaredPidCheck(state.pid); - case "running": - return { - group: "Local services", - label: "cloudflared", - status: "ok", - detail: `running (PID ${state.pid})`, - }; - } +function inferenceRouteCheck(sandboxName: string, route: InferenceRoute): DoctorCheck { + const known = route.provider !== "unknown" || route.model !== "unknown"; + return { + group: "Inference", + label: "Route", + status: known ? "ok" : "warn", + detail: `${route.provider} / ${route.model}`, + hint: known + ? undefined + : `run \`${CLI_NAME} ${sandboxName} status\` after the gateway is healthy`, + }; } -function ollamaDoctorCheck(currentProvider: string): DoctorCheck { - const endpoint = `http://127.0.0.1:${OLLAMA_PORT}/api/tags`; - const result = captureHostCommand( - "curl", - buildValidatedCurlCommandArgs(["-sS", "--connect-timeout", "2", "--max-time", "4", endpoint]), - 6000, - ); - const required = currentProvider === "ollama-local"; - if (result.status !== 0) { - return { - group: "Local services", - label: "Ollama", - status: required ? "fail" : "info", - detail: `not reachable at ${endpoint}`, - hint: required ? "start Ollama or change the sandbox inference provider" : undefined, - }; - } +function isLocalInferenceProvider(provider: string): boolean { + return provider === "ollama-local" || provider === "vllm-local"; +} - let modelCount = "unknown model count"; - try { - const parsed = JSON.parse(result.stdout); - if (Array.isArray(parsed.models)) { - modelCount = `${parsed.models.length} model(s)`; - } - } catch { - /* keep generic detail */ - } +function skippedInferenceGatewayProbe(): ProviderHealthStatus { return { - group: "Local services", - label: "Ollama", - status: "ok", - detail: `reachable at ${endpoint} (${modelCount})`, + ok: false, + probed: false, + providerLabel: "Inference gateway chain", + endpoint: "", + detail: "skipped because the sandbox is not reachable through its named gateway", + probeLabel: "gateway", }; } -/** - * Compare the registry's enabled-channels list with channels the OpenClaw - * runtime actually acknowledged inside the sandbox (config block in - * /sandbox/.openclaw/openclaw.json plus a gateway-log mention). Returns - * null when the probe doesn't apply (no enabled channels, agent has no - * JSON config) so the caller can skip the check entirely instead of - * rendering a no-op line. Fixes #4156 — without this, a sandbox where - * the OpenClaw runtime silently ignored a configured channel looks healthy - * at `doctor` time even though the dashboard shows "No channels found". - */ -function channelRuntimeDoctorCheck( +async function collectInferenceSubprobes( sandboxName: string, - enabledChannels: string[], -): DoctorCheck | null { - if (enabledChannels.length === 0) return null; - let agent: ReturnType; - try { - const sb = registry.getSandbox(sandboxName); - agent = loadAgent(sb?.agent || "openclaw"); - } catch { - return null; - } - if (agent.configPaths.format !== "json") return null; - const configFilePath = `${agent.configPaths.dir}/${agent.configPaths.configFile}`; - const runtime = probeChannelRuntimeStatus({ - configFilePath, - executeSandboxCommand: (script: string) => - executeSandboxCommandForVerification(sandboxName, script), - }); - if (!runtime.ok) { - return { - group: "Messaging", - label: "Runtime channel registry", - status: "warn", - detail: runtime.detail, - hint: - `start the sandbox and rerun \`${CLI_NAME} ${sandboxName} doctor\`, ` + - `or rebuild with \`${CLI_NAME} ${sandboxName} rebuild\` if the config file is missing`, - }; + provider: string, + sandboxReachable: boolean, + existing: ProviderHealthStatus[], +): Promise { + if (!isLocalInferenceProvider(provider)) return existing; + if (!sandboxReachable) return [...existing, skippedInferenceGatewayProbe()]; + const gateway = await probeSandboxInferenceGatewayHealth(sandboxName); + if (!gateway) return existing; + return [ + ...existing, + { + ok: gateway.ok, + probed: true, + providerLabel: "Inference gateway chain", + endpoint: gateway.endpoint, + detail: gateway.detail, + probeLabel: "gateway", + ...(gateway.ok ? {} : { failureLabel: "unreachable" as const }), + }, + ]; +} + +async function collectInferenceChecks( + sandboxName: string, + route: InferenceRoute, + sandboxReachable: boolean, +): Promise { + const checks = [inferenceRouteCheck(sandboxName, route)]; + if (route.provider === "unknown") return checks; + const health = probeProviderHealth(route.provider); + if (!health) { + checks.push({ + group: "Inference", + label: "Provider health", + status: "info", + detail: `no health probe registered for ${route.provider}`, + }); + return checks; } - if (runtime.logProbeOk) { - // Diff against the log-corroborated runtime view. Catches both the - // stale-rebuild path (channel block missing) and the runtime-startup - // path (config has it, log doesn't). - const { missing: notRunning } = compareChannelSets(enabledChannels, runtime.visibleChannels); - if (notRunning.length > 0) { + + const subprobes = await collectInferenceSubprobes( + sandboxName, + route.provider, + sandboxReachable, + health.subprobes ?? [], + ); + pushInferenceHealthCheck(checks, health); + for (const subprobe of subprobes) pushInferenceHealthCheck(checks, subprobe); + return checks; +} + +function agentVersionDoctorCheck(sandboxName: string): DoctorCheck { + try { + const version = sandboxVersion.checkAgentVersion(sandboxName); + const agentName = agentRuntime.getAgentDisplayName(agentRuntime.getSessionAgent(sandboxName)); + if (version.isStale) { return { - group: "Messaging", - label: "Runtime channel registry", + group: "Sandbox", + label: "Agent version", status: "warn", - detail: `not visible to OpenClaw runtime: ${notRunning.join(", ")}`, - hint: - `the OpenClaw dashboard "Channels" panel will show "No channels found" for ` + - `${notRunning.join(", ")}; inspect \`${agent.configPaths.dir}/${agent.configPaths.configFile}\` ` + - `and the gateway log with \`${CLI_NAME} ${sandboxName} logs\`, then re-run ` + - `\`${CLI_NAME} ${sandboxName} rebuild\` if the channels block needs to be regenerated`, + detail: `${agentName} v${version.sandboxVersion || "unknown"}; v${version.expectedVersion} available`, + hint: `run \`${CLI_NAME} ${sandboxName} rebuild\``, }; } - } else { - // Log unavailable: we can still detect a config-only mismatch - // (registry expects telegram but openclaw.json doesn't have it). - // Surface that as a warn so a stale rebuild isn't masked by an - // unreadable log (CodeRabbit on PR #4182). The log-unavailable - // warning below still runs when configMissing is empty. - const { missing: configMissing } = compareChannelSets( - enabledChannels, - runtime.configuredChannels, - ); - if (configMissing.length > 0) { + if (version.sandboxVersion) { return { - group: "Messaging", - label: "Runtime channel registry", - status: "warn", - detail: `missing from sandbox config: ${configMissing.join(", ")}`, - hint: - `\`${agent.configPaths.dir}/${agent.configPaths.configFile}\` is missing the channel block ` + - `for ${configMissing.join(", ")}; re-run \`${CLI_NAME} ${sandboxName} rebuild\` so the config is regenerated`, + group: "Sandbox", + label: "Agent version", + status: "ok", + detail: `${agentName} v${version.sandboxVersion}`, }; } - } - if (!runtime.logProbeOk) { - return { - group: "Messaging", - label: "Runtime channel registry", - status: "warn", - detail: `${enabledChannels.join(", ")} present in config; gateway log unavailable, runtime startup not confirmed`, - hint: - `start the sandbox and rerun \`${CLI_NAME} ${sandboxName} doctor\`, or inspect ` + - `the gateway log with \`${CLI_NAME} ${sandboxName} logs\``, - }; - } - return { - group: "Messaging", - label: "Runtime channel registry", - status: "ok", - detail: `${enabledChannels.join(", ")} acknowledged by OpenClaw runtime`, - }; -} - -function messagingDoctorCheck(sandboxName: string, sb: SandboxEntry): DoctorCheck { - const registeredChannels = registry.getConfiguredMessagingChannelsFromEntry(sb); - const disabledChannels = new Set(registry.getDisabledMessagingChannelsFromEntry(sb)); - const channels = registeredChannels.filter((channel: string) => !disabledChannels.has(channel)); - const pausedChannels = registeredChannels.filter((channel: string) => - disabledChannels.has(channel), - ); - if (registeredChannels.length === 0) { return { - group: "Messaging", - label: "Channels", + group: "Sandbox", + label: "Agent version", status: "info", - detail: "no messaging channels registered", + detail: "could not detect version", }; - } - - if (channels.length === 0) { + } catch { return { - group: "Messaging", - label: "Channels", + group: "Sandbox", + label: "Agent version", status: "info", - detail: `all messaging channels paused (${pausedChannels.join(", ")})`, - hint: `run \`${CLI_NAME} ${sandboxName} channels start \` to re-enable one`, - }; - } - - const statusDeps = buildStatusCommandDeps(ROOT); - const degraded = statusDeps.checkMessagingBridgeHealth?.(sandboxName, channels, sb.agent) || []; - const overlaps = (statusDeps.findMessagingOverlaps?.() ?? []).filter( - (overlap) => channels.includes(overlap.channel) && overlap.sandboxes.includes(sandboxName), - ); - const pausedSuffix = - pausedChannels.length > 0 ? `; paused channels skipped: ${pausedChannels.join(", ")}` : ""; - const warningDetails = [ - ...degraded.map( - (item: { channel: string; conflicts: number }) => - `${item.channel}: ${item.conflicts} conflict(s)`, - ), - ...overlaps.map(formatMessagingOverlapDoctorDetail), - ]; - if (warningDetails.length === 0) { - const deepProbeDiagnostic = channels - .map(getChannelStatusDiagnostic) - .find((diagnostic) => diagnostic?.doctorWhenNoHealthSignals); - if (deepProbeDiagnostic?.doctorWhenNoHealthSignals) { - const templateContext = { - channel: deepProbeDiagnostic.channelId, - channels: channels.join(", "), - cli: CLI_NAME, - pausedSuffix, - sandbox: sandboxName, - }; - return { - group: "Messaging", - label: "Channels", - status: "info", - detail: formatDiagnosticTemplate( - deepProbeDiagnostic.doctorWhenNoHealthSignals.detail, - templateContext, - ), - hint: formatDiagnosticTemplate( - deepProbeDiagnostic.doctorWhenNoHealthSignals.hint, - templateContext, - ), - }; - } - return { - group: "Messaging", - label: "Channels", - status: "ok", - detail: `${channels.join(", ")} enabled; no recent conflict signatures${pausedSuffix}`, + detail: "version check unavailable", }; } +} +function shieldsDoctorCheck(sandboxName: string): DoctorCheck { + const posture = shields.getShieldsPosture(sandboxName, true); + const status: DoctorStatus = + posture.mode === "locked" + ? "ok" + : posture.mode === "temporarily_unlocked" || posture.mode === "error" + ? "warn" + : "info"; + const hint = + posture.mode === "mutable_default" + ? `run \`${CLI_NAME} ${sandboxName} shields up\` to opt into lockdown` + : posture.mode === "locked" + ? undefined + : `run \`${CLI_NAME} ${sandboxName} shields status\` for details`; return { - group: "Messaging", - label: "Channels", - status: "warn", - detail: warningDetails.join("; ") + pausedSuffix, - hint: `run \`${CLI_NAME} ${sandboxName} logs --follow\` for enabled bridge details`, + group: "Sandbox", + label: "Shields", + status, + detail: posture.detail, + hint, }; } -function getChannelStatusDiagnostic(channelName: string): MessagingChannelDiagnosticSpec | null { - return ( - CHANNEL_STATUS_DIAGNOSTICS.find((diagnostic) => diagnostic.channelId === channelName) ?? null - ); -} - -function formatMessagingOverlapDoctorDetail(overlap: { - readonly channel: string; - readonly sandboxes: readonly [string, string]; - readonly message?: string; -}): string { - const detail = overlap.message - ? formatDiagnosticTemplate(overlap.message, { - channel: overlap.channel, - first: overlap.sandboxes[0], - second: overlap.sandboxes[1], - }) - : `'${overlap.sandboxes[0]}' and '${overlap.sandboxes[1]}' overlap`; - return `${overlap.channel}: ${detail}`; +function collectRegisteredSandboxChecks( + sandboxName: string, + sb: SandboxEntry | null | undefined, + wantsFix: boolean, + sandboxReachable: boolean, +): DoctorCheck[] { + if (!sb) return []; + const checks = [agentVersionDoctorCheck(sandboxName), shieldsDoctorCheck(sandboxName)]; + const permsCheck = buildConfigPermsCheck(sandboxName, wantsFix, { + inspect: shields.inspectMutableConfigPerms, + repair: shields.repairMutableConfigPerms, + cliName: CLI_NAME, + }); + if (permsCheck) checks.push(permsCheck); + checks.push(...collectMessagingDoctorChecks(sandboxName, sb, sandboxReachable)); + return checks; } -function formatDiagnosticTemplate( - template: string, - values: Readonly>, -): string { - let result = template; - for (const [key, value] of Object.entries(values)) { - result = result.replaceAll(`{${key}}`, value); - } - return result; +function collectToolScopeChecks( + sandboxName: string, + sb: SandboxEntry | null | undefined, + sandboxReachable: boolean, + wantsFix: boolean, +): DoctorCheck[] { + if (!sb || !sandboxReachable || (sb.agent ?? "openclaw") !== "openclaw") return []; + return buildToolScopeChecks(sandboxName, CLI_NAME, wantsFix, { + exec: (name, script) => + executeSandboxCommandForVerification(name, wrapSandboxShellScript(script)), + runApprovalPass: (name) => { + const result = runSandboxAutoPairApprovalPass(name, { capture: true }); + return { reported: result.reported, approved: result.approved }; + }, + }); } -/** - * Decide whether to inspect the legacy k3s gateway container - * (`openshell-cluster-`). That container only exists for the legacy - * Kubernetes gateway driver. The current Linux/arm64 Docker-driver gateway runs - * as a host process (or a separate `nemoclaw-openshell-gateway` compatibility - * container), so inspecting `openshell-cluster-nemoclaw` there always fails and - * produces a false doctor failure even when OpenShell reports the named gateway - * as connected (#4502). Prefer the sandbox's recorded driver; fall back to - * platform detection for older registry entries that predate the field. - */ -function shouldInspectLegacyGatewayContainer(sb: SandboxEntry | null | undefined): boolean { - const driver = sb?.openshellDriver; - if (driver === "docker" || driver === "vm") return false; - if (driver === "kubernetes") return true; - return !isLinuxDockerDriverGatewayEnabled(); +async function collectDoctorChecks( + sandboxName: string, + sb: SandboxEntry | null | undefined, + gatewayName: string, + intent: DoctorIntent, +): Promise { + const host = collectHostChecks(); + const gateway = await collectGatewayChecks(gatewayName, sb, host.openshellBin, !intent.asJson); + const sandbox = collectSandboxReadinessChecks(sandboxName, host.openshellBin, gateway.connected); + const route = resolveInferenceRoute(sb, host.openshellBin, gateway.connected); + return [ + ...host.checks, + ...gateway.checks, + ...sandbox.checks, + ...(await collectInferenceChecks(sandboxName, route, sandbox.reachable)), + ...collectRegisteredSandboxChecks(sandboxName, sb, intent.wantsFix, sandbox.reachable), + ...collectToolScopeChecks(sandboxName, sb, sandbox.reachable, intent.wantsFix), + ollamaDoctorCheck(route.provider), + cloudflaredDoctorCheck(sandboxName), + ]; } -type RunSandboxDoctorOptions = { - quietJson?: boolean; -}; - -// eslint-disable-next-line complexity export async function runSandboxDoctor( sandboxName: string, args: string[] = [], options: RunSandboxDoctorOptions = {}, ): Promise { - const asJson = args.includes("--json"); - const wantsFix = args.includes("--fix"); - const helpRequested = args.includes("--help") || args.includes("-h"); - const unknown = args.filter((arg) => !["--json", "--fix", "--help", "-h"].includes(arg)); - if (helpRequested) { - console.log(` Usage: ${CLI_NAME} doctor [--json] [--fix]`); - console.log( - ` --fix Restore the mutable OpenClaw config permission contract if it was tightened,`, - ); - console.log(` and approve pending allowlisted dashboard/CLI tool-scope upgrades`); - return; - } - if (unknown.length > 0) { - console.error( - ` Unknown doctor argument${unknown.length === 1 ? "" : "s"}: ${unknown.join(" ")}`, - ); - console.error(` Usage: ${CLI_NAME} doctor [--json] [--fix]`); - process.exit(1); - } - // `--fix` mutates sandbox permissions; `--json` is the machine-readable - // readiness-gate path. Refuse the combination so automation consuming JSON - // can never trigger a silent repair (the JSON report has no dedicated - // repair-intent field). Run `doctor --json` to detect, then `doctor --fix` - // to repair. - if (wantsFix && asJson) { - console.error(` ${CLI_NAME} doctor: --fix cannot be combined with --json`); - console.error( - ` Run \`${CLI_NAME} ${sandboxName} doctor --json\` to detect, then \`${CLI_NAME} ${sandboxName} doctor --fix\` to repair`, - ); - process.exit(1); - } + const intent = parseDoctorIntent(sandboxName, args); + if (!intent) return undefined; const sb = registry.getSandbox(sandboxName); const gatewayName = sb ? resolveSandboxGatewayName(sb) : resolveGatewayName(GATEWAY_PORT); - const checks: DoctorCheck[] = []; - // Tracks whether the named sandbox is present-and-Ready, so live-only probes - // (e.g. the #4616 dashboard tool-scope diagnostic) only run when they can - // actually reach the sandbox via `openshell sandbox exec`. - let sandboxReachable = false; - - checks.push({ - group: "Host", - label: "CLI build", - status: fs.existsSync(path.join(ROOT, "dist", "nemoclaw.js")) ? "ok" : "fail", - detail: fs.existsSync(path.join(ROOT, "dist", "nemoclaw.js")) - ? "dist/nemoclaw.js present" - : "dist/nemoclaw.js missing", - hint: fs.existsSync(path.join(ROOT, "dist", "nemoclaw.js")) - ? undefined - : "run `npm run build:cli`", - }); - - const dockerInfo = captureHostCommand("docker", ["info", "--format", "{{.ServerVersion}}"], 8000); - checks.push({ - group: "Host", - label: "Docker daemon", - status: dockerInfo.status === 0 ? "ok" : "fail", - detail: - dockerInfo.status === 0 - ? `server ${dockerInfo.stdout.trim() || "unknown"}` - : oneLine(dockerInfo.stderr || dockerInfo.error?.message || "docker info failed"), - hint: - dockerInfo.status === 0 - ? undefined - : "start Docker and verify your user can access the daemon", - }); - - const openshellBin = resolveOpenshell(); - checks.push({ - group: "Host", - label: "OpenShell CLI", - status: openshellBin ? "ok" : "fail", - detail: openshellBin || "not found on PATH", - hint: openshellBin ? undefined : "install OpenShell before using sandbox commands", - }); - - let openshellConnected = false; - if (openshellBin) { - const recovery = await recoverNamedGatewayRuntime({ gatewayName }); - const lifecycle = recovery.after || recovery.before; - const cleanStatus = stripAnsi(lifecycle?.status || ""); - openshellConnected = lifecycle?.state === "healthy_named"; - checks.push({ - group: "Gateway", - label: "OpenShell status", - status: openshellConnected ? "ok" : "fail", - detail: openshellConnected - ? `connected to ${gatewayName}` - : oneLine(cleanStatus || lifecycle?.gatewayInfo || `not connected to ${gatewayName}`), - hint: openshellConnected - ? undefined - : `run \`openshell gateway select ${gatewayName}\` and retry`, - }); - } - - if (shouldInspectLegacyGatewayContainer(sb)) { - checks.push( - ...dockerInspectGateway(`openshell-cluster-${gatewayName}`, { - namedGatewayConnected: openshellConnected, - gatewayName, - }), - ); - } - - if (openshellBin && openshellConnected) { - const list = captureOpenshell(["sandbox", "list"], { - ignoreError: true, - timeout: OPENSHELL_PROBE_TIMEOUT_MS, - }); - const liveNames = parseLiveSandboxNames(list.output || ""); - const present = list.status === 0 && liveNames.has(sandboxName); - const line = findSandboxListLine(list.output || "", sandboxName); - const ready = inferSandboxReadyFromLine(line); - sandboxReachable = present && ready === true; - checks.push({ - group: "Sandbox", - label: "Live sandbox", - status: present && ready === true ? "ok" : "fail", - detail: present - ? ready === true - ? `${sandboxName} present (Ready)` - : `${sandboxName} present${line ? ` (${oneLine(line)})` : ""}` - : `${sandboxName} not present in live OpenShell sandbox list`, - hint: present - ? ready === true - ? undefined - : `run \`${CLI_NAME} ${sandboxName} status\` or \`${CLI_NAME} ${sandboxName} logs --follow\`` - : `run \`${CLI_NAME} ${sandboxName} status\` or recreate with \`${CLI_NAME} onboard\``, - }); - } else if (openshellBin) { - checks.push({ - group: "Sandbox", - label: "Live sandbox", - status: "fail", - detail: "skipped because the nemoclaw gateway is not connected", - hint: "fix the gateway check above before trusting sandbox readiness", - }); - } - - const live = - openshellBin && openshellConnected - ? parseGatewayInference( - captureOpenshell(["inference", "get"], { - ignoreError: true, - timeout: OPENSHELL_PROBE_TIMEOUT_MS, - }).output, - ) - : null; - const currentModel = (live && live.model) || (sb && sb.model) || "unknown"; - const currentProvider = (live && live.provider) || (sb && sb.provider) || "unknown"; - checks.push({ - group: "Inference", - label: "Route", - status: currentProvider !== "unknown" || currentModel !== "unknown" ? "ok" : "warn", - detail: `${currentProvider} / ${currentModel}`, - hint: - currentProvider !== "unknown" || currentModel !== "unknown" - ? undefined - : `run \`${CLI_NAME} ${sandboxName} status\` after the gateway is healthy`, - }); - - if (typeof currentProvider === "string" && currentProvider !== "unknown") { - const inferenceHealth = probeProviderHealth(currentProvider); - if (!inferenceHealth) { - checks.push({ - group: "Inference", - label: "Provider health", - status: "info", - detail: `no health probe registered for ${currentProvider}`, - }); - } else { - // #3265 optional 3rd line — append gateway-chain probe for local - // providers so doctor sees the full path the agent uses. - if (currentProvider === "ollama-local" || currentProvider === "vllm-local") { - const gatewayChain = await probeSandboxInferenceGatewayHealth(sandboxName); - if (gatewayChain) { - inferenceHealth.subprobes = [ - ...(inferenceHealth.subprobes ?? []), - { - ok: gatewayChain.ok, - probed: true, - providerLabel: "Inference gateway chain", - endpoint: gatewayChain.endpoint, - detail: gatewayChain.detail, - probeLabel: "gateway", - ...(gatewayChain.ok ? {} : { failureLabel: "unreachable" as const }), - }, - ]; - } - } - pushInferenceHealthCheck(checks, inferenceHealth); - for (const sub of inferenceHealth.subprobes ?? []) { - pushInferenceHealthCheck(checks, sub); - } - } - } - - if (sb) { - try { - const versionCheck = sandboxVersion.checkAgentVersion(sandboxName); - const agent = agentRuntime.getSessionAgent(sandboxName); - const agentName = agentRuntime.getAgentDisplayName(agent); - if (versionCheck.isStale) { - checks.push({ - group: "Sandbox", - label: "Agent version", - status: "warn", - detail: `${agentName} v${versionCheck.sandboxVersion || "unknown"}; v${versionCheck.expectedVersion} available`, - hint: `run \`${CLI_NAME} ${sandboxName} rebuild\``, - }); - } else if (versionCheck.sandboxVersion) { - checks.push({ - group: "Sandbox", - label: "Agent version", - status: "ok", - detail: `${agentName} v${versionCheck.sandboxVersion}`, - }); - } else { - checks.push({ - group: "Sandbox", - label: "Agent version", - status: "info", - detail: "could not detect version", - }); - } - } catch { - checks.push({ - group: "Sandbox", - label: "Agent version", - status: "info", - detail: "version check unavailable", - }); - } - - const shieldsPosture = shields.getShieldsPosture(sandboxName, true); - const shieldsStatus: DoctorStatus = - shieldsPosture.mode === "locked" - ? "ok" - : shieldsPosture.mode === "temporarily_unlocked" || shieldsPosture.mode === "error" - ? "warn" - : "info"; - const shieldsHint = - shieldsPosture.mode === "mutable_default" - ? `run \`${CLI_NAME} ${sandboxName} shields up\` to opt into lockdown` - : shieldsPosture.mode === "locked" - ? undefined - : `run \`${CLI_NAME} ${sandboxName} shields status\` for details`; - checks.push({ - group: "Sandbox", - label: "Shields", - status: shieldsStatus, - detail: shieldsPosture.detail, - hint: shieldsHint, - }); - - // #4538: detect (and optionally repair with --fix) a mutable OpenClaw config - // tree that `openclaw doctor --fix` tightened from the NemoClaw contract - // (setgid + group-writable 2770/660) back to single-user 700/600. When that - // happens the gateway UID can no longer persist config edits. - const permsCheck = buildConfigPermsCheck(sandboxName, wantsFix, { - inspect: shields.inspectMutableConfigPerms, - repair: shields.repairMutableConfigPerms, - cliName: CLI_NAME, - }); - if (permsCheck) checks.push(permsCheck); - - checks.push(messagingDoctorCheck(sandboxName, sb)); - // #4156: bridge the gap between "configured" and "runtime-visible" — the - // existing messaging check above probes provider attachment, not whether - // OpenClaw's runtime config actually surfaces each enabled channel. - const registeredChannels = registry.getConfiguredMessagingChannelsFromEntry(sb); - const disabledChannelsSet = new Set(registry.getDisabledMessagingChannelsFromEntry(sb)); - const enabledChannels = registeredChannels.filter( - (channel: string) => !disabledChannelsSet.has(channel), - ); - const runtimeCheck = channelRuntimeDoctorCheck(sandboxName, enabledChannels); - if (runtimeCheck) checks.push(runtimeCheck); - } - - // #4616: surface (and, with --fix, repair) late OpenClaw dashboard/tool-call - // device-scope approvals. Dashboard-only users never run `connect`, so a - // pending tool-scope upgrade — visible as a gateway 1006 close, a "scope - // upgrade pending approval" error, and a loopback policy denial — has no - // recovery path. The probe is read-only; `--fix` runs the same narrow - // allowlisted approval pass that `connect` runs. Only run it when the sandbox - // is actually reachable so a stopped sandbox doesn't add noise, and only for - // OpenClaw — the `openclaw devices`/auto-pair scope-upgrade mechanism is - // OpenClaw-specific. Hermes (device_pairing: false) uses a different tool - // gateway, so probing it would emit an inaccurate OpenClaw-only check. - // Legacy registry entries with no recorded agent default to OpenClaw. - if (sb && sandboxReachable && (sb.agent ?? "openclaw") === "openclaw") { - const toolScopeChecks = buildToolScopeChecks(sandboxName, CLI_NAME, wantsFix, { - // OpenShell exec rejects multi-line args, so base64-wrap the probe payload. - exec: (name, script) => - executeSandboxCommandForVerification(name, wrapSandboxShellScript(script)), - runApprovalPass: (name) => { - const result = runSandboxAutoPairApprovalPass(name, { capture: true }); - return { reported: result.reported, approved: result.approved }; - }, - }); - for (const check of toolScopeChecks) checks.push(check); - } - - checks.push(ollamaDoctorCheck(currentProvider)); - checks.push(cloudflaredDoctorCheck(sandboxName)); - + const checks = await collectDoctorChecks(sandboxName, sb, gatewayName, intent); const report = buildDoctorReport(sandboxName, checks); - if (asJson && options.quietJson) return report; + if (intent.asJson && options.quietJson) return report; - const exitCode = renderDoctorReport(report, asJson); + const exitCode = renderDoctorReport(report, intent.asJson); if (exitCode !== 0) process.exit(exitCode); return undefined; }