diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index f77b64a90ea..960770c6aee 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -1894,12 +1894,18 @@ $$nemoclaw my-assistant channels start telegram Run messaging channel status checks. Without `--channel`, the command prints a compact summary for every configured channel, including registration, policy coverage, and non-secret rendered config comparisons. +For channels that support a live health probe (WhatsApp, Telegram), the summary adds a `Runtime health: not checked in summary view` pointer instead of running the probe, so it never reads as healthy without an explicit check. With `--channel`, it prints the detailed status for that channel. For WhatsApp, `--channel whatsapp` also probes the sandbox to separately report pairing/session state, the Noise WebSocket connection, inbound event delivery, and policy coverage. A paired channel with no observed inbound delivery exits non-zero with verdict `idle` so an unhealthy bridge cannot pass as healthy. The detailed WhatsApp probe stays focused on QR/session runtime diagnostics and does not include rendered-config comparison lines. +For Telegram, `--channel telegram` probes the sandbox to report the gateway process, Bot API reachability, and inbound delivery alongside the config comparison, and classifies the current state into a verdict such as `healthy`, `idle`, `unreachable` (network or egress), `token_rejected`, or `not_started`. +It reads the gateway's own startup and poll log breadcrumbs rather than issuing its own Bot API request, so the resolved bot token never leaves the gateway. +The verdict reflects the most recent evidence in the log window, so a bridge that recovered after a blocked start reports `healthy` while one blocked again reports `unreachable`. +Telegram health is probed only for OpenClaw sandboxes; a Hermes Telegram sandbox uses the basic config report. + For registered non-WhatsApp channel details and the compact summary, the status output compares non-secret config inputs from the sandbox registry against the values rendered into the agent config, such as Telegram group policy in `openclaw.json` or mention mode in Hermes config. Secret inputs, including tokens, are not printed. If the registry contains a non-secret expected value but NemoClaw cannot read or check the rendered source, the comparison is a warning and the detail includes `(not checked)`. @@ -1908,14 +1914,15 @@ Optional unset inputs remain informational. ```bash $$nemoclaw my-assistant channels status $$nemoclaw my-assistant channels status --channel whatsapp +$$nemoclaw my-assistant channels status --channel telegram ``` | Flag | Description | |------|-------------| | `--channel ` | Channel to inspect in detail | -| `--json` | Emit the status report as JSON (for the detailed WhatsApp probe, exit non-zero when the verdict is not `healthy` or `unknown`) | +| `--json` | Emit the status report as JSON (for the detailed WhatsApp and Telegram probes, exit non-zero when the verdict is not `healthy` or `unknown`) | -The probe is bounded by an in-sandbox `openshell sandbox exec` with a hard timeout, captures only short matched bridge log signals (e.g. `connection.open`, `401 unauthorized`, `qr expired`), and never forwards message bodies to the host diagnostic output. +Each probe is bounded by an in-sandbox `openshell sandbox exec` with a hard timeout and returns only matched bridge/gateway log lines (e.g. `connection.open`, `401 unauthorized`, `qr expired`, or `[telegram]` startup breadcrumbs) to the host, where NemoClaw reduces them to fixed classifications; the raw lines are never rendered, so the diagnostic output carries only those classifications, never message bodies or tokens. diff --git a/src/lib/actions/sandbox/channel-status-config-core.test.ts b/src/lib/actions/sandbox/channel-status-config-core.test.ts index fcbcbaade13..7bdc5184cfd 100644 --- a/src/lib/actions/sandbox/channel-status-config-core.test.ts +++ b/src/lib/actions/sandbox/channel-status-config-core.test.ts @@ -2,12 +2,18 @@ // SPDX-License-Identifier: Apache-2.0 import { describe, expect, it } from "vitest"; -import { entry, makeDeps, showSandboxChannelStatus } from "./channel-status.test-helpers"; +import { + entry, + makeDeps, + reportSignals, + showSandboxChannelStatus, + withTelegramProbe, +} from "./channel-status.test-helpers"; describe("showSandboxChannelStatus config comparison", () => { it("marks rendered config ok when the sandbox config matches the sandbox entry", async () => { const { deps, out_lines } = makeDeps({ - exec: (_sandbox, command) => + exec: withTelegramProbe((_sandbox, command) => command.includes("/sandbox/.openclaw/openclaw.json") ? { status: 0, @@ -30,6 +36,7 @@ describe("showSandboxChannelStatus config comparison", () => { stderr: "", } : { status: 1, stdout: "", stderr: "" }, + ), sandbox: entry(["telegram"], [], { telegram: [ { @@ -61,14 +68,15 @@ describe("showSandboxChannelStatus config comparison", () => { ], }), appliedPresets: ["telegram"], + gatewayPresets: ["telegram"], }); const result = await showSandboxChannelStatus("alpha", { deps, channel: "telegram", }); - expect(result && "verdict" in result && result.verdict).toBe("info"); - const signals = result && "signals" in result ? result.signals : []; + expect(result && "report" in result && result.report.verdict).toBe("unknown"); + const signals = reportSignals(result); expect( signals.find((signal) => signal.label === "Telegram group policy (TELEGRAM_GROUP_POLICY)"), ).toMatchObject({ @@ -92,7 +100,7 @@ describe("showSandboxChannelStatus config comparison", () => { it("marks Telegram all-message mode ok when OpenClaw omits the groups stanza (#5691)", async () => { const { deps } = makeDeps({ - exec: () => ({ + exec: withTelegramProbe(() => ({ status: 0, stdout: JSON.stringify({ channels: { @@ -106,7 +114,7 @@ describe("showSandboxChannelStatus config comparison", () => { }, }), stderr: "", - }), + })), sandbox: entry(["telegram"], [], { telegram: [ { @@ -130,13 +138,14 @@ describe("showSandboxChannelStatus config comparison", () => { ], }), appliedPresets: ["telegram"], + gatewayPresets: ["telegram"], }); const result = await showSandboxChannelStatus("alpha", { deps, channel: "telegram", }); - const signals = result && "signals" in result ? result.signals : []; + const signals = reportSignals(result); expect( signals.find( (signal) => signal.label === "Telegram group mention mode (TELEGRAM_REQUIRE_MENTION)", @@ -149,7 +158,7 @@ describe("showSandboxChannelStatus config comparison", () => { it("does not compare Hermes Telegram group policy when the manifest does not render it", async () => { const { deps } = makeDeps({ - exec: (_sandbox, command) => + exec: withTelegramProbe((_sandbox, command) => command.includes("/sandbox/.hermes/.env") ? { status: 0, @@ -167,6 +176,7 @@ describe("showSandboxChannelStatus config comparison", () => { stdout: "", stderr: "", }, + ), agentName: "hermes", sandbox: entry( ["telegram"], @@ -205,13 +215,14 @@ describe("showSandboxChannelStatus config comparison", () => { "hermes", ), appliedPresets: ["telegram"], + gatewayPresets: ["telegram"], }); const result = await showSandboxChannelStatus("alpha", { deps, channel: "telegram", }); - const signals = result && "signals" in result ? result.signals : []; + const signals = reportSignals(result); expect( signals.find( (signal) => signal.label === "Telegram User ID (for DM access) (TELEGRAM_ALLOWED_IDS)", @@ -260,7 +271,7 @@ describe("showSandboxChannelStatus config comparison", () => { channel: "teams", }); - const signals = result && "signals" in result ? result.signals : []; + const signals = reportSignals(result); expect(signals.filter((signal) => signal.label === "Rendered config source")).toEqual([ expect.objectContaining({ severity: "warn", @@ -278,7 +289,7 @@ describe("showSandboxChannelStatus config comparison", () => { it("warns when rendered config differs from the sandbox entry", async () => { const { deps } = makeDeps({ - exec: () => ({ + exec: withTelegramProbe(() => ({ status: 0, stdout: JSON.stringify({ channels: { @@ -292,7 +303,7 @@ describe("showSandboxChannelStatus config comparison", () => { }, }), stderr: "", - }), + })), sandbox: entry(["telegram"], [], { telegram: [ { @@ -307,13 +318,14 @@ describe("showSandboxChannelStatus config comparison", () => { ], }), appliedPresets: ["telegram"], + gatewayPresets: ["telegram"], }); const result = await showSandboxChannelStatus("alpha", { deps, channel: "telegram", }); - const signals = result && "signals" in result ? result.signals : []; + const signals = reportSignals(result); expect( signals.find((signal) => signal.label === "Telegram group policy (TELEGRAM_GROUP_POLICY)"), ).toMatchObject({ @@ -385,7 +397,7 @@ describe("showSandboxChannelStatus config comparison", () => { channel: "teams", }); - const signals = result && "signals" in result ? result.signals : []; + const signals = reportSignals(result); const sourceWarnings = signals.filter((signal) => signal.label === "Rendered config source"); expect(sourceWarnings).toHaveLength(1); expect(sourceWarnings[0]).toMatchObject({ @@ -458,7 +470,7 @@ describe("showSandboxChannelStatus config comparison", () => { channel: "teams", }); - const signals = result && "signals" in result ? result.signals : []; + const signals = reportSignals(result); expect(signals.filter((signal) => signal.label === "Rendered config source")).toEqual([ expect.objectContaining({ severity: "warn", @@ -506,7 +518,7 @@ describe("showSandboxChannelStatus config comparison", () => { channel: "teams", }); - const signals = result && "signals" in result ? result.signals : []; + const signals = reportSignals(result); expect( signals.find( (signal) => signal.label === "Microsoft Teams mention mode (TEAMS_REQUIRE_MENTION)", @@ -550,7 +562,7 @@ describe("showSandboxChannelStatus config comparison", () => { channel: "teams", }); - const signals = result && "signals" in result ? result.signals : []; + const signals = reportSignals(result); expect( signals.find( (signal) => diff --git a/src/lib/actions/sandbox/channel-status-summary.test.ts b/src/lib/actions/sandbox/channel-status-summary.test.ts index 867addc2dc4..2586839da51 100644 --- a/src/lib/actions/sandbox/channel-status-summary.test.ts +++ b/src/lib/actions/sandbox/channel-status-summary.test.ts @@ -2,7 +2,12 @@ // SPDX-License-Identifier: Apache-2.0 import { describe, expect, it, vi } from "vitest"; -import { entry, makeDeps, showSandboxChannelStatus } from "./channel-status.test-helpers"; +import { + entry, + makeDeps, + showSandboxChannelStatus, + withTelegramProbe, +} from "./channel-status.test-helpers"; describe("showSandboxChannelStatus summary", () => { it("emits a compact all-channel report when no channel is selected", async () => { @@ -102,19 +107,39 @@ describe("showSandboxChannelStatus summary", () => { expect(dump).not.toMatch(/NemoClaw channels status:/); }); - it("emits a basic per-channel report for non-whatsapp channels", async () => { + it("runs the telegram health probe for an explicit --channel telegram", async () => { const { deps, out_lines } = makeDeps({ - exec: () => ({ status: 0, stdout: "", stderr: "" }), + exec: withTelegramProbe(() => ({ status: 0, stdout: "", stderr: "" })), sandbox: entry(["telegram"]), appliedPresets: ["telegram"], + gatewayPresets: ["telegram"], }); const result = await showSandboxChannelStatus("alpha", { deps, channel: "telegram", }); - expect(result && "verdict" in result && result.verdict).toBe("info"); + expect(result && "report" in result && result.report.verdict).toBe("unknown"); const dump = out_lines.join("\n"); - expect(dump).toMatch(/telegram registered/); - expect(dump).toMatch(/preset applied/); + expect(dump).toMatch(/telegram channel registered/); + expect(dump).toMatch(/telegram preset applied/); + }); + + it("uses the basic config report for a Hermes telegram sandbox (no OpenClaw producer)", async () => { + const { deps } = makeDeps({ + exec: () => ({ status: 0, stdout: "", stderr: "" }), + sandbox: entry(["telegram"], [], {}, "hermes"), + agentName: "hermes", + appliedPresets: ["telegram"], + gatewayPresets: ["telegram"], + }); + const result = await showSandboxChannelStatus("alpha", { + deps, + channel: "telegram", + }); + // Hermes lacks the OpenClaw log-tail breadcrumb producer, so it falls back + // to the basic { verdict: "info" } report instead of a misleading health + // verdict (#6743 review). + expect(result && "verdict" in result && result.verdict).toBe("info"); + expect(result && "report" in result).toBe(false); }); }); diff --git a/src/lib/actions/sandbox/channel-status-telegram-policy.test.ts b/src/lib/actions/sandbox/channel-status-telegram-policy.test.ts index a4ee248837f..106b753cb01 100644 --- a/src/lib/actions/sandbox/channel-status-telegram-policy.test.ts +++ b/src/lib/actions/sandbox/channel-status-telegram-policy.test.ts @@ -1,13 +1,19 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, expect, it } from "vitest"; -import { entry, makeDeps, showSandboxChannelStatus } from "./channel-status.test-helpers"; +import { describe, expect, it, vi } from "vitest"; +import { + entry, + makeDeps, + reportSignals, + showSandboxChannelStatus, + withTelegramProbe, +} from "./channel-status.test-helpers"; describe("showSandboxChannelStatus Telegram group policy", () => { it("uses manifest defaults when no stored config value exists", async () => { const { deps, out_lines } = makeDeps({ - exec: () => ({ + exec: withTelegramProbe(() => ({ status: 0, stdout: JSON.stringify({ channels: { @@ -26,16 +32,17 @@ describe("showSandboxChannelStatus Telegram group policy", () => { }, }), stderr: "", - }), + })), sandbox: entry(["telegram"]), appliedPresets: ["telegram"], + gatewayPresets: ["telegram"], }); const result = await showSandboxChannelStatus("alpha", { deps, channel: "telegram", }); - const signals = result && "signals" in result ? result.signals : []; + const signals = reportSignals(result); expect( signals.find((signal) => signal.label === "Telegram group policy (TELEGRAM_GROUP_POLICY)"), ).toMatchObject({ @@ -60,7 +67,7 @@ describe("showSandboxChannelStatus Telegram group policy", () => { it("accepts Telegram disabled group policy from rendered config", async () => { const { deps } = makeDeps({ - exec: () => ({ + exec: withTelegramProbe(() => ({ status: 0, stdout: JSON.stringify({ channels: { @@ -74,7 +81,7 @@ describe("showSandboxChannelStatus Telegram group policy", () => { }, }), stderr: "", - }), + })), sandbox: entry(["telegram"], [], { telegram: [ { @@ -89,13 +96,14 @@ describe("showSandboxChannelStatus Telegram group policy", () => { ], }), appliedPresets: ["telegram"], + gatewayPresets: ["telegram"], }); const result = await showSandboxChannelStatus("alpha", { deps, channel: "telegram", }); - const signals = result && "signals" in result ? result.signals : []; + const signals = reportSignals(result); expect( signals.find((signal) => signal.label === "Telegram group policy (TELEGRAM_GROUP_POLICY)"), ).toMatchObject({ @@ -104,3 +112,38 @@ describe("showSandboxChannelStatus Telegram group policy", () => { }); }); }); + +describe("showSandboxChannelStatus Telegram health exit propagation", () => { + it("exits non-zero in text mode for an unhealthy (unreachable) telegram probe (#6743)", async () => { + // The whole point of the probe is a non-zero exit on an unhealthy channel so + // automation cannot treat a failed health check as success. Drive an + // `unreachable` verdict end-to-end and assert the command exits 1. + const unreachableProbe = [ + "NEMOCLAW_TG_DIAG_OK", + "NEMOCLAW_TG_LOG_BEGIN", + "[telegram] [default] Bot API startup probe failed: ETIMEDOUT", + "NEMOCLAW_TG_LOG_END", + "PROC 42 node /opt/openclaw gateway", + "NEMOCLAW_TG_PROC_DONE", + ].join("\n"); + const exitSpy = vi.spyOn(process, "exit").mockImplementation(((code?: number) => { + throw new Error(`process.exit(${code})`); + }) as never); + const { deps, out_lines } = makeDeps({ + exec: withTelegramProbe(() => ({ status: 0, stdout: "{}", stderr: "" }), unreachableProbe), + sandbox: entry(["telegram"]), + appliedPresets: ["telegram"], + gatewayPresets: ["telegram"], + }); + let threw: Error | null = null; + try { + await showSandboxChannelStatus("alpha", { deps, channel: "telegram" }); + } catch (err) { + threw = err as Error; + } finally { + exitSpy.mockRestore(); + } + expect(threw?.message).toBe("process.exit(1)"); + expect(out_lines.join("\n")).toMatch(/Verdict:.*unreachable/); + }); +}); diff --git a/src/lib/actions/sandbox/channel-status.test-helpers.ts b/src/lib/actions/sandbox/channel-status.test-helpers.ts index 330c8e23d66..fa126c725dc 100644 --- a/src/lib/actions/sandbox/channel-status.test-helpers.ts +++ b/src/lib/actions/sandbox/channel-status.test-helpers.ts @@ -40,6 +40,7 @@ vi.mock("./process-recovery", () => ({ import type { AgentDefinition } from "../../agent/defs"; import type { SandboxMessagingInputReference } from "../../messaging/manifest"; +import type { DiagnosticSignal } from "../../messaging/channels/channel-health"; import type { SandboxEntry } from "../../state/registry"; type ShowSandboxChannelStatus = typeof import("./channel-status").showSandboxChannelStatus; @@ -190,3 +191,39 @@ export function makeDeps(opts: { out_lines: calls, }; } + +// A telegram log-tail probe stdout that yields the "unknown" verdict +// (reachable + gateway process alive + no conclusive breadcrumbs) so a +// config-focused telegram test does not trip the health exit code. Pair it +// with `gatewayPresets: ["telegram"]` so the policy signal is not a gap. +export const TELEGRAM_PROBE_UNKNOWN_STDOUT = [ + "NEMOCLAW_TG_DIAG_OK", + "NEMOCLAW_TG_LOG_BEGIN", + "NEMOCLAW_TG_LOG_END", + "PROC 42 node /opt/openclaw gateway", + "NEMOCLAW_TG_PROC_DONE", +].join("\n"); + +// Wrap a config-read exec so the telegram log-tail probe command (which tails +// /tmp/gateway.log) returns a benign probe response instead of the config +// payload. Everything else falls through to the provided config exec. +export function withTelegramProbe( + configExec: (sandboxName: string, command: string, timeoutMs?: number) => ExecResult | null, + probeStdout: string = TELEGRAM_PROBE_UNKNOWN_STDOUT, +): (sandboxName: string, command: string, timeoutMs?: number) => ExecResult | null { + return (sandboxName, command, timeoutMs) => + command.includes("/tmp/gateway.log") + ? { status: 0, stdout: probeStdout, stderr: "" } + : configExec(sandboxName, command, timeoutMs); +} + +// Read signals from either channel-status report shape: the basic +// `{ verdict, signals }` report or the deep `{ report: { signals } }` report. +export function reportSignals( + result: Awaited>, +): DiagnosticSignal[] { + if (!result) return []; + if ("signals" in result) return result.signals; + if ("report" in result) return result.report.signals; + return []; +} diff --git a/src/lib/actions/sandbox/channel-status.test.ts b/src/lib/actions/sandbox/channel-status.test.ts index 1f045cd87e3..063595a7510 100644 --- a/src/lib/actions/sandbox/channel-status.test.ts +++ b/src/lib/actions/sandbox/channel-status.test.ts @@ -322,5 +322,37 @@ describe("showSandboxChannelStatus (whatsapp)", () => { expect(result && "verdict" in result && result.verdict).toBe("info"); const dump = out_lines.join("\n"); expect(dump).toMatch(/registered but currently paused/); + // The paused fallback must not claim it is the summary view nor tell the + // operator to rerun the --channel command they are already running (#6887). + const runtime = + result && "signals" in result + ? result.signals.find((s) => s.label === "Runtime health") + : undefined; + expect(runtime?.detail).toBe("not checked — whatsapp is currently paused"); + expect(runtime?.hint).toBeUndefined(); + }); + + it("labels a paused telegram channel as paused rather than summary view under --channel (#6887)", async () => { + // A probe-capable channel that is paused lands on the basic report even + // under an explicit --channel request, since the probe is gated on + // !channelIsPaused. The Runtime health signal must reflect the paused state. + const execSpy = vi.fn(() => ({ status: 0, stdout: "", stderr: "" })); + const { deps } = makeDeps({ + exec: () => ({ status: 0, stdout: "", stderr: "" }), + sandbox: entry(["telegram"], ["telegram"]), + }); + deps.execSandbox = execSpy as unknown as typeof deps.execSandbox; + const result = await showSandboxChannelStatus("alpha", { deps, channel: "telegram" }); + // The config-value read still runs, but the deep gateway-log probe must not. + const probeCommands = execSpy.mock.calls + .map((call) => String((call as unknown[])[1])) + .join("\n"); + expect(probeCommands).not.toMatch(/gateway\.log|pgrep/); + const runtime = + result && "signals" in result + ? result.signals.find((s) => s.label === "Runtime health") + : undefined; + expect(runtime?.detail).toBe("not checked — telegram is currently paused"); + expect(runtime?.hint).toBeUndefined(); }); }); diff --git a/src/lib/actions/sandbox/channel-status.ts b/src/lib/actions/sandbox/channel-status.ts index 853f8d65d79..8e063a0dae9 100644 --- a/src/lib/actions/sandbox/channel-status.ts +++ b/src/lib/actions/sandbox/channel-status.ts @@ -19,10 +19,19 @@ import { createBuiltInChannelManifestRegistry, getMessagingManifestAvailabilityContext, } from "../../messaging"; +import { + type ChannelHealthReport, + channelHealthProbeInputs, +} from "../../messaging/channels/channel-health"; import { collectBuiltInMessagingChannelDiagnostics, type MessagingChannelDiagnosticSpec, } from "../../messaging/diagnostics"; +import { createBuiltInMessagingHookRegistry } from "../../messaging/hooks"; +import { + readChannelHealthOutputs, + runMessagingStatusHooks, +} from "../../messaging/hooks/status-runner"; import * as policies from "../../policy"; import { type DiagnosticSeverity, @@ -77,7 +86,12 @@ export type ChannelStatusOptions = { }; type ChannelStatusSingleReport = - | { schemaVersion: 1; sandbox: string; channel: string; report: WhatsappDiagnosticReport } + | { + schemaVersion: 1; + sandbox: string; + channel: string; + report: WhatsappDiagnosticReport | ChannelHealthReport; + } | { schemaVersion: 1; sandbox: string; @@ -493,7 +507,7 @@ function buildBasicChannelReport( agent: AgentDefinition, deps: Required, diagnostic: MessagingChannelDiagnosticSpec, - options: { readonly includeDeepDiagnostics?: boolean } = {}, + options: { readonly includeDeepDiagnostics?: boolean; readonly channelPaused?: boolean } = {}, ): ChannelStatusSingleReport { const entry = deps.getSandbox(sandboxName); const enabled = registry.getConfiguredMessagingChannelsFromEntry(entry).includes(channelName); @@ -529,7 +543,21 @@ function buildBasicChannelReport( if (enabled) { signals.push(...buildConfigStatusSignals(sandboxName, channelName, entry, agent, deps)); } - if (options.includeDeepDiagnostics ?? true) { + if (diagnostic.deepProbe !== undefined) { + // Channel has a deep probe this path does not run: the summary view never + // runs it, and a paused channel is not probed in detail view. Say so instead + // of leaving a silent all-[ok] that reads as healthy (#6743). + signals.push({ + label: "Runtime health", + severity: "info", + detail: options.channelPaused + ? `not checked — ${channelName} is currently paused` + : "not checked in summary view", + hint: options.channelPaused + ? undefined + : `run \`${CLI_NAME} ${sandboxName} channels status --channel ${channelName}\` to probe live health`, + }); + } else if (options.includeDeepDiagnostics ?? true) { signals.push({ label: "Deep diagnostics", severity: "info", @@ -579,6 +607,58 @@ function channelSupportedByAgent(channelName: string, agent: AgentDefinition): b .some((manifest) => manifest.id === channelName); } +// Runs a `log-tail` deep-probe channel's `phase:"status"` health hook through +// the generic status-hook runner and returns its channel-health report. All +// channel-specific probing + classification lives in the channel's own hook +// (e.g. channels/telegram/hooks/status-health.ts); this stays channel-agnostic. +// The hook's own `agents` gate skips channels with no breadcrumb producer for +// the requested agent (e.g. Hermes), so the caller falls back to the basic +// report when no health output is returned. +function runChannelHealthHook( + sandboxName: string, + channelName: string, + agent: AgentDefinition, + deps: Required, + diagnostic: MessagingChannelDiagnosticSpec, +): ChannelHealthReport | undefined { + const entry = deps.getSandbox(sandboxName); + const channelEnabledInRegistry = registry + .getConfiguredMessagingChannelsFromEntry(entry) + .includes(channelName); + const policyPresets = + diagnostic.policyPresets.length > 0 ? diagnostic.policyPresets : [channelName]; + const appliedPresets = deps.getAppliedPresets(sandboxName); + const presetInRegistry = policyPresets.some((preset) => appliedPresets.includes(preset)); + let presetOnGateway: boolean | null = null; + try { + const gatewayPresets = deps.getGatewayPresets(sandboxName); + presetOnGateway = + gatewayPresets === null + ? null + : policyPresets.some((preset) => gatewayPresets.includes(preset)); + } catch { + presetOnGateway = null; + } + + const results = runMessagingStatusHooks({ + agent: agent.name === "hermes" ? "hermes" : "openclaw", + channels: new Set([channelName]), + currentSandbox: sandboxName, + hookRegistry: createBuiltInMessagingHookRegistry({ + statusHealth: { executeSandboxCommand: deps.execSandbox }, + }), + extraInputs: channelHealthProbeInputs({ + currentSandbox: sandboxName, + agent: agent.name, + probedAt: deps.now().toISOString(), + channelEnabledInRegistry, + presetInRegistry, + presetOnGateway, + }), + }); + return results.flatMap(readChannelHealthOutputs)[0]; +} + /** * Run the WhatsApp diagnostic or a thin per-channel summary for the named * sandbox. The function never throws: any unexpected condition is rendered @@ -653,6 +733,14 @@ export async function showSandboxChannelStatus( const disabledChannels = new Set(registry.getDisabledMessagingChannelsFromEntry(entry)); const channelIsPaused = disabledChannels.has(channelName); + // A `log-tail` deep-probe channel runs its `phase:"status"` health hook via + // the generic status-hook runner (the hook lives in the channel folder). The + // hook's `agents` gate skips channels with no breadcrumb producer for this + // agent (e.g. Hermes telegram), so those fall back to the basic config report. + const healthReport = + diagnostic.deepProbe === "log-tail" && !channelIsPaused + ? runChannelHealthHook(sandboxName, channelName, agent, deps, diagnostic) + : undefined; let report: ChannelStatusReport; if (diagnostic.deepProbe === "in-sandbox-qr" && !channelIsPaused) { const input = buildWhatsappProbeInput(sandboxName, agent, deps); @@ -663,8 +751,27 @@ export async function showSandboxChannelStatus( channel: channelName, report: whatsappReport, }; + } else if (healthReport) { + // Append the config-value signals (#5691/#5695: group policy, mention mode, + // allowed IDs) the basic report shows, so `--channel ` reports both the + // channel config and live runtime health. + const configSignals = buildConfigStatusSignals( + sandboxName, + channelName, + deps.getSandbox(sandboxName), + agent, + deps, + ); + report = { + schemaVersion: 1, + sandbox: sandboxName, + channel: channelName, + report: { ...healthReport, signals: [...healthReport.signals, ...configSignals] }, + }; } else { - report = buildBasicChannelReport(sandboxName, channelName, agent, deps, diagnostic); + report = buildBasicChannelReport(sandboxName, channelName, agent, deps, diagnostic, { + channelPaused: channelIsPaused, + }); } if (!(asJson && quietJson)) { diff --git a/src/lib/messaging/channels/channel-health.ts b/src/lib/messaging/channels/channel-health.ts new file mode 100644 index 00000000000..0b99e4d10bd --- /dev/null +++ b/src/lib/messaging/channels/channel-health.ts @@ -0,0 +1,93 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Shared shapes for channel runtime-health status hooks. + * + * A channel that can be probed at `channels status` time declares a + * `phase: "status"` hook whose handler returns a `messaging-channel-health` + * status output carrying a {@link ChannelHealthReport}. The generic status + * command renders that report without importing any per-channel code, so + * channel-specific probing/classification stays inside the channel folder. + */ + +import type { MessagingSerializableValue } from "../manifest"; + +export type DiagnosticSeverity = "ok" | "warn" | "fail" | "info"; + +export type DiagnosticSignal = { + label: string; + severity: DiagnosticSeverity; + detail: string; + hint?: string; +}; + +/** + * Structured runtime-health verdict a status hook emits. `verdict` is a plain + * string so the generic renderer stays channel-agnostic; each channel narrows + * it to its own union internally. + */ +export type ChannelHealthReport = { + schemaVersion: 1; + channel: string; + agent: string; + verdict: string; + probedAt: string; + signals: DiagnosticSignal[]; + hints: string[]; +}; + +/** `kind: "status"` output `value.type` carrying a {@link ChannelHealthReport}. */ +export const MESSAGING_CHANNEL_HEALTH_OUTPUT_TYPE = "messaging-channel-health"; + +export interface ChannelHealthCommandResult { + readonly status?: number | null; + readonly stdout?: unknown; + readonly stderr?: unknown; +} + +/** Runs one command inside a named sandbox; used by channel-health status hooks. */ +export type ChannelHealthCommandRunner = ( + sandboxName: string, + command: string, + timeoutMs: number, +) => ChannelHealthCommandResult | null | undefined; + +/** + * Options a channel-health status hook accepts. `executeSandboxCommand` is a + * host capability threaded once (top-level) into every channel's health hook — + * the way `openclawBridgeHealth` is threaded — so the generic status command + * never names a specific channel to enable probing. + */ +export interface ChannelStatusHealthHookOptions { + readonly executeSandboxCommand?: ChannelHealthCommandRunner; + readonly timeoutMs?: number; +} + +/** + * Host-side facts the generic status command computes per channel and passes as + * status-hook inputs. Generic — a channel-health probe hook reads these keys + * rather than the orchestrator embedding channel-specific probing. + */ +export interface ChannelHealthProbeFacts { + readonly currentSandbox: string; + readonly agent: string; + readonly probedAt: string; + readonly channelEnabledInRegistry: boolean; + readonly presetInRegistry: boolean; + readonly presetOnGateway: boolean | null; +} + +/** Serializable status-hook input map built from {@link ChannelHealthProbeFacts}. */ +export function channelHealthProbeInputs( + facts: ChannelHealthProbeFacts, +): Record { + return { + currentSandbox: facts.currentSandbox, + agent: facts.agent, + probedAt: facts.probedAt, + channelEnabledInRegistry: facts.channelEnabledInRegistry, + presetInRegistry: facts.presetInRegistry, + presetOnGateway: facts.presetOnGateway, + }; +} diff --git a/src/lib/messaging/channels/telegram/hooks/get-me-reachability.ts b/src/lib/messaging/channels/telegram/hooks/get-me-reachability.ts index 3827357ec5e..aa4aef13670 100644 --- a/src/lib/messaging/channels/telegram/hooks/get-me-reachability.ts +++ b/src/lib/messaging/channels/telegram/hooks/get-me-reachability.ts @@ -16,6 +16,10 @@ import { createTelegramOpenClawBridgeHealthHookRegistration, type OpenClawBridgeHealthHookOptions, } from "./openclaw-bridge-health"; +import { + createTelegramStatusHealthHookRegistration, + type TelegramStatusHealthHookOptions, +} from "./status-health"; export const TELEGRAM_GET_ME_REACHABILITY_HOOK_ID = "telegram.getMeReachability"; const DEFAULT_TELEGRAM_REACHABILITY_TIMEOUT_MS = 10_000; @@ -47,6 +51,7 @@ export interface TelegramGetMeReachabilityHookOptions extends TelegramAllowlistA export interface TelegramHookOptions extends TelegramGetMeReachabilityHookOptions { readonly openclawBridgeHealth?: OpenClawBridgeHealthHookOptions; readonly gatewayConflictStatus?: TelegramGatewayConflictStatusHookOptions; + readonly statusHealth?: TelegramStatusHealthHookOptions; } export function createTelegramGetMeReachabilityHook( @@ -104,6 +109,7 @@ export function createTelegramHookRegistrations( createTelegramAllowlistAliasesHookRegistration(options), createTelegramOpenClawBridgeHealthHookRegistration(options.openclawBridgeHealth), createTelegramGatewayConflictStatusHookRegistration(options.gatewayConflictStatus), + createTelegramStatusHealthHookRegistration(options.statusHealth), { id: TELEGRAM_GET_ME_REACHABILITY_HOOK_ID, handler: createTelegramGetMeReachabilityHook(options), diff --git a/src/lib/messaging/channels/telegram/hooks/index.ts b/src/lib/messaging/channels/telegram/hooks/index.ts index ff7be83ebe2..905cd20f354 100644 --- a/src/lib/messaging/channels/telegram/hooks/index.ts +++ b/src/lib/messaging/channels/telegram/hooks/index.ts @@ -5,3 +5,5 @@ export * from "./allowlist-aliases"; export * from "./gateway-conflict-status"; export * from "./get-me-reachability"; export * from "./openclaw-bridge-health"; +export * from "./status-health"; +export * from "./status-health-eval"; diff --git a/src/lib/messaging/channels/telegram/hooks/status-health-eval.test.ts b/src/lib/messaging/channels/telegram/hooks/status-health-eval.test.ts new file mode 100644 index 00000000000..5657fcabfd6 --- /dev/null +++ b/src/lib/messaging/channels/telegram/hooks/status-health-eval.test.ts @@ -0,0 +1,320 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; +import { + evaluateTelegramDiagnostics, + parseTelegramBreadcrumbs, + type TelegramBreadcrumbs, + type TelegramProbeInput, +} from "./status-health-eval"; + +function baseInput(overrides: Partial = {}): TelegramProbeInput { + return { + agent: "openclaw", + probeReachable: true, + gatewayProcessAlive: true, + breadcrumbs: null, + probedAt: "2026-07-14T00:00:00.000Z", + presetInRegistry: true, + presetOnGateway: true, + channelEnabledInRegistry: true, + ...overrides, + }; +} + +function breadcrumbs(overrides: Partial = {}): TelegramBreadcrumbs { + return { + providerReady: false, + tokenRejected: false, + credentialUnresolved: false, + startupFailedNetwork: false, + startupHttpError: null, + bridgeNotStarted: false, + inboundReceived: false, + ...overrides, + }; +} + +describe("evaluateTelegramDiagnostics verdict", () => { + it("reports healthy when the provider is ready and inbound was observed (#6743)", () => { + const report = evaluateTelegramDiagnostics( + baseInput({ breadcrumbs: breadcrumbs({ providerReady: true, inboundReceived: true }) }), + ); + expect(report.verdict).toBe("healthy"); + }); + + it("reports idle when ready but no inbound was observed (#6743)", () => { + const report = evaluateTelegramDiagnostics( + baseInput({ breadcrumbs: breadcrumbs({ providerReady: true }) }), + ); + expect(report.verdict).toBe("idle"); + }); + + it("distinguishes a rejected token from a network failure (#6743)", () => { + const rejected = evaluateTelegramDiagnostics( + baseInput({ breadcrumbs: breadcrumbs({ tokenRejected: true }) }), + ); + expect(rejected.verdict).toBe("token_rejected"); + + const credential = evaluateTelegramDiagnostics( + baseInput({ breadcrumbs: breadcrumbs({ credentialUnresolved: true }) }), + ); + expect(credential.verdict).toBe("token_rejected"); + + const unreachable = evaluateTelegramDiagnostics( + baseInput({ breadcrumbs: breadcrumbs({ startupFailedNetwork: true }) }), + ); + expect(unreachable.verdict).toBe("unreachable"); + expect( + unreachable.signals.some((s) => s.label === "Bot API reachability" && s.severity === "fail"), + ).toBe(true); + }); + + it("reports not_started when the gateway process is dead or the bridge never started", () => { + expect(evaluateTelegramDiagnostics(baseInput({ gatewayProcessAlive: false })).verdict).toBe( + "not_started", + ); + expect( + evaluateTelegramDiagnostics( + baseInput({ breadcrumbs: breadcrumbs({ bridgeNotStarted: true }) }), + ).verdict, + ).toBe("not_started"); + }); + + it("reports config_gap / policy_gap before any runtime verdict", () => { + expect( + evaluateTelegramDiagnostics(baseInput({ channelEnabledInRegistry: false })).verdict, + ).toBe("config_gap"); + expect(evaluateTelegramDiagnostics(baseInput({ presetInRegistry: false })).verdict).toBe( + "policy_gap", + ); + }); + + it("reports probe_failed when the sandbox could not be reached", () => { + const report = evaluateTelegramDiagnostics( + baseInput({ probeReachable: false, gatewayProcessAlive: null, breadcrumbs: null }), + ); + expect(report.verdict).toBe("probe_failed"); + }); + + it("reports unknown when reachable but no conclusive startup breadcrumb", () => { + const report = evaluateTelegramDiagnostics(baseInput({ breadcrumbs: breadcrumbs() })); + expect(report.verdict).toBe("unknown"); + }); + + it("never claims healthy while a runtime signal fails", () => { + const report = evaluateTelegramDiagnostics( + baseInput({ breadcrumbs: breadcrumbs({ providerReady: true, tokenRejected: true }) }), + ); + expect(report.verdict).toBe("token_rejected"); + }); + + it("reports unreachable (not not_started) when the bridge failed on a network error (#6743)", () => { + const report = evaluateTelegramDiagnostics( + baseInput({ + breadcrumbs: breadcrumbs({ startupFailedNetwork: true, bridgeNotStarted: true }), + }), + ); + expect(report.verdict).toBe("unreachable"); + }); + + it("prefers a confirmed provider-ready over a stale bridge-did-not-start (#6743)", () => { + const report = evaluateTelegramDiagnostics( + baseInput({ breadcrumbs: breadcrumbs({ bridgeNotStarted: true, providerReady: true }) }), + ); + expect(report.verdict).toBe("idle"); + }); + + it("prefers a confirmed provider-ready over a transient network error (#6743)", () => { + const report = evaluateTelegramDiagnostics( + baseInput({ breadcrumbs: breadcrumbs({ startupFailedNetwork: true, providerReady: true }) }), + ); + expect(report.verdict).toBe("idle"); + }); +}); + +describe("parseTelegramBreadcrumbs", () => { + it("returns null when no [telegram] line is present", () => { + expect( + parseTelegramBreadcrumbs(["[slack] [default] provider ready", "random line"]), + ).toBeNull(); + }); + + it("classifies the known startup phrases", () => { + const bc = parseTelegramBreadcrumbs([ + "[telegram] [default] provider ready (Bot API reachable; agent replies use inference.local)", + "[telegram] [default] inbound update received (update_id=present; message_id=present)", + ]); + expect(bc).toMatchObject({ providerReady: true, inboundReceived: true }); + }); + + it("classifies a rejected token vs a network failure vs credential gap", () => { + expect( + parseTelegramBreadcrumbs([ + "[telegram] [default] Bot API rejected startup probe with HTTP 401; token invalid or credential placeholder unresolved", + ]), + ).toMatchObject({ tokenRejected: true }); + + expect( + parseTelegramBreadcrumbs(["[telegram] [default] Bot API startup probe failed: ETIMEDOUT"]), + ).toMatchObject({ startupFailedNetwork: true }); + + expect( + parseTelegramBreadcrumbs([ + "[telegram] [default] credential placeholder configured but TELEGRAM_BOT_TOKEN is missing from runtime env", + ]), + ).toMatchObject({ credentialUnresolved: true }); + }); + + it("captures a non-auth HTTP startup error code", () => { + expect( + parseTelegramBreadcrumbs(["[telegram] [default] Bot API startup probe returned HTTP 502"]), + ).toMatchObject({ startupHttpError: 502 }); + }); + + it("flags a bridge that did not start", () => { + expect( + parseTelegramBreadcrumbs(["[telegram] [default] bridge did not start within 15s"]), + ).toMatchObject({ bridgeNotStarted: true }); + }); + + it("classifies OpenClaw native (timestamped) network-failure lines (#6743)", () => { + // A network failure outranks the bridge-did-not-start timeout it causes. + const bc = parseTelegramBreadcrumbs([ + "2026-07-14T18:55:23.313+00:00 [telegram] deleteWebhook failed: Network request for 'deleteWebhook' failed!", + "[telegram] [default] bridge did not start within 15s; check channels.telegram.enabled", + ]); + expect(bc).toMatchObject({ startupFailedNetwork: true, bridgeNotStarted: false }); + }); + + it("treats a later inbound as recovery over a stale startup network failure (#6743)", () => { + // Bridge started while the network was blocked, then recovered and received + // a message — the latest evidence (inbound) must win over the stale failure. + const bc = parseTelegramBreadcrumbs([ + "2026-07-14T19:51:41.423+00:00 [telegram] deleteWebhook failed: Network request for 'deleteWebhook' failed!", + "[telegram] [default] bridge did not start within 15s", + "2026-07-14T20:03:22.254+00:00 [telegram] [diag] isolated polling ingress started spool=/sandbox/.openclaw/telegram/ingress-spool-default", + "2026-07-14T20:03:23.312+00:00 [telegram] Inbound message telegram:5209865443 -> @bot (direct, 2 chars)", + ]); + expect(bc).toMatchObject({ + providerReady: true, + inboundReceived: true, + startupFailedNetwork: false, + bridgeNotStarted: false, + }); + }); + + it("treats a later network failure as the current state over an earlier inbound (#6743)", () => { + const bc = parseTelegramBreadcrumbs([ + "2026-07-14T20:03:23.312+00:00 [telegram] Inbound message telegram:5209865443 -> @bot (direct, 2 chars)", + "[telegram] transport attempt marked temporarily unhealthy for 10000ms (codes=UND_ERR_SOCKET)", + ]); + expect(bc).toMatchObject({ startupFailedNetwork: true, providerReady: false }); + }); +}); + +describe("evaluateTelegramDiagnostics over real gateway-log windows (#6743)", () => { + it("reports healthy for a bridge that recovered after a blocked startup", () => { + const bc = parseTelegramBreadcrumbs([ + "2026-07-14T19:51:41.423+00:00 [telegram] deleteWebhook failed: Network request for 'deleteWebhook' failed!", + "[telegram] [default] bridge did not start within 15s", + "2026-07-14T20:03:22.254+00:00 [telegram] [diag] isolated polling ingress started spool=/sandbox/x", + "2026-07-14T20:03:23.312+00:00 [telegram] Inbound message telegram:5209865443 -> @bot (direct, 2 chars)", + ]); + const report = evaluateTelegramDiagnostics(baseInput({ breadcrumbs: bc })); + expect(report.verdict).toBe("healthy"); + }); + + it("reports unreachable once the network fails again after working", () => { + const bc = parseTelegramBreadcrumbs([ + "2026-07-14T20:03:23.312+00:00 [telegram] Inbound message telegram:5209865443 -> @bot (direct, 2 chars)", + "[telegram] transport attempt marked temporarily unhealthy for 20000ms (codes=UND_ERR_SOCKET)", + ]); + const report = evaluateTelegramDiagnostics(baseInput({ breadcrumbs: bc })); + expect(report.verdict).toBe("unreachable"); + }); + + it("does not stay token_rejected when a later provider-ready follows a stale 401 (#6887)", () => { + const bc = parseTelegramBreadcrumbs([ + "[telegram] [default] Bot API rejected startup probe with HTTP 401; token invalid or credential placeholder unresolved", + "[telegram] [default] provider ready (Bot API reachable; agent replies use inference.local)", + "[telegram] [default] inbound update received (update_id=present; message_id=present)", + ]); + expect(bc).toMatchObject({ + tokenRejected: false, + credentialUnresolved: false, + providerReady: true, + inboundReceived: true, + }); + expect(evaluateTelegramDiagnostics(baseInput({ breadcrumbs: bc })).verdict).toBe("healthy"); + }); + + it("reports token_rejected when a 401 is the latest evidence after a working bridge (#6887)", () => { + const bc = parseTelegramBreadcrumbs([ + "[telegram] [default] inbound update received (update_id=present; message_id=present)", + "[telegram] [default] Bot API rejected startup probe with HTTP 401; token invalid", + ]); + expect(bc).toMatchObject({ tokenRejected: true, providerReady: false }); + expect(evaluateTelegramDiagnostics(baseInput({ breadcrumbs: bc })).verdict).toBe( + "token_rejected", + ); + }); + + it("clears a stale HTTP 5xx error once a later provider-ready follows (#6887)", () => { + const bc = parseTelegramBreadcrumbs([ + "[telegram] [default] Bot API startup probe returned HTTP 502", + "[telegram] [default] provider ready (Bot API reachable; agent replies use inference.local)", + "[telegram] [default] inbound update received (update_id=present; message_id=present)", + ]); + expect(bc).toMatchObject({ + startupHttpError: null, + providerReady: true, + inboundReceived: true, + }); + const report = evaluateTelegramDiagnostics(baseInput({ breadcrumbs: bc })); + expect(report.verdict).toBe("healthy"); + // A stale 5xx must not surface as a warn while the verdict reads healthy. + const reach = report.signals.find((s) => s.label === "Bot API reachability"); + expect(reach?.severity).toBe("ok"); + }); + + it("does not carry a pre-outage inbound across a later failure into healthy (#6888)", () => { + // inbound → network failure → provider ready: the bridge recovered but no + // inbound has arrived since, so delivery is idle, not healthy. + const bc = parseTelegramBreadcrumbs([ + "[telegram] [default] inbound update received (update_id=present; message_id=present)", + "[telegram] [default] Bot API startup probe failed: ETIMEDOUT", + "[telegram] [default] provider ready (Bot API reachable; agent replies use inference.local)", + ]); + expect(bc).toMatchObject({ providerReady: true, inboundReceived: false }); + expect(evaluateTelegramDiagnostics(baseInput({ breadcrumbs: bc })).verdict).toBe("idle"); + }); + + it("reports healthy again once an inbound arrives after the recovery (#6888)", () => { + const bc = parseTelegramBreadcrumbs([ + "[telegram] [default] inbound update received (update_id=present; message_id=present)", + "[telegram] [default] Bot API startup probe failed: ETIMEDOUT", + "[telegram] [default] provider ready (Bot API reachable; agent replies use inference.local)", + "[telegram] Inbound message telegram:5209865443 -> @bot (direct, 3 chars)", + ]); + expect(bc).toMatchObject({ providerReady: true, inboundReceived: true }); + expect(evaluateTelegramDiagnostics(baseInput({ breadcrumbs: bc })).verdict).toBe("healthy"); + }); + + it("honors a later HTTP 5xx over an earlier network failure (#6888)", () => { + const bc = parseTelegramBreadcrumbs([ + "[telegram] [default] Bot API startup probe failed: ETIMEDOUT", + "[telegram] [default] Bot API startup probe returned HTTP 502", + ]); + expect(bc).toMatchObject({ startupHttpError: 502, startupFailedNetwork: false }); + }); + + it("honors a later HTTP 5xx over an earlier token rejection (#6888)", () => { + const bc = parseTelegramBreadcrumbs([ + "[telegram] [default] Bot API rejected startup probe with HTTP 401; token invalid", + "[telegram] [default] Bot API startup probe returned HTTP 502", + ]); + expect(bc).toMatchObject({ startupHttpError: 502, tokenRejected: false }); + }); +}); diff --git a/src/lib/messaging/channels/telegram/hooks/status-health-eval.ts b/src/lib/messaging/channels/telegram/hooks/status-health-eval.ts new file mode 100644 index 00000000000..4b16a832ce6 --- /dev/null +++ b/src/lib/messaging/channels/telegram/hooks/status-health-eval.ts @@ -0,0 +1,439 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Pure helpers that translate raw probe evidence collected from inside a + * sandbox into a structured Telegram channel-health report. + * + * Consumed by the `telegram.statusHealth` status hook (see `status-health.ts`); + * this module never touches the filesystem, child processes, or the clock so + * the evaluation can be exercised hermetically from fixtures. + * + * Telegram's bridge is an in-process poller inside the OpenClaw gateway, not a + * separate process with a heartbeat file (unlike WhatsApp's Baileys bridge). So + * "liveness" is inferred from two places the gateway already produces: + * 1. the gateway process being alive (pgrep), and + * 2. the `[telegram] [default] …` breadcrumbs the runtime diagnostics preload + * writes to /tmp/gateway.log (see ../runtime/telegram-diagnostics.ts). + * We deliberately do NOT run our own getMe from the probe: verified live, the + * egress MITM proxy refuses a raw `curl` at CONNECT (HTTP 403) while authorizing + * only the gateway's instrumented Node egress, and the resolved token never + * leaves that path. The gateway already performs getMe/getUpdates and logs the + * outcome — we read that outcome instead. + */ + +import type { ChannelHealthReport, DiagnosticSignal } from "../../channel-health"; + +export type TelegramVerdict = + | "healthy" + | "idle" + | "token_rejected" + | "unreachable" + | "not_started" + | "policy_gap" + | "config_gap" + | "unknown" + | "probe_failed"; + +/** + * Classified `[telegram] [default] …` breadcrumbs parsed from the gateway + * log. Every field is a boolean/number derived from a fixed log phrase; no + * raw log text is carried through so message bodies / tokens cannot leak. + */ +export type TelegramBreadcrumbs = { + // "[telegram] [default] provider ready (Bot API reachable …)" + providerReady: boolean; + // "… Bot API rejected startup probe with HTTP 401/404; token invalid …" + tokenRejected: boolean; + // "… credential placeholder … missing from runtime env" / "… mismatch" + credentialUnresolved: boolean; + // Network failure reaching the Bot API — either the diagnostics preload's + // "… Bot API startup probe failed: " or OpenClaw's own + // "… Network request for '' failed" / "recoverable network error". + startupFailedNetwork: boolean; + // "… Bot API startup probe returned HTTP " (n>=300, not 401/404) + startupHttpError: number | null; + // "… bridge did not start within Ns" + bridgeNotStarted: boolean; + // "… inbound update received (update_id=…)" + inboundReceived: boolean; +}; + +export type TelegramProbeInput = { + // Agent owning the sandbox: "openclaw", "hermes", etc. Used for hint text. + agent: string; + // Whether the orchestrator could run `openshell sandbox exec` at all. + probeReachable: boolean; + // True when the OpenClaw gateway process (which hosts the telegram poller) + // was observed running. Null when the process probe could not complete. + gatewayProcessAlive: boolean | null; + // Parsed startup/poll breadcrumbs, or null when no `[telegram]` line was + // found in the tailed gateway log window. + breadcrumbs: TelegramBreadcrumbs | null; + // ISO timestamp captured by the orchestrator when the probe ran. + probedAt: string; + // Whether the telegram preset is recorded in the sandbox registry. + presetInRegistry: boolean; + // Whether the telegram preset's network policy is loaded on the gateway, + // or null when the gateway could not be reached. + presetOnGateway: boolean | null; + // Whether the telegram channel is recorded in the registry messaging plan. + channelEnabledInRegistry: boolean; +}; + +export type TelegramDiagnosticReport = ChannelHealthReport & { + channel: "telegram"; + verdict: TelegramVerdict; +}; + +function configCoverageSignal(input: TelegramProbeInput): DiagnosticSignal { + if (!input.channelEnabledInRegistry) { + return { + label: "Channel registration", + severity: "fail", + detail: "telegram is not in the sandbox messaging plan", + hint: "run `nemoclaw channels add telegram`", + }; + } + return { + label: "Channel registration", + severity: "ok", + detail: "telegram channel registered for the sandbox", + }; +} + +function policyCoverageSignal(input: TelegramProbeInput): DiagnosticSignal { + if (input.presetOnGateway === false && input.presetInRegistry) { + return { + label: "Policy coverage", + severity: "fail", + detail: "telegram preset recorded locally but missing from the gateway policy", + hint: "rebuild the sandbox so the preset is reapplied to the OpenShell gateway", + }; + } + if (!input.presetInRegistry) { + return { + label: "Policy coverage", + severity: "fail", + detail: "telegram preset is not applied to the sandbox", + hint: "run `nemoclaw policy-add telegram` and rebuild the sandbox", + }; + } + if (input.presetOnGateway === null) { + return { + label: "Policy coverage", + severity: "info", + detail: "telegram preset recorded locally; gateway is unreachable for cross-check", + }; + } + return { + label: "Policy coverage", + severity: "ok", + detail: "telegram preset applied and loaded on the gateway", + }; +} + +function bridgeProcessSignal(input: TelegramProbeInput): DiagnosticSignal { + if (input.gatewayProcessAlive === null) { + return { + label: "Bridge process", + severity: "info", + detail: "could not enumerate sandbox processes", + }; + } + if (input.gatewayProcessAlive === false) { + return { + label: "Bridge process", + severity: "fail", + detail: "no OpenClaw gateway process observed — telegram poller is not running", + hint: "check `nemoclaw logs --follow` for gateway startup errors", + }; + } + return { + label: "Bridge process", + severity: "ok", + detail: "gateway process running (telegram poller host)", + }; +} + +/** + * The token-vs-network distinction the VDR item asked for. Reads the + * gateway's own getMe/getUpdates outcome breadcrumbs. + */ +function reachabilitySignal(input: TelegramProbeInput): DiagnosticSignal { + const bc = input.breadcrumbs; + if (!bc) { + return { + label: "Bot API reachability", + severity: input.gatewayProcessAlive === false ? "fail" : "info", + detail: "no telegram startup breadcrumb in the gateway log window", + hint: "the poller may not have started yet — re-run after the gateway settles, or check logs", + }; + } + if (bc.credentialUnresolved) { + return { + label: "Bot API reachability", + severity: "fail", + detail: + "credential placeholder is unresolved — TELEGRAM_BOT_TOKEN missing/mismatched at runtime", + hint: "reset the telegram credential and rebuild: `nemoclaw credentials reset TELEGRAM_BOT_TOKEN && nemoclaw rebuild`", + }; + } + if (bc.tokenRejected) { + return { + label: "Bot API reachability", + severity: "fail", + detail: "Telegram rejected the bot token (HTTP 401/404)", + hint: "verify the token from @BotFather, reset the credential, then rebuild", + }; + } + if (bc.startupFailedNetwork) { + return { + label: "Bot API reachability", + severity: "fail", + detail: "could not reach api.telegram.org from the sandbox (network error)", + hint: "check the telegram egress policy is loaded and the network allows api.telegram.org", + }; + } + if (bc.startupHttpError !== null) { + return { + label: "Bot API reachability", + severity: "warn", + detail: `Telegram startup probe returned HTTP ${bc.startupHttpError}`, + hint: "check `nemoclaw logs --follow`", + }; + } + if (bc.providerReady) { + return { + label: "Bot API reachability", + severity: "ok", + detail: "gateway reached api.telegram.org and the token was accepted", + }; + } + if (bc.bridgeNotStarted) { + return { + label: "Bot API reachability", + severity: "warn", + detail: "bridge did not confirm startup within its probe window", + hint: "check `nemoclaw logs --follow`; rebuild if it stays silent", + }; + } + return { + label: "Bot API reachability", + severity: "info", + detail: "startup outcome not conclusive from the log window", + }; +} + +function inboundSignal(input: TelegramProbeInput): DiagnosticSignal { + const bc = input.breadcrumbs; + if (!bc || !bc.providerReady) { + return { + label: "Inbound delivery", + severity: "info", + detail: "not evaluated (provider not confirmed ready)", + }; + } + if (bc.inboundReceived) { + return { + label: "Inbound delivery", + severity: "ok", + detail: "at least one inbound getUpdates delivery was observed", + }; + } + return { + label: "Inbound delivery", + severity: "info", + detail: "provider polling; no inbound update observed in the log window", + hint: "send a message to the bot from an allowed Telegram account, then re-run", + }; +} + +function pickVerdict(signals: DiagnosticSignal[], input: TelegramProbeInput): TelegramVerdict { + if (!input.probeReachable) return "probe_failed"; + if (signals.some((s) => s.label === "Channel registration" && s.severity === "fail")) { + return "config_gap"; + } + if (signals.some((s) => s.label === "Policy coverage" && s.severity === "fail")) { + return "policy_gap"; + } + const bc = input.breadcrumbs; + // Hard failures first: a bad token or a dead gateway process won't self-heal. + if (bc?.credentialUnresolved || bc?.tokenRejected) return "token_rejected"; + if (input.gatewayProcessAlive === false) return "not_started"; + // A confirmed `provider ready` means the bridge reached Telegram and the + // token works — it outranks the soft "did not start (yet)" and transient + // network-blip signals, which may be older log lines from a slow start. + if (bc?.providerReady) { + return bc.inboundReceived ? "healthy" : "idle"; + } + if (bc?.startupFailedNetwork) return "unreachable"; + if (bc?.bridgeNotStarted) return "not_started"; + return "unknown"; +} + +function buildHints(verdict: TelegramVerdict): string[] { + switch (verdict) { + case "healthy": + return [ + "Telegram is reachable, the token is valid, and inbound updates are being delivered.", + ]; + case "idle": + return [ + "Provider is polling and reachable, but no inbound update was seen. Send a message from an allowed account and re-run.", + ]; + case "token_rejected": + return [ + "Telegram rejected the token. Reset the credential and rebuild — this will not recover on its own.", + ]; + case "unreachable": + return [ + "The sandbox could not reach api.telegram.org. Confirm the telegram egress policy is loaded and the corporate network allows Telegram.", + ]; + case "not_started": + return [ + "The telegram poller did not start. Check `nemoclaw logs --follow` and rebuild if needed.", + ]; + case "policy_gap": + return ["Run `nemoclaw policy-add telegram`, then rebuild."]; + case "config_gap": + return ["Run `nemoclaw channels add telegram` to enable the channel."]; + case "probe_failed": + return [ + "Start the sandbox and verify the OpenShell gateway is healthy, then re-run channels status.", + ]; + case "unknown": + return [ + "Startup outcome was not conclusive. Re-run after the gateway settles, or rebuild the sandbox.", + ]; + } +} + +export function evaluateTelegramDiagnostics(input: TelegramProbeInput): TelegramDiagnosticReport { + const signals: DiagnosticSignal[] = [ + configCoverageSignal(input), + policyCoverageSignal(input), + bridgeProcessSignal(input), + reachabilitySignal(input), + inboundSignal(input), + ]; + const verdict = pickVerdict(signals, input); + return { + schemaVersion: 1, + channel: "telegram", + agent: input.agent, + verdict, + probedAt: input.probedAt, + signals, + hints: buildHints(verdict), + }; +} + +// ── Breadcrumb parser ──────────────────────────────────────────────────── +// Turns the tailed `[telegram] [default] …` gateway-log lines into the +// classified TelegramBreadcrumbs. Fixed phrase matching only; never carries +// raw log text forward. + +export function parseTelegramBreadcrumbs(logLines: readonly string[]): TelegramBreadcrumbs | null { + // Accept both the preload's `[telegram] [default] …` lines and OpenClaw's + // timestamped `… [telegram] …` gateway lines (which carry native network + // errors), so a network-blocked channel is classified rather than left blank. + const telegramLines = logLines.filter((line) => /\[telegram\]/.test(line)); + if (telegramLines.length === 0) return null; + const bc: TelegramBreadcrumbs = { + providerReady: false, + tokenRejected: false, + credentialUnresolved: false, + startupFailedNetwork: false, + startupHttpError: null, + bridgeNotStarted: false, + inboundReceived: false, + }; + // Tailed lines are in chronological (append) order, so the *latest* matching + // line reflects the current reachability state. A stale startup network + // failure must not outrank a later "reached Telegram" line (a bridge that + // started while blocked then recovered), nor vice-versa (a channel that + // worked then got blocked again). Track the last index of each state and let + // the most recent win; a network failure outranks the "bridge did not start" + // timeout it causes. `provider ready` / `inbound update received` are preload + // phrases; `Inbound message telegram:` / `isolated polling ingress started` + // are OpenClaw's own positive lines; `Network request … failed` / + // `temporarily unhealthy` / `UND_ERR_SOCKET` are its transport-failure lines. + const REACHED = + /\bprovider ready\b|inbound update received|inbound message telegram|isolated polling ingress started/i; + const NETWORK_FAIL = + /startup probe failed|network request for .+ failed|recoverable network error|temporarily unhealthy|UND_ERR_SOCKET/i; + let lastReached = -1; + let lastNetworkFail = -1; + let lastBridgeNotStarted = -1; + let lastTokenRejected = -1; + let lastCredentialUnresolved = -1; + let lastHttpError = -1; + let lastHttpErrorCode: number | null = null; + let lastInbound = -1; + telegramLines.forEach((line, index) => { + const httpErr = /startup probe returned HTTP\s+(\d{3})/i.exec(line); + if (httpErr) { + lastHttpError = index; + lastHttpErrorCode = Number(httpErr[1]); + } + if (/inbound update received|inbound message telegram/i.test(line)) lastInbound = index; + if (REACHED.test(line)) lastReached = index; + if (NETWORK_FAIL.test(line)) lastNetworkFail = index; + if (/bridge did not start within/i.test(line)) lastBridgeNotStarted = index; + if (/rejected startup probe with HTTP\s+(401|404)/i.test(line)) lastTokenRejected = index; + if (/credential placeholder.*(missing|mismatch|unresolved)/i.test(line)) { + lastCredentialUnresolved = index; + } + }); + // The most recent evidence wins (a `reached` positive wins ties). A token + // rejection or unresolved credential is a cause, honored only when it is the + // latest evidence over any later "reached Telegram" line — so a stale 401 + // before a currently working bridge is not reported as token_rejected. Among + // failures, a token/network cause outranks the bridge-did-not-start timeout it + // produces; the latest of token vs network wins. + // The single latest cause wins: token/credential rejection, a network + // failure, and a non-auth HTTP 5xx are peers ranked purely by recency, so a + // later 502 supersedes an earlier 401/timeout (#6888). Each of these outranks + // the bridge-did-not-start timeout they produce, so bridge is reported only + // when it is the sole cause. A later `reached` line still supersedes all of + // them (a stale cause before a currently working bridge is not reported). + const lastCause = Math.max( + lastTokenRejected, + lastCredentialUnresolved, + lastNetworkFail, + lastHttpError, + ); + const lastEvidence = Math.max(lastReached, lastCause, lastBridgeNotStarted); + if (lastReached !== -1 && lastReached >= lastEvidence) { + bc.providerReady = true; + } else if (lastCause !== -1) { + if (lastHttpError === lastCause) { + bc.startupHttpError = lastHttpErrorCode; + } else if (lastNetworkFail === lastCause) { + bc.startupFailedNetwork = true; + } else if (lastCredentialUnresolved > lastTokenRejected) { + // A 401 line also carries "credential placeholder unresolved", so on a tie + // prefer tokenRejected; credentialUnresolved wins only when its own line + // (TELEGRAM_BOT_TOKEN missing from env) is strictly later. + bc.credentialUnresolved = true; + } else { + bc.tokenRejected = true; + } + } else if (lastBridgeNotStarted !== -1) { + bc.bridgeNotStarted = true; + } + // Inbound delivery counts as "current" only when the latest inbound is newer + // than the latest outage boundary. A stale inbound from before a later + // failure/recovery must not read as healthy delivery — a recovered bridge + // with no inbound since recovery is idle, not healthy (#6888). + const lastOutage = Math.max( + lastNetworkFail, + lastBridgeNotStarted, + lastTokenRejected, + lastCredentialUnresolved, + lastHttpError, + ); + bc.inboundReceived = lastInbound !== -1 && lastInbound > lastOutage; + return bc; +} diff --git a/src/lib/messaging/channels/telegram/hooks/status-health.test.ts b/src/lib/messaging/channels/telegram/hooks/status-health.test.ts new file mode 100644 index 00000000000..ad2429cd732 --- /dev/null +++ b/src/lib/messaging/channels/telegram/hooks/status-health.test.ts @@ -0,0 +1,152 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import { describe, expect, it, vi } from "vitest"; +import type { MessagingHookContext, MessagingHookResult } from "../../../hooks/types"; +import type { ChannelHealthReport } from "../../channel-health"; +import { createTelegramStatusHealthHook } from "./status-health"; + +const BASE_INPUTS = { + currentSandbox: "alpha", + agent: "openclaw", + probedAt: "2026-07-14T00:00:00.000Z", + channelEnabledInRegistry: true, + presetInRegistry: true, + presetOnGateway: true, +}; + +function context( + inputs: Record = BASE_INPUTS, + channelId = "telegram", +): MessagingHookContext { + return { + channelId, + hookId: "telegram-status-health", + phase: "status", + inputs, + } as unknown as MessagingHookContext; +} + +function probeStdout(logLines: string[], procLines: string[]): string { + return [ + "NEMOCLAW_TG_DIAG_OK", + "NEMOCLAW_TG_LOG_BEGIN", + ...logLines, + "NEMOCLAW_TG_LOG_END", + ...procLines, + "NEMOCLAW_TG_PROC_DONE", + ].join("\n"); +} + +type ExecResult = { status: number; stdout: string; stderr: string } | null; + +function makeExec(result: ExecResult) { + return vi.fn((_sandbox: string, _command: string, _timeout: number): ExecResult => result); +} + +// The hook is synchronous; the handler type is a sync|Promise union, so narrow. +function reportOf( + result: MessagingHookResult | Promise, +): ChannelHealthReport | undefined { + const value = (result as MessagingHookResult).outputs?.channelHealth?.value as unknown as + | { report?: ChannelHealthReport } + | undefined; + return value?.report; +} + +function outputsOf(result: MessagingHookResult | Promise) { + return (result as MessagingHookResult).outputs; +} + +describe("telegram.statusHealth hook", () => { + it("probes the gateway log and reports healthy for a ready bridge with inbound (#6743)", () => { + const exec = makeExec({ + status: 0, + stdout: probeStdout( + [ + "[telegram] [default] provider ready (Bot API reachable; agent replies use inference.local)", + "[telegram] [default] inbound update received (update_id=present; message_id=present)", + ], + ["PROC 42 node /opt/openclaw gateway"], + ), + stderr: "", + }); + const result = createTelegramStatusHealthHook({ executeSandboxCommand: exec })(context()); + expect(reportOf(result)?.verdict).toBe("healthy"); + + // The probe reads the gateway's own breadcrumbs and never calls the Bot API. + const command = exec.mock.calls[0]?.[1] ?? ""; + expect(command).toMatch(/\/tmp\/gateway\.log/); + expect(command).toMatch(/pgrep/); + expect(command).not.toMatch(/getMe/i); + expect(command).not.toMatch(/curl/i); + }); + + it("emits a syntactically valid /bin/sh probe script (#6743)", () => { + // The probe is a multiline sh script (grep/awk pipelines, marker sequencing). + // A shell syntax regression would fail every real probe while mocked-stdout + // tests stay green, so validate the generated command with `sh -n`. + const exec = makeExec({ status: 0, stdout: probeStdout([], []), stderr: "" }); + createTelegramStatusHealthHook({ executeSandboxCommand: exec })(context()); + const command = exec.mock.calls[0]?.[1] ?? ""; + const validation = spawnSync("sh", ["-n", "-c", command], { encoding: "utf-8" }); + expect(validation.status, validation.stderr || validation.stdout).toBe(0); + // The probe must filter its own pgrep line out of the process results. + expect(command).toMatch(/__nemoclaw_tg_self_pid/); + expect(command).toMatch(/pgrep -fa/); + }); + + it("reports not_started when pgrep completes with no gateway process", () => { + const exec = makeExec({ status: 0, stdout: probeStdout([], []), stderr: "" }); + const result = createTelegramStatusHealthHook({ executeSandboxCommand: exec })(context()); + expect(reportOf(result)?.verdict).toBe("not_started"); + }); + + it("reports probe_failed when the sandbox exec fails", () => { + const exec = makeExec(null); + const result = createTelegramStatusHealthHook({ executeSandboxCommand: exec })(context()); + expect(reportOf(result)?.verdict).toBe("probe_failed"); + }); + + it("treats a non-zero exec as a failed probe even with healthy-looking stdout (#6887)", () => { + // A timed-out/killed exec can still carry partial stdout with a stale + // provider-ready line — a non-zero status must not read as healthy. + const exec = makeExec({ + status: 1, + stdout: probeStdout( + [ + "[telegram] [default] provider ready (Bot API reachable; agent replies use inference.local)", + ], + ["PROC 42 node /opt/openclaw gateway"], + ), + stderr: "sandbox exec timed out", + }); + const result = createTelegramStatusHealthHook({ executeSandboxCommand: exec })(context()); + expect(reportOf(result)?.verdict).toBe("probe_failed"); + }); + + it("derives config_gap / policy_gap from the host-fact inputs", () => { + const exec = makeExec({ status: 0, stdout: probeStdout([], []), stderr: "" }); + const hook = createTelegramStatusHealthHook({ executeSandboxCommand: exec }); + expect( + reportOf(hook(context({ ...BASE_INPUTS, channelEnabledInRegistry: false })))?.verdict, + ).toBe("config_gap"); + expect(reportOf(hook(context({ ...BASE_INPUTS, presetInRegistry: false })))?.verdict).toBe( + "policy_gap", + ); + }); + + it("no-ops for a non-telegram channel or without an exec runner", () => { + const exec = makeExec({ status: 0, stdout: probeStdout([], []), stderr: "" }); + expect( + outputsOf( + createTelegramStatusHealthHook({ executeSandboxCommand: exec })( + context(BASE_INPUTS, "slack"), + ), + ), + ).toBeUndefined(); + expect(outputsOf(createTelegramStatusHealthHook({})(context()))).toBeUndefined(); + expect(exec).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/messaging/channels/telegram/hooks/status-health.ts b/src/lib/messaging/channels/telegram/hooks/status-health.ts new file mode 100644 index 00000000000..9dab4534c35 --- /dev/null +++ b/src/lib/messaging/channels/telegram/hooks/status-health.ts @@ -0,0 +1,145 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * `telegram.statusHealth` — a `phase: "status"` hook that probes the live + * Telegram bridge health from inside the sandbox and emits a + * `messaging-channel-health` status output. Run by the generic channels-status + * command via the status-hook runner, so no telegram-specific code lives in the + * generic status orchestrator. + * + * The bridge is an in-process poller inside the OpenClaw gateway, so there is no + * separate process or heartbeat file (unlike WhatsApp). The probe tails the + * gateway log for the `[telegram] …` breadcrumbs the runtime preload writes, + * plus a pgrep for the gateway process. It never runs its own getMe: verified + * live, the egress MITM proxy refuses a raw `curl` at CONNECT (HTTP 403) and + * authorizes only the gateway's instrumented Node egress, so reading the + * gateway's own logged outcome is the sanctioned path. + */ + +import { shellQuote as quotePath } from "../../../../core/shell-quote"; +import type { MessagingHookHandler, MessagingHookRegistration } from "../../../hooks/types"; +import type { MessagingSerializableValue } from "../../../manifest"; +import { + type ChannelStatusHealthHookOptions, + MESSAGING_CHANNEL_HEALTH_OUTPUT_TYPE, +} from "../../channel-health"; +import { + evaluateTelegramDiagnostics, + parseTelegramBreadcrumbs, + type TelegramProbeInput, +} from "./status-health-eval"; + +export const TELEGRAM_STATUS_HEALTH_HOOK_HANDLER_ID = "telegram.statusHealth"; + +const DEFAULT_TIMEOUT_MS = 8_000; +const TG_SHELL_OK = "NEMOCLAW_TG_DIAG_OK"; +const TG_LOG_BEGIN = "NEMOCLAW_TG_LOG_BEGIN"; +const TG_LOG_END = "NEMOCLAW_TG_LOG_END"; +const TG_PROC_DONE = "NEMOCLAW_TG_PROC_DONE"; +const OPENCLAW_GATEWAY_LOG_FILE = "/tmp/gateway.log"; + +/** Telegram uses the generic channel-health hook options unchanged. */ +export type TelegramStatusHealthHookOptions = ChannelStatusHealthHookOptions; + +export function createTelegramStatusHealthHook( + options: TelegramStatusHealthHookOptions = {}, +): MessagingHookHandler { + return (context) => { + if (context.channelId !== "telegram") return {}; + const execute = options.executeSandboxCommand; + const sandboxName = normalizeString(context.inputs?.currentSandbox); + // Without a sandbox target or an exec runner there is nothing to probe (e.g. + // the top-level status runner does not thread an exec runner into this hook). + if (!execute || !sandboxName) return {}; + + const timeoutMs = normalizeTimeoutMs(options.timeoutMs); + const exec = execute(sandboxName, buildTelegramProbeScript(), timeoutMs); + const lines = String(exec?.stdout ?? "").split(/\r?\n/); + // A non-zero exec (timeout/kill/unhealthy sandbox) can still carry partial + // stdout with a stale `provider ready` line; require a clean exit so a failed + // probe classifies as probe_failed rather than a false healthy. + const reachable = exec?.status === 0 && lines.includes(TG_SHELL_OK); + + const logStart = lines.indexOf(TG_LOG_BEGIN); + const logEnd = lines.indexOf(TG_LOG_END); + const logLines = + logStart !== -1 && logEnd > logStart + ? lines + .slice(logStart + 1, logEnd) + .map((line) => line.trim()) + .filter(Boolean) + : []; + const breadcrumbs = reachable ? parseTelegramBreadcrumbs(logLines) : null; + + const sawProc = lines.some((line) => line.startsWith("PROC ")); + const sawProcDone = lines.includes(TG_PROC_DONE); + const gatewayProcessAlive = sawProc ? true : sawProcDone ? false : null; + + const input: TelegramProbeInput = { + agent: normalizeString(context.inputs?.agent) ?? "openclaw", + probeReachable: reachable, + gatewayProcessAlive, + breadcrumbs, + probedAt: normalizeString(context.inputs?.probedAt) ?? "", + presetInRegistry: Boolean(context.inputs?.presetInRegistry), + presetOnGateway: normalizeTristate(context.inputs?.presetOnGateway), + channelEnabledInRegistry: Boolean(context.inputs?.channelEnabledInRegistry), + }; + + const report = evaluateTelegramDiagnostics(input); + return { + outputs: { + channelHealth: { + kind: "status", + value: { + type: MESSAGING_CHANNEL_HEALTH_OUTPUT_TYPE, + report, + } as unknown as MessagingSerializableValue, + }, + }, + }; + }; +} + +export function createTelegramStatusHealthHookRegistration( + options: TelegramStatusHealthHookOptions = {}, +): MessagingHookRegistration { + return { + id: TELEGRAM_STATUS_HEALTH_HOOK_HANDLER_ID, + handler: createTelegramStatusHealthHook(options), + }; +} + +function buildTelegramProbeScript(): string { + return [ + `set +e`, + `printf '%s\\n' ${quotePath(TG_SHELL_OK)}`, + `printf '%s\\n' ${quotePath(TG_LOG_BEGIN)}`, + // Match `[telegram]` anywhere on the line: the diagnostics preload writes + // non-timestamped `[telegram] [default] …` lines, while OpenClaw's own + // gateway logs carry a leading timestamp before `[telegram] …` (these hold + // the native network-failure evidence the evaluator classifies). + `tail -n 400 ${quotePath(OPENCLAW_GATEWAY_LOG_FILE)} 2>/dev/null | grep -aE '\\[telegram\\]' | tail -n 40`, + `printf '%s\\n' ${quotePath(TG_LOG_END)}`, + `__nemoclaw_tg_self_pid=$$`, + `pgrep -fa 'openclaw|openclaw-gateway|node .*gateway' 2>/dev/null | awk -v self="$__nemoclaw_tg_self_pid" '$1 != self && $0 !~ /pgrep -fa/ { print "PROC " $0 }' | head -n 5`, + `printf '%s\\n' ${quotePath(TG_PROC_DONE)}`, + ].join("\n"); +} + +function normalizeString(value: unknown): string | null { + return typeof value === "string" && value.trim().length > 0 ? value.trim() : null; +} + +function normalizeTristate(value: unknown): boolean | null { + if (value === true) return true; + if (value === false) return false; + return null; +} + +function normalizeTimeoutMs(value: number | undefined): number { + return typeof value === "number" && Number.isFinite(value) && value > 0 + ? value + : DEFAULT_TIMEOUT_MS; +} diff --git a/src/lib/messaging/channels/telegram/manifest.ts b/src/lib/messaging/channels/telegram/manifest.ts index 07601b99bbe..2f8c4f84c41 100644 --- a/src/lib/messaging/channels/telegram/manifest.ts +++ b/src/lib/messaging/channels/telegram/manifest.ts @@ -8,6 +8,7 @@ export const telegramManifest = { id: "telegram", displayName: "Telegram", description: "Telegram bot messaging", + diagnosticsProbe: "log-tail", enrollmentNotes: [ "For Telegram group chats, disable privacy mode in @BotFather (/setprivacy -> your bot -> Disable).", "After changing privacy mode, remove and re-add the bot to each group before testing @mentions.", @@ -264,5 +265,17 @@ export const telegramManifest = { }, ], }, + { + id: "telegram-status-health", + phase: "status", + handler: "telegram.statusHealth", + agents: ["openclaw"], + outputs: [ + { + id: "channelHealth", + kind: "status", + }, + ], + }, ], } as const satisfies ChannelManifest; diff --git a/src/lib/messaging/compiler/manifest-compiler.test.ts b/src/lib/messaging/compiler/manifest-compiler.test.ts index 1667739ccdc..5273e8edf42 100644 --- a/src/lib/messaging/compiler/manifest-compiler.test.ts +++ b/src/lib/messaging/compiler/manifest-compiler.test.ts @@ -823,6 +823,7 @@ describe("ManifestCompiler", () => { "telegram-get-me-reachability", "telegram-openclaw-bridge-health", "telegram-gateway-conflict-status", + "telegram-status-health", ]); expect(plan.runtimeSetup).toEqual({ nodePreloads: [], envAliases: [], secretScans: [] }); expect(plan.credentialBindings.map((binding) => binding.channelId)).toEqual(["telegram"]); @@ -1279,6 +1280,7 @@ describe("ManifestCompiler", () => { "telegram-get-me-reachability", "telegram-openclaw-bridge-health", "telegram-gateway-conflict-status", + "telegram-status-health", ]); expect(plan.runtimeSetup).toEqual({ nodePreloads: [], envAliases: [], secretScans: [] }); }); diff --git a/src/lib/messaging/diagnostics.test.ts b/src/lib/messaging/diagnostics.test.ts index c075044aa74..0425574f352 100644 --- a/src/lib/messaging/diagnostics.test.ts +++ b/src/lib/messaging/diagnostics.test.ts @@ -20,6 +20,7 @@ describe("messaging channel diagnostics", () => { expect(specs.find((spec) => spec.channelId === "telegram")).toMatchObject({ policyPresets: ["telegram"], preferredDefault: false, + deepProbe: "log-tail", }); expect(specs.find((spec) => spec.channelId === "wechat")).toMatchObject({ policyPresets: ["wechat"], diff --git a/src/lib/messaging/diagnostics.ts b/src/lib/messaging/diagnostics.ts index 0b91a03e5d8..b875667e538 100644 --- a/src/lib/messaging/diagnostics.ts +++ b/src/lib/messaging/diagnostics.ts @@ -8,7 +8,7 @@ export interface MessagingChannelDiagnosticSpec { readonly channelId: string; readonly policyPresets: readonly string[]; readonly preferredDefault: boolean; - readonly deepProbe?: "in-sandbox-qr"; + readonly deepProbe?: "in-sandbox-qr" | "log-tail"; readonly doctorWhenNoHealthSignals?: { readonly detail: string; readonly hint: string; @@ -29,11 +29,16 @@ export function collectMessagingChannelDiagnostics( manifests: readonly ChannelManifest[], ): MessagingChannelDiagnosticSpec[] { return manifests.map((manifest) => { - const deepProbe = manifest.auth.mode === "in-sandbox-qr" ? "in-sandbox-qr" : undefined; + const deepProbe = + manifest.auth.mode === "in-sandbox-qr" + ? ("in-sandbox-qr" as const) + : manifest.diagnosticsProbe === "log-tail" + ? ("log-tail" as const) + : undefined; return { channelId: manifest.id, policyPresets: policyPresetNames(manifest.policyPresets), - preferredDefault: deepProbe !== undefined, + preferredDefault: manifest.auth.mode === "in-sandbox-qr", ...(deepProbe ? { deepProbe, doctorWhenNoHealthSignals: qrDeepProbeDoctorHint() } : {}), }; }); diff --git a/src/lib/messaging/hooks/builtins.ts b/src/lib/messaging/hooks/builtins.ts index f181b6eabd9..ade5940b2b4 100644 --- a/src/lib/messaging/hooks/builtins.ts +++ b/src/lib/messaging/hooks/builtins.ts @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import type { ChannelStatusHealthHookOptions } from "../channels/channel-health"; import { createDiscordHookRegistrations, type DiscordHookOptions } from "../channels/discord/hooks"; import type { OpenClawBridgeHealthHookOptions } from "../channels/openclaw-bridge-health"; import { createSlackHookRegistrations, type SlackHookOptions } from "../channels/slack/hooks"; @@ -22,6 +23,9 @@ export interface BuiltInMessagingHookOptions { readonly teams?: TeamsHookOptions; readonly telegram?: TelegramHookOptions; readonly wechat?: WechatHookOptions; + // Host capability threaded into every channel's `phase:"status"` health hook, + // so a status caller enables live probing without naming a specific channel. + readonly statusHealth?: ChannelStatusHealthHookOptions; } export function createBuiltInMessagingHookRegistrations( @@ -37,7 +41,10 @@ export function createBuiltInMessagingHookRegistrations( ), ...createTeamsHookRegistrations(options.teams), ...createTelegramHookRegistrations( - withOpenClawBridgeHealthOptions(options.telegram, options.openclawBridgeHealth), + withStatusHealthOptions( + withOpenClawBridgeHealthOptions(options.telegram, options.openclawBridgeHealth), + options.statusHealth, + ), ), ...createWechatHookRegistrations(options.wechat), ]; @@ -62,3 +69,15 @@ function withOpenClawBridgeHealthOptions< }, } as T; } + +function withStatusHealthOptions< + T extends { readonly statusHealth?: ChannelStatusHealthHookOptions }, +>(options: T | undefined, statusHealth: ChannelStatusHealthHookOptions | undefined): T { + return { + ...options, + statusHealth: { + ...statusHealth, + ...options?.statusHealth, + }, + } as T; +} diff --git a/src/lib/messaging/hooks/hook-runner.test.ts b/src/lib/messaging/hooks/hook-runner.test.ts index 2c9d9cf1633..47c0ef437a6 100644 --- a/src/lib/messaging/hooks/hook-runner.test.ts +++ b/src/lib/messaging/hooks/hook-runner.test.ts @@ -47,6 +47,7 @@ describe("MessagingHookRegistry", () => { "telegram.allowlistAliases", "telegram.openclawBridgeHealth", "telegram.gatewayConflictStatus", + "telegram.statusHealth", "telegram.getMeReachability", "wechat.ilinkLogin", "wechat.seedOpenClawAccount", diff --git a/src/lib/messaging/hooks/status-runner.test.ts b/src/lib/messaging/hooks/status-runner.test.ts new file mode 100644 index 00000000000..96702c404a9 --- /dev/null +++ b/src/lib/messaging/hooks/status-runner.test.ts @@ -0,0 +1,66 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; +import { type MessagingStatusHookRunResult, readChannelHealthOutputs } from "./status-runner"; + +function runResult( + outputs: Record, +): MessagingStatusHookRunResult { + return { + channelId: "telegram", + hookId: "telegram-status-health", + outputs, + } as unknown as MessagingStatusHookRunResult; +} + +const VALID_REPORT = { + schemaVersion: 1, + channel: "telegram", + agent: "openclaw", + verdict: "healthy", + probedAt: "2026-07-15T00:00:00.000Z", + signals: [], + hints: [], +}; + +describe("readChannelHealthOutputs (#6888)", () => { + it("returns a well-formed messaging-channel-health report", () => { + const out = readChannelHealthOutputs( + runResult({ + channelHealth: { + kind: "status", + value: { type: "messaging-channel-health", report: VALID_REPORT }, + }, + }), + ); + expect(out).toEqual([VALID_REPORT]); + }); + + it("drops a malformed report (missing signals) instead of passing it through", () => { + const out = readChannelHealthOutputs( + runResult({ + channelHealth: { + kind: "status", + value: { + type: "messaging-channel-health", + report: { ...VALID_REPORT, signals: undefined }, + }, + }, + }), + ); + expect(out).toEqual([]); + }); + + it("ignores non-health status outputs (e.g. bridge conflicts)", () => { + const out = readChannelHealthOutputs( + runResult({ + bridgeHealth: { + kind: "status", + value: { type: "messaging-bridge-health", channel: "telegram", conflicts: 1 }, + }, + }), + ); + expect(out).toEqual([]); + }); +}); diff --git a/src/lib/messaging/hooks/status-runner.ts b/src/lib/messaging/hooks/status-runner.ts new file mode 100644 index 00000000000..4a0c6945247 --- /dev/null +++ b/src/lib/messaging/hooks/status-runner.ts @@ -0,0 +1,131 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Generic runner for `phase: "status"` messaging hooks. Both `nemoclaw status` + * (bridge-health/overlap outputs) and `nemoclaw channels status` + * (per-channel runtime-health output) drive channel status probing through this + * one path, so no channel-specific code lives in either command. + */ + +import { isObjectRecord } from "../../core/json-types"; +import type * as registry from "../../state/registry"; +import { createBuiltInChannelManifestRegistry } from "../channels"; +import { + type ChannelHealthReport, + MESSAGING_CHANNEL_HEALTH_OUTPUT_TYPE, +} from "../channels/channel-health"; +import type { ChannelHookSpec, MessagingAgentId, MessagingSerializableValue } from "../manifest"; +import { createBuiltInMessagingHookRegistry } from "./builtins"; +import { runMessagingHookSync } from "./hook-runner"; + +export interface MessagingStatusHookRunOptions { + readonly agent?: MessagingAgentId; + readonly agents?: ReadonlySet; + readonly channels?: ReadonlySet; + readonly currentSandbox?: string; + readonly registryEntries?: readonly registry.SandboxEntry[]; + readonly hookRegistry?: ReturnType; + /** Extra serializable inputs merged into every status hook's input map. */ + readonly extraInputs?: Readonly>; +} + +export type MessagingStatusHookRunResult = { + readonly channelId: string; + readonly hookId: string; + readonly outputs: ReturnType["outputs"]; +}; + +export function runMessagingStatusHooks( + options: MessagingStatusHookRunOptions, +): MessagingStatusHookRunResult[] { + const hookRegistry = options.hookRegistry ?? createBuiltInMessagingHookRegistry(); + const manifestRegistry = createBuiltInChannelManifestRegistry(); + const agents: ReadonlySet = options.agent + ? new Set([options.agent]) + : (options.agents ?? new Set(["openclaw"])); + const hookResults: MessagingStatusHookRunResult[] = []; + const seen = new Set(); + + for (const agent of agents) { + for (const manifest of manifestRegistry.listAvailable({ agent })) { + if (options.channels && !options.channels.has(manifest.id)) continue; + for (const hook of manifest.hooks) { + if (!shouldRunStatusHook(hook, agent)) continue; + const key = `${manifest.id}\0${hook.id}\0${hook.handler}`; + if (seen.has(key)) continue; + seen.add(key); + try { + const result = runMessagingHookSync(hook, hookRegistry, { + channelId: manifest.id, + inputs: createMessagingStatusHookInputs(options), + }); + hookResults.push({ + channelId: manifest.id, + hookId: hook.id, + outputs: result.outputs, + }); + } catch { + // Status hooks are advisory; a broken hook must not hide the rest of + // `nemoclaw status` or a channels-status probe. + } + } + } + } + return hookResults; +} + +function shouldRunStatusHook(hook: ChannelHookSpec, agent: MessagingAgentId): boolean { + return hook.phase === "status" && (!hook.agents || hook.agents.includes(agent)); +} + +function createMessagingStatusHookInputs( + options: MessagingStatusHookRunOptions, +): Record { + const inputs: Record = { ...options.extraInputs }; + if (options.currentSandbox) inputs.currentSandbox = options.currentSandbox; + if (options.registryEntries) { + inputs.registryEntries = options.registryEntries.map(serializeRegistryEntry); + } + return inputs; +} + +function serializeRegistryEntry(entry: registry.SandboxEntry): MessagingSerializableValue { + return { + name: entry.name, + gatewayName: entry.gatewayName ?? null, + messaging: entry.messaging?.plan + ? { + plan: entry.messaging.plan as unknown as MessagingSerializableValue, + } + : null, + }; +} + +/** Extract `messaging-channel-health` reports from status-hook outputs. */ +export function readChannelHealthOutputs( + result: MessagingStatusHookRunResult, +): ChannelHealthReport[] { + return Object.values(result.outputs).flatMap((output) => { + if (output.kind !== "status" || !isObjectRecord(output.value)) return []; + if (output.value.type !== MESSAGING_CHANNEL_HEALTH_OUTPUT_TYPE) return []; + // The runner only validates output kind + JSON-serializability, so + // field-check the nested report here. A malformed report is dropped (the + // caller falls back to the basic status report) rather than being cast and + // crashing the renderer downstream. + const report = output.value.report; + return isChannelHealthReport(report) ? [report] : []; + }); +} + +function isChannelHealthReport(value: unknown): value is ChannelHealthReport { + return ( + isObjectRecord(value) && + typeof value.channel === "string" && + typeof value.agent === "string" && + typeof value.verdict === "string" && + typeof value.probedAt === "string" && + Array.isArray(value.signals) && + Array.isArray(value.hints) + ); +} diff --git a/src/lib/messaging/manifest/types.ts b/src/lib/messaging/manifest/types.ts index e94a806d3ea..ec8ed6c80ef 100644 --- a/src/lib/messaging/manifest/types.ts +++ b/src/lib/messaging/manifest/types.ts @@ -46,6 +46,13 @@ export interface ChannelManifest { readonly runtime?: ChannelRuntimeByAgentSpec; readonly agentPackages?: readonly ChannelAgentPackageSpec[]; readonly hooks: readonly ChannelHookSpec[]; + /** + * Opt-in host-side health probe for `channels status --channel `. + * "log-tail": the channel ships a runtime preload that writes classified + * `[] [default] …` breadcrumbs to the gateway log and has an evaluator + * in `sandbox/-diagnostics.ts`. Absent → basic report only. + */ + readonly diagnosticsProbe?: "log-tail"; } /** Manifest-owned network policy preset metadata. */ diff --git a/src/lib/status-command-deps.ts b/src/lib/status-command-deps.ts index cd9b33d60f0..10b0ba67d94 100644 --- a/src/lib/status-command-deps.ts +++ b/src/lib/status-command-deps.ts @@ -17,13 +17,12 @@ import type { ShowStatusCommandDeps, } from "./inventory"; import { findAllOverlaps } from "./messaging/applier"; -import { createBuiltInChannelManifestRegistry } from "./messaging/channels"; -import { createBuiltInMessagingHookRegistry, runMessagingHookSync } from "./messaging/hooks"; -import type { - ChannelHookSpec, - MessagingAgentId, - MessagingSerializableValue, -} from "./messaging/manifest"; +import { createBuiltInMessagingHookRegistry } from "./messaging/hooks"; +import { + type MessagingStatusHookRunResult, + runMessagingStatusHooks, +} from "./messaging/hooks/status-runner"; +import type { MessagingAgentId } from "./messaging/manifest"; import { resolveGatewayName } from "./onboard/gateway-binding"; import * as registry from "./state/registry"; import { createSystemDeps, parseSshProcesses } from "./state/sandbox-session"; @@ -97,64 +96,6 @@ function normalizeMessagingAgentId(agent: string | null | undefined): MessagingA return agent === "hermes" ? "hermes" : "openclaw"; } -interface MessagingStatusHookRunOptions { - readonly agent?: MessagingAgentId; - readonly agents?: ReadonlySet; - readonly channels?: ReadonlySet; - readonly currentSandbox?: string; - readonly registryEntries?: readonly registry.SandboxEntry[]; - readonly hookRegistry?: ReturnType; -} - -type MessagingStatusHookRunResult = { - readonly channelId: string; - readonly hookId: string; - readonly outputs: ReturnType["outputs"]; -}; - -function runMessagingStatusHooks( - options: MessagingStatusHookRunOptions, -): MessagingStatusHookRunResult[] { - const hookRegistry = options.hookRegistry ?? createBuiltInMessagingHookRegistry(); - const manifestRegistry = createBuiltInChannelManifestRegistry(); - const agents: ReadonlySet = options.agent - ? new Set([options.agent]) - : (options.agents ?? new Set(["openclaw"])); - const hookResults: MessagingStatusHookRunResult[] = []; - const seen = new Set(); - - for (const agent of agents) { - for (const manifest of manifestRegistry.listAvailable({ agent })) { - if (options.channels && !options.channels.has(manifest.id)) continue; - for (const hook of manifest.hooks) { - if (!shouldRunStatusHook(hook, agent)) continue; - const key = `${manifest.id}\0${hook.id}\0${hook.handler}`; - if (seen.has(key)) continue; - seen.add(key); - try { - const result = runMessagingHookSync(hook, hookRegistry, { - channelId: manifest.id, - inputs: createMessagingStatusHookInputs(options), - }); - hookResults.push({ - channelId: manifest.id, - hookId: hook.id, - outputs: result.outputs, - }); - } catch { - // Status hooks are advisory; a broken hook must not hide the rest of - // `nemoclaw status`. - } - } - } - } - return hookResults; -} - -function shouldRunStatusHook(hook: ChannelHookSpec, agent: MessagingAgentId): boolean { - return hook.phase === "status" && (!hook.agents || hook.agents.includes(agent)); -} - function executeSandboxCommand( rootDir: string, openshell: string, @@ -182,29 +123,6 @@ function executeSandboxCommand( } } -function createMessagingStatusHookInputs( - options: MessagingStatusHookRunOptions, -): Record { - const inputs: Record = {}; - if (options.currentSandbox) inputs.currentSandbox = options.currentSandbox; - if (options.registryEntries) { - inputs.registryEntries = options.registryEntries.map(serializeRegistryEntry); - } - return inputs; -} - -function serializeRegistryEntry(entry: registry.SandboxEntry): MessagingSerializableValue { - return { - name: entry.name, - gatewayName: entry.gatewayName ?? null, - messaging: entry.messaging?.plan - ? { - plan: entry.messaging.plan as unknown as MessagingSerializableValue, - } - : null, - }; -} - function safeListRegistryEntries(): readonly registry.SandboxEntry[] { try { return registry.listSandboxes().sandboxes;