From 98ccb03e586ff819a4aaeb7849984f9712b3a846 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sun, 21 Jun 2026 15:32:11 -0700 Subject: [PATCH 01/13] test(e2e): migrate Telegram injection to Vitest --- .github/workflows/e2e-vitest-scenarios.yaml | 78 ++++ .../live/phase6-messaging-helpers.ts | 347 ++++++++++++++++++ .../live/telegram-injection.test.ts | 229 ++++++++++++ 3 files changed, 654 insertions(+) create mode 100644 test/e2e-scenario/live/phase6-messaging-helpers.ts create mode 100644 test/e2e-scenario/live/telegram-injection.test.ts diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index 95c4093c6a8..844109e4bba 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -4056,6 +4056,83 @@ jobs: docker logout docker.io || true rm -rf "${DOCKER_CONFIG}" + telegram-injection-vitest: + needs: generate-matrix + if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',telegram-injection-vitest,') || contains(format(',{0},', inputs.scenarios), ',telegram-injection,') }} + runs-on: ubuntu-latest + timeout-minutes: 45 + env: + FREE_STANDING_VITEST_JOB: "1" + FREE_STANDING_SCENARIO_ID: "telegram-injection" + DOCKER_CONFIG: ${{ github.workspace }}/.docker-config-telegram-injection + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/telegram-injection + NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js + NEMOCLAW_RUN_E2E_SCENARIOS: "1" + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + NEMOCLAW_SANDBOX_NAME: "e2e-telegram-injection" + OPENSHELL_GATEWAY: "nemoclaw" + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Authenticate to Docker Hub + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + shell: bash + run: | + set -euo pipefail + if [[ -z "${DOCKERHUB_USERNAME}" || -z "${DOCKERHUB_TOKEN}" ]]; then + echo "::notice::Docker Hub credentials not configured; continuing with anonymous pulls." + exit 0 + fi + mkdir -p "${DOCKER_CONFIG}" + chmod 700 "${DOCKER_CONFIG}" + echo "${DOCKERHUB_TOKEN}" | timeout 30s docker login docker.io --username "${DOCKERHUB_USERNAME}" --password-stdin || echo "::warning::Docker Hub login failed; continuing with anonymous pulls." + + - name: Set up Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 + with: + node-version: 22 + cache: npm + + - name: Install root dependencies + run: npm ci --ignore-scripts + + - name: Build CLI + run: npm run build:cli + + - name: Run Telegram injection live test + # Migrated from test/e2e/test-telegram-injection.sh. Preserves the + # real OpenShell sandbox boundary for shell metacharacter payloads, + # process-table leak checks, and validateName rejection probes. + env: + NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }} + run: | + set -euo pipefail + npx vitest run --project e2e-scenarios-live \ + test/e2e-scenario/live/telegram-injection.test.ts \ + --silent=false --reporter=default + + - name: Upload Telegram injection artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: e2e-vitest-scenarios-telegram-injection + path: e2e-artifacts/vitest/telegram-injection/ + include-hidden-files: false + if-no-files-found: ignore + retention-days: 14 + + - name: Clean up Docker auth + if: always() + run: | + set -euo pipefail + docker logout docker.io || true + rm -rf "${DOCKER_CONFIG}" + issue-2478-crash-loop-recovery-vitest: needs: generate-matrix if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',issue-2478-crash-loop-recovery-vitest,') || contains(format(',{0},', inputs.scenarios), ',issue-2478-crash-loop-recovery,') }} @@ -4226,6 +4303,7 @@ jobs: gateway-health-honest-vitest, device-auth-health-vitest, channels-add-remove-vitest, + telegram-injection-vitest, ] if: ${{ always() && github.event_name == 'workflow_dispatch' }} permissions: diff --git a/test/e2e-scenario/live/phase6-messaging-helpers.ts b/test/e2e-scenario/live/phase6-messaging-helpers.ts new file mode 100644 index 00000000000..f884b7a073d --- /dev/null +++ b/test/e2e-scenario/live/phase6-messaging-helpers.ts @@ -0,0 +1,347 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import type { HostCliClient } from "../fixtures/clients/host.ts"; +import { + type SandboxClient, + sandboxAccessEnv, + trustedSandboxShellScript, + validateSandboxName, +} from "../fixtures/clients/sandbox.ts"; +import { expect } from "../fixtures/e2e-test.ts"; +import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; +import { isNvidiaEndpointRateLimitFailure } from "./messaging-providers-helpers.ts"; + +export const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); +export const CLI = process.env.NEMOCLAW_CLI_BIN ?? path.join(REPO_ROOT, "bin", "nemoclaw.js"); +export const REGISTRY_FILE = path.join( + process.env.HOME ?? os.homedir(), + ".nemoclaw", + "sandboxes.json", +); + +export const INSTALL_TIMEOUT_MS = 45 * 60_000; +export const REBUILD_TIMEOUT_MS = 30 * 60_000; +export const COMMAND_TIMEOUT_MS = 120_000; + +export type AgentKind = "openclaw" | "hermes"; +export type JsonRecord = Record; + +export type Phase6Tokens = { + telegram: string; + discord: string; + slackBot: string; + slackApp: string; + wechat: string; +}; + +export function stripAnsi(value: string): string { + return value.replace(/\u001b\[[0-9;]*m/g, ""); +} + +export function resultText(result: Pick): string { + return [result.stdout, result.stderr].filter(Boolean).join("\n"); +} + +export function shellQuote(value: string): string { + return `'${value.replace(/'/g, `'\\''`)}'`; +} + +export function base64(value: string): string { + return Buffer.from(value, "utf8").toString("base64"); +} + +export function isFakeSlackToken(value: string): boolean { + return /^(xoxb|xapp)-(fake|test)-/.test(value); +} + +export function phase6Tokens(suffix: string): Phase6Tokens { + return { + telegram: process.env.TELEGRAM_BOT_TOKEN ?? `test-fake-telegram-token-${suffix}`, + discord: process.env.DISCORD_BOT_TOKEN ?? `test-fake-discord-token-${suffix}`, + slackBot: process.env.SLACK_BOT_TOKEN ?? `xoxb-fake-slack-token-${suffix}`, + slackApp: process.env.SLACK_APP_TOKEN ?? `xapp-fake-slack-token-${suffix}`, + wechat: process.env.WECHAT_BOT_TOKEN ?? `test-fake-wechat-token-${suffix}`, + }; +} + +export function phase6Env(options: { + sandboxName: string; + agent?: AgentKind; + apiKey?: string; + tokens?: Phase6Tokens; + extra?: NodeJS.ProcessEnv; +}): NodeJS.ProcessEnv { + validateSandboxName(options.sandboxName); + const tokens = options.tokens; + const env: NodeJS.ProcessEnv = { + ...buildAvailabilityProbeEnv(), + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_RECREATE_SANDBOX: "1", + NEMOCLAW_FRESH: "1", + NEMOCLAW_POLICY_TIER: process.env.NEMOCLAW_POLICY_TIER ?? "open", + NEMOCLAW_SANDBOX_NAME: options.sandboxName, + OPENSHELL_GATEWAY: process.env.OPENSHELL_GATEWAY ?? "nemoclaw", + ...(options.agent ? { NEMOCLAW_AGENT: options.agent } : {}), + ...(options.apiKey + ? { NVIDIA_INFERENCE_API_KEY: options.apiKey, NVIDIA_API_KEY: options.apiKey } + : {}), + ...(tokens + ? { + TELEGRAM_BOT_TOKEN: tokens.telegram, + TELEGRAM_ALLOWED_IDS: process.env.TELEGRAM_ALLOWED_IDS ?? "123456789,987654321", + TELEGRAM_REQUIRE_MENTION: process.env.TELEGRAM_REQUIRE_MENTION ?? "0", + DISCORD_BOT_TOKEN: tokens.discord, + DISCORD_SERVER_ID: process.env.DISCORD_SERVER_ID ?? "1491590992753590594", + DISCORD_SERVER_IDS: + process.env.DISCORD_SERVER_IDS ?? + process.env.DISCORD_SERVER_ID ?? + "1491590992753590594", + DISCORD_USER_ID: process.env.DISCORD_USER_ID ?? "1005536447329222676", + DISCORD_ALLOWED_IDS: + process.env.DISCORD_ALLOWED_IDS ?? process.env.DISCORD_USER_ID ?? "1005536447329222676", + DISCORD_REQUIRE_MENTION: process.env.DISCORD_REQUIRE_MENTION ?? "0", + SLACK_BOT_TOKEN: tokens.slackBot, + SLACK_APP_TOKEN: tokens.slackApp, + SLACK_ALLOWED_USERS: process.env.SLACK_ALLOWED_USERS ?? "U0123456789,U09ABCDEFGH", + WECHAT_BOT_TOKEN: tokens.wechat, + WECHAT_ACCOUNT_ID: + process.env.WECHAT_ACCOUNT_ID ?? `e2e-fake-account-${options.sandboxName}`, + WECHAT_BASE_URL: process.env.WECHAT_BASE_URL ?? "https://ilinkai.wechat.com", + WECHAT_USER_ID: process.env.WECHAT_USER_ID ?? "wxid_e2e_operator", + WECHAT_ALLOWED_IDS: + process.env.WECHAT_ALLOWED_IDS ?? process.env.WECHAT_USER_ID ?? "wxid_e2e_operator", + } + : {}), + ...options.extra, + }; + + if (tokens?.telegram.includes("fake") && !env.NEMOCLAW_SKIP_TELEGRAM_REACHABILITY) { + env.NEMOCLAW_SKIP_TELEGRAM_REACHABILITY = "1"; + } + if ( + tokens && + !env.NEMOCLAW_SKIP_SLACK_AUTH_VALIDATION && + (isFakeSlackToken(tokens.slackBot) || isFakeSlackToken(tokens.slackApp)) + ) { + env.NEMOCLAW_SKIP_SLACK_AUTH_VALIDATION = "1"; + } + return env; +} + +export function redactionValues(apiKey: string | undefined, tokens?: Phase6Tokens): string[] { + return [apiKey, ...(tokens ? Object.values(tokens) : [])].filter( + (value): value is string => typeof value === "string" && value.length > 0, + ); +} + +export async function bestEffort(run: () => Promise): Promise { + try { + await run(); + } catch { + // Cleanup and diagnostics must not hide primary test failures. + } +} + +export function expectExitZero(result: ShellProbeResult, label: string): void { + expect(result.exitCode, `${label}\n${resultText(result)}`).toBe(0); +} + +export async function precleanSandbox( + host: HostCliClient, + sandboxName: string, + env: NodeJS.ProcessEnv, + redactions: string[], + prefix: string, +): Promise { + await bestEffort(() => + host.command("node", [CLI, sandboxName, "destroy", "--yes"], { + artifactName: `${prefix}-nemoclaw-destroy`, + env, + redactionValues: redactions, + timeoutMs: 15 * 60_000, + }), + ); + await bestEffort(() => + host.command("openshell", ["sandbox", "delete", sandboxName], { + artifactName: `${prefix}-openshell-sandbox-delete`, + env, + redactionValues: redactions, + timeoutMs: 120_000, + }), + ); +} + +export async function cleanupSandbox( + host: HostCliClient, + sandboxName: string, + env: NodeJS.ProcessEnv, + redactions: string[], + prefix: string, +): Promise { + await precleanSandbox(host, sandboxName, env, redactions, prefix); +} + +export async function installSandbox( + host: HostCliClient, + env: NodeJS.ProcessEnv, + redactions: string[], + artifactName: string, +): Promise { + const result = await host.command("bash", ["install.sh", "--non-interactive"], { + artifactName, + cwd: REPO_ROOT, + env, + redactionValues: redactions, + timeoutMs: INSTALL_TIMEOUT_MS, + }); + if (result.exitCode !== 0 && isNvidiaEndpointRateLimitFailure(resultText(result))) { + throw new Error(`NVIDIA_ENDPOINT_RATE_LIMIT:${artifactName}`); + } + return result; +} + +export async function rebuildSandbox( + host: HostCliClient, + sandboxName: string, + env: NodeJS.ProcessEnv, + redactions: string[], + artifactName: string, +): Promise { + return host.command("node", [CLI, sandboxName, "rebuild", "--yes"], { + artifactName, + env, + redactionValues: redactions, + timeoutMs: REBUILD_TIMEOUT_MS, + }); +} + +export async function expectSandboxReady( + host: HostCliClient, + sandboxName: string, + env: NodeJS.ProcessEnv, + redactions: string[], + artifactName: string, +): Promise { + const list = await host.command("openshell", ["sandbox", "list"], { + artifactName, + env, + redactionValues: redactions, + timeoutMs: 60_000, + }); + expectExitZero(list, "openshell sandbox list"); + const row = stripAnsi(list.stdout) + .split(/\r?\n/) + .find((line) => line.includes(sandboxName)); + expect(row, resultText(list)).toMatch(/\bReady\b/i); +} + +export function readRegistryEntry(sandboxName: string): JsonRecord { + expect(fs.existsSync(REGISTRY_FILE), `${REGISTRY_FILE} missing`).toBe(true); + const registry = JSON.parse(fs.readFileSync(REGISTRY_FILE, "utf8")) as { + sandboxes?: Record; + }; + const entry = registry.sandboxes?.[sandboxName]; + expect(entry, `registry entry ${sandboxName} missing`).toBeTruthy(); + if (!entry) throw new Error(`registry entry ${sandboxName} missing`); + return entry; +} + +export function messagingPlan(sandboxName: string): JsonRecord { + const messaging = readRegistryEntry(sandboxName).messaging; + expect(messaging && typeof messaging === "object", "registry messaging state missing").toBe(true); + const plan = (messaging as JsonRecord).plan; + expect(plan && typeof plan === "object", "registry messaging.plan missing").toBe(true); + if (!plan || typeof plan !== "object") throw new Error("registry messaging.plan missing"); + return plan as JsonRecord; +} + +export function arrayRecords(value: unknown): JsonRecord[] { + return Array.isArray(value) + ? value.filter((item): item is JsonRecord => Boolean(item) && typeof item === "object") + : []; +} + +export function stringArray(value: unknown): string[] { + return Array.isArray(value) + ? value.filter((item): item is string => typeof item === "string") + : []; +} + +export async function sandboxSh( + sandbox: SandboxClient, + sandboxName: string, + script: string, + options: { + artifactName: string; + redactionValues?: string[]; + timeoutMs?: number; + }, +): Promise { + return sandbox.execShell(sandboxName, trustedSandboxShellScript(script), { + artifactName: options.artifactName, + env: sandboxAccessEnv(), + redactionValues: options.redactionValues ?? [], + timeoutMs: options.timeoutMs ?? COMMAND_TIMEOUT_MS, + }); +} + +export async function sandboxEncodedSh( + sandbox: SandboxClient, + sandboxName: string, + script: string, + args: string[], + options: { + artifactName: string; + redactionValues?: string[]; + timeoutMs?: number; + }, +): Promise { + const command = [ + "tmp=$(mktemp)", + "trap 'rm -f \"$tmp\"' EXIT", + `printf %s ${shellQuote(base64(script))} | base64 -d > "$tmp"`, + `sh "$tmp" ${args.map(shellQuote).join(" ")}`, + ].join("; "); + return sandboxSh(sandbox, sandboxName, command, options); +} + +export async function sandboxNode( + sandbox: SandboxClient, + sandboxName: string, + source: string, + env: Record, + options: { + artifactName: string; + redactionValues?: string[]; + timeoutMs?: number; + }, +): Promise { + const exports = Object.entries(env) + .map(([key, value]) => `export ${key}=${shellQuote(value)}`) + .join("\n"); + return sandboxEncodedSh( + sandbox, + sandboxName, + `${exports}\nnode --input-type=module <<'NODE'\n${source}\nNODE\n`, + [], + options, + ); +} + +export async function dockerInfo( + host: HostCliClient, + env: NodeJS.ProcessEnv, +): Promise { + return host.command("docker", ["info"], { + artifactName: "phase6-docker-info", + env, + timeoutMs: 30_000, + }); +} diff --git a/test/e2e-scenario/live/telegram-injection.test.ts b/test/e2e-scenario/live/telegram-injection.test.ts new file mode 100644 index 00000000000..156852c70d1 --- /dev/null +++ b/test/e2e-scenario/live/telegram-injection.test.ts @@ -0,0 +1,229 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** Live Vitest replacement for test/e2e/test-telegram-injection.sh. */ + +import path from "node:path"; + +import { expect, test } from "../fixtures/e2e-test.ts"; +import { shouldRunLiveE2EScenarios } from "../fixtures/live-project-gate.ts"; +import { + bestEffort, + CLI, + COMMAND_TIMEOUT_MS, + cleanupSandbox, + dockerInfo, + expectExitZero, + expectSandboxReady, + installSandbox, + phase6Env, + REPO_ROOT, + redactionValues, + resultText, + sandboxSh, + shellQuote, +} from "./phase6-messaging-helpers.ts"; + +const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-telegram-injection"; +const LIVE_TIMEOUT_MS = 35 * 60_000; + +function base64(value: string): string { + return Buffer.from(value, "utf8").toString("base64"); +} + +function openshellStdinCommand(payload: string, remoteShell: string): string { + return [ + "set -euo pipefail", + `printf %s ${shellQuote(base64(payload))} | base64 -d | openshell sandbox exec --name ${shellQuote(SANDBOX_NAME)} -- sh -lc ${shellQuote(remoteShell)}`, + ].join("; "); +} + +async function sendPayloadViaSandboxStdin( + host: import("../fixtures/clients/host.ts").HostCliClient, + payload: string, + remoteShell: string, + env: NodeJS.ProcessEnv, + artifactName: string, + redactions: string[], +) { + return host.command("bash", ["-lc", openshellStdinCommand(payload, remoteShell)], { + artifactName, + env, + redactionValues: redactions, + timeoutMs: COMMAND_TIMEOUT_MS, + }); +} + +test.skipIf(!shouldRunLiveE2EScenarios())( + "Telegram bridge-style message handling treats shell metacharacters as data", + { timeout: LIVE_TIMEOUT_MS }, + async ({ artifacts, cleanup, host, sandbox, secrets, skip }) => { + const apiKey = secrets.required("NVIDIA_INFERENCE_API_KEY"); + const env = phase6Env({ sandboxName: SANDBOX_NAME, agent: "openclaw", apiKey }); + const redactions = redactionValues(apiKey); + + await artifacts.writeJson("scenario.json", { + id: "telegram-injection", + legacySource: "test/e2e/test-telegram-injection.sh", + boundary: + "install.sh OpenClaw sandbox + OpenShell sandbox exec stdin path + process table and validateName probes", + sandboxName: SANDBOX_NAME, + contracts: [ + "command substitution payloads are literal input and do not create files", + "parameter expansion does not leak NVIDIA_INFERENCE_API_KEY", + "host and sandbox process tables do not expose the API key after setup", + "invalid sandbox names with shell metacharacters are rejected by validateName", + "normal messages and benign special characters still pass through", + ], + }); + + cleanup.add(`destroy telegram injection sandbox ${SANDBOX_NAME}`, () => + cleanupSandbox(host, SANDBOX_NAME, env, redactions, "cleanup-telegram-injection"), + ); + await cleanupSandbox(host, SANDBOX_NAME, env, redactions, "preclean-telegram-injection"); + + const docker = await dockerInfo(host, env); + expect(docker.exitCode, resultText(docker)).toBe(0); + + try { + const install = await installSandbox(host, env, redactions, "install-telegram-injection"); + expectExitZero(install, "install.sh --non-interactive"); + } catch (error) { + if (String(error).includes("NVIDIA_ENDPOINT_RATE_LIMIT")) { + skip( + "NVIDIA endpoint validation was rate-limited before Telegram injection assertions ran", + ); + return; + } + throw error; + } + await expectSandboxReady( + host, + SANDBOX_NAME, + env, + redactions, + "sandbox-list-telegram-injection", + ); + + for (const [label, marker, payload] of [ + [ + "command-substitution", + "/tmp/injection-proof-t1", + "$(touch /tmp/injection-proof-t1 && echo INJECTED)", + ], + ["backtick-substitution", "/tmp/injection-proof-t2", "`touch /tmp/injection-proof-t2`"], + [ + "single-quote-breakout", + "/tmp/injection-proof-t3", + "'; touch /tmp/injection-proof-t3; echo '", + ], + ] as const) { + await sandboxSh(sandbox, SANDBOX_NAME, `rm -f ${shellQuote(marker)}`, { + artifactName: `remove-${label}-marker`, + redactionValues: redactions, + }); + const send = await sendPayloadViaSandboxStdin( + host, + payload, + 'MSG=$(cat) && echo "Received: $MSG"', + env, + `send-${label}-payload`, + redactions, + ); + expect(send.exitCode, resultText(send)).toBe(0); + const markerCheck = await sandboxSh( + sandbox, + SANDBOX_NAME, + `test -f ${shellQuote(marker)} && echo EXPLOITED || echo SAFE`, + { artifactName: `check-${label}-marker`, redactionValues: redactions }, + ); + expectExitZero(markerCheck, `check ${label} marker`); + expect(markerCheck.stdout.trim(), resultText(markerCheck)).toBe("SAFE"); + } + + const envPayload = await sendPayloadViaSandboxStdin( + host, + "${NVIDIA_INFERENCE_API_KEY}", + 'MSG=$(cat) && echo "$MSG"', + env, + "parameter-expansion-payload", + redactions, + ); + expect(envPayload.exitCode, resultText(envPayload)).toBe(0); + expect(resultText(envPayload)).not.toContain(apiKey.slice(0, 15)); + expect(resultText(envPayload)).toContain("${NVIDIA_INFERENCE_API_KEY}"); + + const apiKeyPrefix = apiKey.slice(0, 15); + const hostPs = await host.command("ps", ["aux"], { + artifactName: "host-process-table-telegram-injection", + env, + redactionValues: redactions, + timeoutMs: 30_000, + }); + const sandboxPs = await sandboxSh(sandbox, SANDBOX_NAME, "ps aux", { + artifactName: "sandbox-process-table-telegram-injection", + redactionValues: redactions, + }); + expect(resultText(hostPs)).not.toContain(apiKeyPrefix); + expect(resultText(sandboxPs)).not.toContain(apiKeyPrefix); + + const invalidNames = [ + "foo;rm -rf /", + "--help", + "$(whoami)", + "`id`", + "foo bar", + "../etc/passwd", + "UPPERCASE", + ]; + for (const invalidName of invalidNames) { + const validation = await host.command( + "node", + [ + "-e", + `const { validateName } = require(${JSON.stringify(path.join(REPO_ROOT, "dist/lib/runner"))});\ntry { validateName(process.argv[1], "SANDBOX_NAME"); console.log("ACCEPTED"); } catch (error) { console.log("REJECTED:" + error.message); }`, + invalidName, + ], + { + artifactName: `validate-name-${invalidName.replace(/[^a-z0-9]+/gi, "-")}`, + env, + redactionValues: redactions, + timeoutMs: 30_000, + }, + ); + expectExitZero(validation, `validateName ${invalidName}`); + expect(validation.stdout, invalidName).toContain("REJECTED"); + } + + const normal = await sendPayloadViaSandboxStdin( + host, + "Hello, what is two plus two?", + 'MSG=$(cat) && echo "Received: $MSG"', + env, + "normal-message-passthrough", + redactions, + ); + expect(normal.exitCode, resultText(normal)).toBe(0); + expect(resultText(normal)).toContain("Hello, what is two plus two?"); + + const special = await sendPayloadViaSandboxStdin( + host, + "What's the meaning of life? It costs $5 & is 100% free!", + 'MSG=$(cat) && echo "$MSG"', + env, + "special-message-passthrough", + redactions, + ); + expect(special.exitCode, resultText(special)).toBe(0); + expect(resultText(special).trim()).not.toBe(""); + + await bestEffort(() => + host.command("node", [CLI, SANDBOX_NAME, "status"], { + artifactName: "post-assert-status-telegram-injection", + env, + redactionValues: redactions, + timeoutMs: 60_000, + }), + ); + }, +); From 7a1acdf8683e6d95b04d4a6685a60a67b32c26ce Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sun, 21 Jun 2026 18:19:33 -0700 Subject: [PATCH 02/13] test(e2e): keep Telegram injection test linear --- .../live/phase6-messaging-helpers.ts | 18 +++++++++++++++ .../live/telegram-injection.test.ts | 23 ++++++++----------- 2 files changed, 28 insertions(+), 13 deletions(-) diff --git a/test/e2e-scenario/live/phase6-messaging-helpers.ts b/test/e2e-scenario/live/phase6-messaging-helpers.ts index f884b7a073d..d4e195c6119 100644 --- a/test/e2e-scenario/live/phase6-messaging-helpers.ts +++ b/test/e2e-scenario/live/phase6-messaging-helpers.ts @@ -207,6 +207,24 @@ export async function installSandbox( return result; } +export async function installSandboxOrSkipOnRateLimit( + host: HostCliClient, + env: NodeJS.ProcessEnv, + redactions: string[], + artifactName: string, + skip: (note?: string) => never, + skipMessage: string, +): Promise { + try { + return await installSandbox(host, env, redactions, artifactName); + } catch (error) { + if (String(error).includes("NVIDIA_ENDPOINT_RATE_LIMIT")) { + skip(skipMessage); + } + throw error; + } +} + export async function rebuildSandbox( host: HostCliClient, sandboxName: string, diff --git a/test/e2e-scenario/live/telegram-injection.test.ts b/test/e2e-scenario/live/telegram-injection.test.ts index 156852c70d1..5c52ed0cb3c 100644 --- a/test/e2e-scenario/live/telegram-injection.test.ts +++ b/test/e2e-scenario/live/telegram-injection.test.ts @@ -15,7 +15,7 @@ import { dockerInfo, expectExitZero, expectSandboxReady, - installSandbox, + installSandboxOrSkipOnRateLimit, phase6Env, REPO_ROOT, redactionValues, @@ -85,18 +85,15 @@ test.skipIf(!shouldRunLiveE2EScenarios())( const docker = await dockerInfo(host, env); expect(docker.exitCode, resultText(docker)).toBe(0); - try { - const install = await installSandbox(host, env, redactions, "install-telegram-injection"); - expectExitZero(install, "install.sh --non-interactive"); - } catch (error) { - if (String(error).includes("NVIDIA_ENDPOINT_RATE_LIMIT")) { - skip( - "NVIDIA endpoint validation was rate-limited before Telegram injection assertions ran", - ); - return; - } - throw error; - } + const install = await installSandboxOrSkipOnRateLimit( + host, + env, + redactions, + "install-telegram-injection", + skip, + "NVIDIA endpoint validation was rate-limited before Telegram injection assertions ran", + ); + expectExitZero(install, "install.sh --non-interactive"); await expectSandboxReady( host, SANDBOX_NAME, From e46a00b59dee19999f8d71090662d6af8db06a84 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sun, 21 Jun 2026 18:03:34 -0700 Subject: [PATCH 03/13] test(e2e): migrate channels stop/start to Vitest --- .github/workflows/e2e-vitest-scenarios.yaml | 89 +++++ .../live/channels-stop-start-helpers.ts | 345 ++++++++++++++++++ .../live/channels-stop-start.test.ts | 18 + 3 files changed, 452 insertions(+) create mode 100644 test/e2e-scenario/live/channels-stop-start-helpers.ts create mode 100644 test/e2e-scenario/live/channels-stop-start.test.ts diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index 844109e4bba..e2178b349c6 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -4133,6 +4133,94 @@ jobs: docker logout docker.io || true rm -rf "${DOCKER_CONFIG}" + channels-stop-start-vitest: + needs: generate-matrix + if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',channels-stop-start-vitest,') || contains(format(',{0},', inputs.scenarios), ',channels-stop-start,') }} + runs-on: ubuntu-latest + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + agent: [openclaw, hermes] + env: + FREE_STANDING_VITEST_JOB: "1" + FREE_STANDING_SCENARIO_ID: "channels-stop-start" + DOCKER_CONFIG: ${{ github.workspace }}/.docker-config-channels-stop-start-${{ matrix.agent }} + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/channels-stop-start/${{ matrix.agent }} + NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js + NEMOCLAW_RUN_E2E_SCENARIOS: "1" + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + NEMOCLAW_AGENT: ${{ matrix.agent }} + NEMOCLAW_CHANNELS_STOP_START_AGENT: ${{ matrix.agent }} + NEMOCLAW_SANDBOX_NAME: e2e-channels-stop-start-${{ matrix.agent }} + OPENSHELL_GATEWAY: "nemoclaw" + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Authenticate to Docker Hub + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + shell: bash + run: | + set -euo pipefail + if [[ -z "${DOCKERHUB_USERNAME}" || -z "${DOCKERHUB_TOKEN}" ]]; then + echo "::notice::Docker Hub credentials not configured; continuing with anonymous pulls." + exit 0 + fi + mkdir -p "${DOCKER_CONFIG}" + chmod 700 "${DOCKER_CONFIG}" + echo "${DOCKERHUB_TOKEN}" | timeout 30s docker login docker.io --username "${DOCKERHUB_USERNAME}" --password-stdin || echo "::warning::Docker Hub login failed; continuing with anonymous pulls." + + - name: Set up Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 + with: + node-version: 22 + cache: npm + + - name: Install root dependencies + run: npm ci --ignore-scripts + + - name: Build CLI + run: npm run build:cli + + - name: Run channels stop/start live test + # Migrated from test/e2e/test-channels-stop-start.sh. Preserves the + # OpenClaw/Hermes messaging channel stop/start, rebuild, provider + # reuse, registry, policy-list, and in-sandbox config contracts. + env: + NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }} + TELEGRAM_BOT_TOKEN: test-fake-telegram-token-stop-start-${{ matrix.agent }} + DISCORD_BOT_TOKEN: test-fake-discord-token-stop-start-${{ matrix.agent }} + SLACK_BOT_TOKEN: xoxb-fake-slack-token-stop-start-${{ matrix.agent }} + SLACK_APP_TOKEN: xapp-fake-slack-token-stop-start-${{ matrix.agent }} + WECHAT_BOT_TOKEN: test-fake-wechat-token-stop-start-${{ matrix.agent }} + run: | + set -euo pipefail + npx vitest run --project e2e-scenarios-live \ + test/e2e-scenario/live/channels-stop-start.test.ts \ + --silent=false --reporter=default + + - name: Upload channels stop/start artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: e2e-vitest-scenarios-channels-stop-start-${{ matrix.agent }} + path: e2e-artifacts/vitest/channels-stop-start/${{ matrix.agent }}/ + include-hidden-files: false + if-no-files-found: ignore + retention-days: 14 + + - name: Clean up Docker auth + if: always() + run: | + set -euo pipefail + docker logout docker.io || true + rm -rf "${DOCKER_CONFIG}" + issue-2478-crash-loop-recovery-vitest: needs: generate-matrix if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',issue-2478-crash-loop-recovery-vitest,') || contains(format(',{0},', inputs.scenarios), ',issue-2478-crash-loop-recovery,') }} @@ -4304,6 +4392,7 @@ jobs: device-auth-health-vitest, channels-add-remove-vitest, telegram-injection-vitest, + channels-stop-start-vitest, ] if: ${{ always() && github.event_name == 'workflow_dispatch' }} permissions: diff --git a/test/e2e-scenario/live/channels-stop-start-helpers.ts b/test/e2e-scenario/live/channels-stop-start-helpers.ts new file mode 100644 index 00000000000..d660803ba19 --- /dev/null +++ b/test/e2e-scenario/live/channels-stop-start-helpers.ts @@ -0,0 +1,345 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** Live Vitest replacement for test/e2e/test-channels-stop-start.sh. */ + +import { expect } from "../fixtures/e2e-test.ts"; +import { + type AgentKind, + arrayRecords, + cleanupSandbox, + dockerInfo, + expectExitZero, + expectSandboxReady, + installSandboxOrSkipOnRateLimit, + messagingPlan, + phase6Env, + phase6Tokens, + rebuildSandbox, + redactionValues, + resultText, + sandboxSh, + stringArray, +} from "./phase6-messaging-helpers.ts"; + +const AGENT = (process.env.NEMOCLAW_CHANNELS_STOP_START_AGENT ?? + process.env.NEMOCLAW_AGENT ?? + "openclaw") as AgentKind; +if (AGENT !== "openclaw" && AGENT !== "hermes") { + throw new Error(`NEMOCLAW_CHANNELS_STOP_START_AGENT must be openclaw or hermes, got ${AGENT}`); +} +const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? `e2e-channels-stop-start-${AGENT}`; +const CHANNELS = ["telegram", "discord", "wechat", "slack", "whatsapp"] as const; +const PROVIDERS: Record string[]> = { + telegram: (sandbox) => [`${sandbox}-telegram-bridge`], + discord: (sandbox) => [`${sandbox}-discord-bridge`], + wechat: (sandbox) => [`${sandbox}-wechat-bridge`], + slack: (sandbox) => [`${sandbox}-slack-bridge`, `${sandbox}-slack-app`], + whatsapp: () => [], +}; +export const LIVE_TIMEOUT_MS = 80 * 60_000; + +type ChannelState = "active" | "disabled"; + +function planChannel(channelId: string) { + return arrayRecords(messagingPlan(SANDBOX_NAME).channels).find( + (channel) => channel.channelId === channelId, + ); +} + +function expectPlanChannelState(channelId: string, expected: ChannelState): void { + const plan = messagingPlan(SANDBOX_NAME); + const channel = planChannel(channelId); + expect(channel, `${channelId} missing from messaging.plan.channels`).toBeTruthy(); + expect(channel?.configured, `${channelId} configured`).toBe(true); + expect(plan.sandboxName, "messaging.plan.sandboxName").toBe(SANDBOX_NAME); + expect(plan.agent, "messaging.plan.agent").toBe(AGENT); + + const disabledChannels = stringArray(plan.disabledChannels); + if (expected === "active") { + expect(channel?.active, `${channelId} active`).toBe(true); + expect(channel?.disabled, `${channelId} disabled unexpectedly`).not.toBe(true); + expect(disabledChannels, `${channelId} unexpectedly disabled`).not.toContain(channelId); + } else { + expect(channel?.disabled, `${channelId} disabled`).toBe(true); + expect(channel?.active, `${channelId} active unexpectedly`).not.toBe(true); + expect(disabledChannels, `${channelId} missing from disabledChannels`).toContain(channelId); + } + + const networkPolicy = + plan.networkPolicy && typeof plan.networkPolicy === "object" + ? (plan.networkPolicy as Record) + : {}; + expect(stringArray(networkPolicy.presets), `${channelId} policy preset`).toContain(channelId); + expect( + arrayRecords(networkPolicy.entries).some((entry) => entry.channelId === channelId), + `${channelId} policy entry`, + ).toBe(true); + const credentialBindings = arrayRecords(plan.credentialBindings); + if (channelId !== "whatsapp") { + expect( + credentialBindings.some((entry) => entry.channelId === channelId), + `${channelId} credential binding`, + ).toBe(true); + } + expect(Object.hasOwn(plan, "agentRender"), "messaging.plan.agentRender should not persist").toBe( + false, + ); +} + +function expectChannelInputs(): void { + const expected: Record> = { + telegram: { + allowedIds: process.env.TELEGRAM_ALLOWED_IDS ?? "123456789,987654321", + requireMention: process.env.TELEGRAM_REQUIRE_MENTION ?? "0", + }, + discord: { + serverId: process.env.DISCORD_SERVER_ID ?? "1491590992753590594", + userId: process.env.DISCORD_USER_ID ?? "1005536447329222676", + requireMention: process.env.DISCORD_REQUIRE_MENTION ?? "0", + }, + slack: { allowedUsers: process.env.SLACK_ALLOWED_USERS ?? "U0123456789,U09ABCDEFGH" }, + wechat: { + allowedIds: + process.env.WECHAT_ALLOWED_IDS ?? process.env.WECHAT_USER_ID ?? "wxid_e2e_operator", + }, + }; + for (const [channelId, inputs] of Object.entries(expected)) { + const channel = planChannel(channelId); + const planInputs = arrayRecords(channel?.inputs); + for (const [inputId, value] of Object.entries(inputs)) { + expect( + planInputs.find((input) => input.inputId === inputId)?.value, + `${channelId}.${inputId}`, + ).toBe(value); + } + } +} + +function openClawChannelKey(channel: string): string { + return channel === "wechat" ? "openclaw-weixin" : channel; +} + +async function agentConfigContains( + sandbox: import("../fixtures/clients/sandbox.ts").SandboxClient, + channel: string, + redactions: string[], +): Promise { + if (AGENT === "openclaw") { + const result = await sandboxSh( + sandbox, + SANDBOX_NAME, + `python3 - <<'PY'\nimport json\nchannel=${JSON.stringify(openClawChannelKey(channel))}\ncfg=json.load(open('/sandbox/.openclaw/openclaw.json'))\nprint('yes' if channel in cfg.get('channels', {}) else 'no')\nPY`, + { artifactName: `config-channel-${AGENT}-${channel}`, redactionValues: redactions }, + ); + expectExitZero(result, `read OpenClaw channel ${channel}`); + return result.stdout.trim() === "yes"; + } + + const probes: Record = { + telegram: + 'grep -Eq "^TELEGRAM_BOT_TOKEN=openshell:resolve:env:TELEGRAM_BOT_TOKEN$" /sandbox/.hermes/.env', + discord: + 'grep -Eq "^DISCORD_BOT_TOKEN=openshell:resolve:env:DISCORD_BOT_TOKEN$" /sandbox/.hermes/.env', + wechat: + 'grep -Eq "^WEIXIN_TOKEN=openshell:resolve:env:WECHAT_BOT_TOKEN$" /sandbox/.hermes/.env', + slack: + 'grep -Eq "^SLACK_BOT_TOKEN=xoxb-OPENSHELL-RESOLVE-ENV-SLACK_BOT_TOKEN$" /sandbox/.hermes/.env && grep -Eq "^SLACK_APP_TOKEN=xapp-OPENSHELL-RESOLVE-ENV-SLACK_APP_TOKEN$" /sandbox/.hermes/.env', + whatsapp: + 'grep -Eq "^WHATSAPP_ENABLED=true$" /sandbox/.hermes/.env && grep -Eq "^WHATSAPP_MODE=bot$" /sandbox/.hermes/.env', + }; + const result = await sandboxSh( + sandbox, + SANDBOX_NAME, + `if [ -r /sandbox/.hermes/.env ] && ${probes[channel]}; then echo yes; else echo no; fi`, + { artifactName: `config-channel-${AGENT}-${channel}`, redactionValues: redactions }, + ); + expectExitZero(result, `read Hermes channel ${channel}`); + return result.stdout.trim() === "yes"; +} + +async function expectAgentConfig( + sandbox: import("../fixtures/clients/sandbox.ts").SandboxClient, + expected: "present" | "absent", + redactions: string[], +): Promise { + for (const channel of CHANNELS) { + const present = await agentConfigContains(sandbox, channel, redactions); + expect(present, `${AGENT}/${channel} config ${expected}`).toBe(expected === "present"); + } +} + +async function expectProvidersExist( + host: import("../fixtures/clients/host.ts").HostCliClient, + env: NodeJS.ProcessEnv, + redactions: string[], + context: string, +): Promise { + for (const channel of CHANNELS) { + for (const provider of PROVIDERS[channel](SANDBOX_NAME)) { + const result = await host.command("openshell", ["provider", "get", provider], { + artifactName: `provider-${provider}-${context}`, + env, + redactionValues: redactions, + timeoutMs: 60_000, + }); + expectExitZero(result, `${provider} exists ${context}`); + } + } +} + +async function policyPresetActive( + host: import("../fixtures/clients/host.ts").HostCliClient, + env: NodeJS.ProcessEnv, + redactions: string[], + channel: string, +): Promise { + const result = await host.command( + "node", + [process.env.NEMOCLAW_CLI_BIN ?? "bin/nemoclaw.js", SANDBOX_NAME, "policy-list"], + { + artifactName: `policy-list-${channel}-${AGENT}`, + env, + redactionValues: redactions, + timeoutMs: 60_000, + }, + ); + expectExitZero(result, `policy-list ${channel}`); + return resultText(result).includes(`● ${channel}`); +} + +async function runChannelCommand( + host: import("../fixtures/clients/host.ts").HostCliClient, + env: NodeJS.ProcessEnv, + redactions: string[], + action: "add" | "stop" | "start", + channel: string, +): Promise { + const result = await host.command( + "node", + [process.env.NEMOCLAW_CLI_BIN ?? "bin/nemoclaw.js", SANDBOX_NAME, "channels", action, channel], + { + artifactName: `channels-${action}-${channel}-${AGENT}`, + env, + redactionValues: redactions, + timeoutMs: 10 * 60_000, + }, + ); + expectExitZero(result, `channels ${action} ${channel}`); + const expectedText = + action === "add" + ? `Enabled ${channel} channel` + : `Marked ${channel} ${action === "stop" ? "disabled" : "enabled"}`; + expect(resultText(result)).toContain(expectedText); +} + +export const CHANNELS_STOP_START_TEST_NAME = `${AGENT} channels stop/start preserves credentials and toggles runtime config`; + +export async function runChannelsStopStartScenario({ + artifacts, + cleanup, + host, + sandbox, + secrets, + skip, +}: import("../fixtures/e2e-test.ts").E2EScenarioFixtures & { + skip: (note?: string) => never; +}): Promise { + const apiKey = secrets.required("NVIDIA_INFERENCE_API_KEY"); + const tokens = phase6Tokens(AGENT); + const env = phase6Env({ sandboxName: SANDBOX_NAME, agent: AGENT, apiKey, tokens }); + const redactions = redactionValues(apiKey, tokens); + + await artifacts.writeJson("scenario.json", { + id: "channels-stop-start", + legacySource: "test/e2e/test-channels-stop-start.sh", + boundary: + "install.sh messaging onboard + channels stop/start CLI + rebuild + sandbox config probes", + agent: AGENT, + sandboxName: SANDBOX_NAME, + channels: CHANNELS, + }); + + cleanup.add(`destroy channels stop/start sandbox ${SANDBOX_NAME}`, () => + cleanupSandbox(host, SANDBOX_NAME, env, redactions, `cleanup-channels-stop-start-${AGENT}`), + ); + await cleanupSandbox( + host, + SANDBOX_NAME, + env, + redactions, + `preclean-channels-stop-start-${AGENT}`, + ); + + const docker = await dockerInfo(host, env); + expect(docker.exitCode, resultText(docker)).toBe(0); + const install = await installSandboxOrSkipOnRateLimit( + host, + env, + redactions, + `install-channels-stop-start-${AGENT}`, + skip, + "NVIDIA endpoint validation was rate-limited before channel lifecycle assertions ran", + ); + expectExitZero(install, `${AGENT} install.sh`); + await expectSandboxReady( + host, + SANDBOX_NAME, + env, + redactions, + `sandbox-list-channels-stop-start-${AGENT}`, + ); + + if (!planChannel("whatsapp")) { + await runChannelCommand(host, env, redactions, "add", "whatsapp"); + const rebuild = await rebuildSandbox( + host, + SANDBOX_NAME, + env, + redactions, + `rebuild-add-whatsapp-${AGENT}`, + ); + expectExitZero(rebuild, "rebuild after adding WhatsApp"); + } + + expectChannelInputs(); + for (const channel of CHANNELS) expectPlanChannelState(channel, "active"); + await expectAgentConfig(sandbox, "present", redactions); + await expectProvidersExist(host, env, redactions, "baseline"); + for (const channel of CHANNELS) { + expect( + await policyPresetActive(host, env, redactions, channel), + `${channel} policy active`, + ).toBe(true); + } + + for (const channel of CHANNELS) await runChannelCommand(host, env, redactions, "stop", channel); + expectChannelInputs(); + for (const channel of CHANNELS) expectPlanChannelState(channel, "disabled"); + const stopRebuild = await rebuildSandbox( + host, + SANDBOX_NAME, + env, + redactions, + `rebuild-stop-all-${AGENT}`, + ); + expectExitZero(stopRebuild, "rebuild after stopping all channels"); + await expectAgentConfig(sandbox, "absent", redactions); + await expectProvidersExist(host, env, redactions, "after-stop"); + for (const channel of CHANNELS) expectPlanChannelState(channel, "disabled"); + + for (const channel of CHANNELS) await runChannelCommand(host, env, redactions, "start", channel); + expectChannelInputs(); + for (const channel of CHANNELS) expectPlanChannelState(channel, "active"); + const startRebuild = await rebuildSandbox( + host, + SANDBOX_NAME, + env, + redactions, + `rebuild-start-all-${AGENT}`, + ); + expectExitZero(startRebuild, "rebuild after starting all channels"); + await expectAgentConfig(sandbox, "present", redactions); + await expectProvidersExist(host, env, redactions, "after-start"); + for (const channel of CHANNELS) expectPlanChannelState(channel, "active"); +} diff --git a/test/e2e-scenario/live/channels-stop-start.test.ts b/test/e2e-scenario/live/channels-stop-start.test.ts new file mode 100644 index 00000000000..b2d356f26cb --- /dev/null +++ b/test/e2e-scenario/live/channels-stop-start.test.ts @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** Live Vitest replacement for test/e2e/test-channels-stop-start.sh. */ + +import { test } from "../fixtures/e2e-test.ts"; +import { shouldRunLiveE2EScenarios } from "../fixtures/live-project-gate.ts"; +import { + CHANNELS_STOP_START_TEST_NAME, + LIVE_TIMEOUT_MS, + runChannelsStopStartScenario, +} from "./channels-stop-start-helpers.ts"; + +test.skipIf(!shouldRunLiveE2EScenarios())( + CHANNELS_STOP_START_TEST_NAME, + { timeout: LIVE_TIMEOUT_MS }, + runChannelsStopStartScenario, +); From d45654f648765dd63eff409524f7a8e6f7d82f47 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sun, 21 Jun 2026 20:39:17 -0700 Subject: [PATCH 04/13] test(e2e): address Telegram injection review feedback --- .github/workflows/e2e-vitest-scenarios.yaml | 38 +++- .../live/phase6-messaging-helpers.ts | 170 +----------------- .../live/telegram-injection.test.ts | 102 +++++++---- 3 files changed, 111 insertions(+), 199 deletions(-) diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index 844109e4bba..58166c45ac4 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -4064,7 +4064,6 @@ jobs: env: FREE_STANDING_VITEST_JOB: "1" FREE_STANDING_SCENARIO_ID: "telegram-injection" - DOCKER_CONFIG: ${{ github.workspace }}/.docker-config-telegram-injection E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/telegram-injection NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_RUN_E2E_SCENARIOS: "1" @@ -4077,6 +4076,9 @@ jobs: with: persist-credentials: false + - name: Configure isolated Docker auth directory + run: echo "DOCKER_CONFIG=${RUNNER_TEMP}/docker-config-telegram-injection" >> "$GITHUB_ENV" + - name: Authenticate to Docker Hub env: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} @@ -4090,7 +4092,20 @@ jobs: fi mkdir -p "${DOCKER_CONFIG}" chmod 700 "${DOCKER_CONFIG}" - echo "${DOCKERHUB_TOKEN}" | timeout 30s docker login docker.io --username "${DOCKERHUB_USERNAME}" --password-stdin || echo "::warning::Docker Hub login failed; continuing with anonymous pulls." + login_succeeded=0 + for attempt in 1 2 3; do + if echo "${DOCKERHUB_TOKEN}" | timeout 30s docker login docker.io --username "${DOCKERHUB_USERNAME}" --password-stdin; then + login_succeeded=1 + break + fi + if [[ "$attempt" -lt 3 ]]; then + echo "::warning::Docker Hub login attempt ${attempt} failed; retrying." + sleep 5 + fi + done + if [[ "$login_succeeded" -ne 1 ]]; then + echo "::warning::Docker Hub login failed after 3 attempts; continuing with anonymous pulls." + fi - name: Set up Node uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 @@ -4104,6 +4119,13 @@ jobs: - name: Build CLI run: npm run build:cli + - name: Install OpenShell + env: + NEMOCLAW_NON_INTERACTIVE: "1" + run: | + set -euo pipefail + env -u DOCKER_CONFIG -u DOCKERHUB_USERNAME -u DOCKERHUB_TOKEN -u NVIDIA_API_KEY -u GITHUB_TOKEN bash scripts/install-openshell.sh + - name: Run Telegram injection live test # Migrated from test/e2e/test-telegram-injection.sh. Preserves the # real OpenShell sandbox boundary for shell metacharacter payloads, @@ -4112,6 +4134,18 @@ jobs: NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }} run: | set -euo pipefail + export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH" + if command -v openshell >/dev/null 2>&1; then + OPENSHELL_BIN="$(command -v openshell)" + elif [ -x "$HOME/.local/bin/openshell" ]; then + OPENSHELL_BIN="$HOME/.local/bin/openshell" + else + echo "::error::OpenShell CLI not found after install" + ls -la /usr/local/bin/openshell "$HOME/.local/bin/openshell" 2>&1 || true + exit 1 + fi + export OPENSHELL_BIN + "$OPENSHELL_BIN" --version npx vitest run --project e2e-scenarios-live \ test/e2e-scenario/live/telegram-injection.test.ts \ --silent=false --reporter=default diff --git a/test/e2e-scenario/live/phase6-messaging-helpers.ts b/test/e2e-scenario/live/phase6-messaging-helpers.ts index d4e195c6119..f78c0a16d93 100644 --- a/test/e2e-scenario/live/phase6-messaging-helpers.ts +++ b/test/e2e-scenario/live/phase6-messaging-helpers.ts @@ -1,8 +1,6 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import fs from "node:fs"; -import os from "node:os"; import path from "node:path"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; @@ -19,26 +17,11 @@ import { isNvidiaEndpointRateLimitFailure } from "./messaging-providers-helpers. export const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); export const CLI = process.env.NEMOCLAW_CLI_BIN ?? path.join(REPO_ROOT, "bin", "nemoclaw.js"); -export const REGISTRY_FILE = path.join( - process.env.HOME ?? os.homedir(), - ".nemoclaw", - "sandboxes.json", -); export const INSTALL_TIMEOUT_MS = 45 * 60_000; -export const REBUILD_TIMEOUT_MS = 30 * 60_000; export const COMMAND_TIMEOUT_MS = 120_000; export type AgentKind = "openclaw" | "hermes"; -export type JsonRecord = Record; - -export type Phase6Tokens = { - telegram: string; - discord: string; - slackBot: string; - slackApp: string; - wechat: string; -}; export function stripAnsi(value: string): string { return value.replace(/\u001b\[[0-9;]*m/g, ""); @@ -56,30 +39,14 @@ export function base64(value: string): string { return Buffer.from(value, "utf8").toString("base64"); } -export function isFakeSlackToken(value: string): boolean { - return /^(xoxb|xapp)-(fake|test)-/.test(value); -} - -export function phase6Tokens(suffix: string): Phase6Tokens { - return { - telegram: process.env.TELEGRAM_BOT_TOKEN ?? `test-fake-telegram-token-${suffix}`, - discord: process.env.DISCORD_BOT_TOKEN ?? `test-fake-discord-token-${suffix}`, - slackBot: process.env.SLACK_BOT_TOKEN ?? `xoxb-fake-slack-token-${suffix}`, - slackApp: process.env.SLACK_APP_TOKEN ?? `xapp-fake-slack-token-${suffix}`, - wechat: process.env.WECHAT_BOT_TOKEN ?? `test-fake-wechat-token-${suffix}`, - }; -} - export function phase6Env(options: { sandboxName: string; agent?: AgentKind; apiKey?: string; - tokens?: Phase6Tokens; extra?: NodeJS.ProcessEnv; }): NodeJS.ProcessEnv { validateSandboxName(options.sandboxName); - const tokens = options.tokens; - const env: NodeJS.ProcessEnv = { + return { ...buildAvailabilityProbeEnv(), NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", NEMOCLAW_NON_INTERACTIVE: "1", @@ -92,53 +59,12 @@ export function phase6Env(options: { ...(options.apiKey ? { NVIDIA_INFERENCE_API_KEY: options.apiKey, NVIDIA_API_KEY: options.apiKey } : {}), - ...(tokens - ? { - TELEGRAM_BOT_TOKEN: tokens.telegram, - TELEGRAM_ALLOWED_IDS: process.env.TELEGRAM_ALLOWED_IDS ?? "123456789,987654321", - TELEGRAM_REQUIRE_MENTION: process.env.TELEGRAM_REQUIRE_MENTION ?? "0", - DISCORD_BOT_TOKEN: tokens.discord, - DISCORD_SERVER_ID: process.env.DISCORD_SERVER_ID ?? "1491590992753590594", - DISCORD_SERVER_IDS: - process.env.DISCORD_SERVER_IDS ?? - process.env.DISCORD_SERVER_ID ?? - "1491590992753590594", - DISCORD_USER_ID: process.env.DISCORD_USER_ID ?? "1005536447329222676", - DISCORD_ALLOWED_IDS: - process.env.DISCORD_ALLOWED_IDS ?? process.env.DISCORD_USER_ID ?? "1005536447329222676", - DISCORD_REQUIRE_MENTION: process.env.DISCORD_REQUIRE_MENTION ?? "0", - SLACK_BOT_TOKEN: tokens.slackBot, - SLACK_APP_TOKEN: tokens.slackApp, - SLACK_ALLOWED_USERS: process.env.SLACK_ALLOWED_USERS ?? "U0123456789,U09ABCDEFGH", - WECHAT_BOT_TOKEN: tokens.wechat, - WECHAT_ACCOUNT_ID: - process.env.WECHAT_ACCOUNT_ID ?? `e2e-fake-account-${options.sandboxName}`, - WECHAT_BASE_URL: process.env.WECHAT_BASE_URL ?? "https://ilinkai.wechat.com", - WECHAT_USER_ID: process.env.WECHAT_USER_ID ?? "wxid_e2e_operator", - WECHAT_ALLOWED_IDS: - process.env.WECHAT_ALLOWED_IDS ?? process.env.WECHAT_USER_ID ?? "wxid_e2e_operator", - } - : {}), ...options.extra, }; - - if (tokens?.telegram.includes("fake") && !env.NEMOCLAW_SKIP_TELEGRAM_REACHABILITY) { - env.NEMOCLAW_SKIP_TELEGRAM_REACHABILITY = "1"; - } - if ( - tokens && - !env.NEMOCLAW_SKIP_SLACK_AUTH_VALIDATION && - (isFakeSlackToken(tokens.slackBot) || isFakeSlackToken(tokens.slackApp)) - ) { - env.NEMOCLAW_SKIP_SLACK_AUTH_VALIDATION = "1"; - } - return env; } -export function redactionValues(apiKey: string | undefined, tokens?: Phase6Tokens): string[] { - return [apiKey, ...(tokens ? Object.values(tokens) : [])].filter( - (value): value is string => typeof value === "string" && value.length > 0, - ); +export function redactionValues(apiKey: string | undefined): string[] { + return [apiKey].filter((value): value is string => typeof value === "string" && value.length > 0); } export async function bestEffort(run: () => Promise): Promise { @@ -225,21 +151,6 @@ export async function installSandboxOrSkipOnRateLimit( } } -export async function rebuildSandbox( - host: HostCliClient, - sandboxName: string, - env: NodeJS.ProcessEnv, - redactions: string[], - artifactName: string, -): Promise { - return host.command("node", [CLI, sandboxName, "rebuild", "--yes"], { - artifactName, - env, - redactionValues: redactions, - timeoutMs: REBUILD_TIMEOUT_MS, - }); -} - export async function expectSandboxReady( host: HostCliClient, sandboxName: string, @@ -260,38 +171,6 @@ export async function expectSandboxReady( expect(row, resultText(list)).toMatch(/\bReady\b/i); } -export function readRegistryEntry(sandboxName: string): JsonRecord { - expect(fs.existsSync(REGISTRY_FILE), `${REGISTRY_FILE} missing`).toBe(true); - const registry = JSON.parse(fs.readFileSync(REGISTRY_FILE, "utf8")) as { - sandboxes?: Record; - }; - const entry = registry.sandboxes?.[sandboxName]; - expect(entry, `registry entry ${sandboxName} missing`).toBeTruthy(); - if (!entry) throw new Error(`registry entry ${sandboxName} missing`); - return entry; -} - -export function messagingPlan(sandboxName: string): JsonRecord { - const messaging = readRegistryEntry(sandboxName).messaging; - expect(messaging && typeof messaging === "object", "registry messaging state missing").toBe(true); - const plan = (messaging as JsonRecord).plan; - expect(plan && typeof plan === "object", "registry messaging.plan missing").toBe(true); - if (!plan || typeof plan !== "object") throw new Error("registry messaging.plan missing"); - return plan as JsonRecord; -} - -export function arrayRecords(value: unknown): JsonRecord[] { - return Array.isArray(value) - ? value.filter((item): item is JsonRecord => Boolean(item) && typeof item === "object") - : []; -} - -export function stringArray(value: unknown): string[] { - return Array.isArray(value) - ? value.filter((item): item is string => typeof item === "string") - : []; -} - export async function sandboxSh( sandbox: SandboxClient, sandboxName: string, @@ -310,49 +189,6 @@ export async function sandboxSh( }); } -export async function sandboxEncodedSh( - sandbox: SandboxClient, - sandboxName: string, - script: string, - args: string[], - options: { - artifactName: string; - redactionValues?: string[]; - timeoutMs?: number; - }, -): Promise { - const command = [ - "tmp=$(mktemp)", - "trap 'rm -f \"$tmp\"' EXIT", - `printf %s ${shellQuote(base64(script))} | base64 -d > "$tmp"`, - `sh "$tmp" ${args.map(shellQuote).join(" ")}`, - ].join("; "); - return sandboxSh(sandbox, sandboxName, command, options); -} - -export async function sandboxNode( - sandbox: SandboxClient, - sandboxName: string, - source: string, - env: Record, - options: { - artifactName: string; - redactionValues?: string[]; - timeoutMs?: number; - }, -): Promise { - const exports = Object.entries(env) - .map(([key, value]) => `export ${key}=${shellQuote(value)}`) - .join("\n"); - return sandboxEncodedSh( - sandbox, - sandboxName, - `${exports}\nnode --input-type=module <<'NODE'\n${source}\nNODE\n`, - [], - options, - ); -} - export async function dockerInfo( host: HostCliClient, env: NodeJS.ProcessEnv, diff --git a/test/e2e-scenario/live/telegram-injection.test.ts b/test/e2e-scenario/live/telegram-injection.test.ts index 5c52ed0cb3c..c9db270223d 100644 --- a/test/e2e-scenario/live/telegram-injection.test.ts +++ b/test/e2e-scenario/live/telegram-injection.test.ts @@ -8,6 +8,7 @@ import path from "node:path"; import { expect, test } from "../fixtures/e2e-test.ts"; import { shouldRunLiveE2EScenarios } from "../fixtures/live-project-gate.ts"; import { + base64, bestEffort, CLI, COMMAND_TIMEOUT_MS, @@ -27,10 +28,6 @@ import { const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-telegram-injection"; const LIVE_TIMEOUT_MS = 35 * 60_000; -function base64(value: string): string { - return Buffer.from(value, "utf8").toString("base64"); -} - function openshellStdinCommand(payload: string, remoteShell: string): string { return [ "set -euo pipefail", @@ -54,6 +51,74 @@ async function sendPayloadViaSandboxStdin( }); } +async function assertParameterPayloadStaysLiteral( + host: import("../fixtures/clients/host.ts").HostCliClient, + env: NodeJS.ProcessEnv, + redactions: string[], +): Promise { + const payload = "${NVIDIA_INFERENCE_API_KEY}"; + const command = [ + "set -euo pipefail", + 'prefix="${NVIDIA_INFERENCE_API_KEY:0:15}"', + 'out="$(printf %s "${PAYLOAD_B64}" | base64 -d | openshell sandbox exec --name "${SANDBOX_NAME}" -- sh -lc \'MSG=$(cat) && echo "$MSG"\' 2>&1)"', + 'if printf %s "$out" | grep -Fq "$prefix"; then echo SECRET_LEAK; exit 20; fi', + 'printf "%s\\n" "$out"', + ].join("; "); + const result = await host.command("bash", ["-lc", command], { + artifactName: "parameter-expansion-payload", + env: { ...env, PAYLOAD_B64: base64(payload), SANDBOX_NAME }, + redactionValues: redactions, + timeoutMs: COMMAND_TIMEOUT_MS, + }); + expect(result.exitCode, resultText(result)).toBe(0); + expect(resultText(result)).not.toContain("SECRET_LEAK"); + expect(resultText(result)).toContain(payload); +} + +async function assertHostProcessTableDoesNotExposeSecret( + host: import("../fixtures/clients/host.ts").HostCliClient, + env: NodeJS.ProcessEnv, + redactions: string[], +): Promise { + const command = [ + "set -euo pipefail", + 'prefix="${NVIDIA_INFERENCE_API_KEY:0:15}"', + 'matches="$(ps aux 2>/dev/null | grep -F "$prefix" | grep -v grep || true)"', + 'if [ -n "$matches" ]; then echo SECRET_LEAK; exit 21; fi', + "echo SECRET_ABSENT", + ].join("; "); + const result = await host.command("bash", ["-lc", command], { + artifactName: "host-process-table-telegram-injection", + env, + redactionValues: redactions, + timeoutMs: 30_000, + }); + expect(result.exitCode, resultText(result)).toBe(0); + expect(result.stdout.trim(), resultText(result)).toBe("SECRET_ABSENT"); +} + +async function assertSandboxProcessTableDoesNotExposeSecret( + host: import("../fixtures/clients/host.ts").HostCliClient, + env: NodeJS.ProcessEnv, + redactions: string[], +): Promise { + const command = [ + "set -euo pipefail", + 'prefix="${NVIDIA_INFERENCE_API_KEY:0:15}"', + 'out="$(openshell sandbox exec --name "${SANDBOX_NAME}" -- sh -lc \'ps aux\' 2>&1)"', + 'if printf %s "$out" | grep -Fq "$prefix"; then echo SECRET_LEAK; exit 22; fi', + "echo SECRET_ABSENT", + ].join("; "); + const result = await host.command("bash", ["-lc", command], { + artifactName: "sandbox-process-table-telegram-injection", + env: { ...env, SANDBOX_NAME }, + redactionValues: redactions, + timeoutMs: COMMAND_TIMEOUT_MS, + }); + expect(result.exitCode, resultText(result)).toBe(0); + expect(result.stdout.trim(), resultText(result)).toBe("SECRET_ABSENT"); +} + test.skipIf(!shouldRunLiveE2EScenarios())( "Telegram bridge-style message handling treats shell metacharacters as data", { timeout: LIVE_TIMEOUT_MS }, @@ -138,31 +203,9 @@ test.skipIf(!shouldRunLiveE2EScenarios())( expect(markerCheck.stdout.trim(), resultText(markerCheck)).toBe("SAFE"); } - const envPayload = await sendPayloadViaSandboxStdin( - host, - "${NVIDIA_INFERENCE_API_KEY}", - 'MSG=$(cat) && echo "$MSG"', - env, - "parameter-expansion-payload", - redactions, - ); - expect(envPayload.exitCode, resultText(envPayload)).toBe(0); - expect(resultText(envPayload)).not.toContain(apiKey.slice(0, 15)); - expect(resultText(envPayload)).toContain("${NVIDIA_INFERENCE_API_KEY}"); - - const apiKeyPrefix = apiKey.slice(0, 15); - const hostPs = await host.command("ps", ["aux"], { - artifactName: "host-process-table-telegram-injection", - env, - redactionValues: redactions, - timeoutMs: 30_000, - }); - const sandboxPs = await sandboxSh(sandbox, SANDBOX_NAME, "ps aux", { - artifactName: "sandbox-process-table-telegram-injection", - redactionValues: redactions, - }); - expect(resultText(hostPs)).not.toContain(apiKeyPrefix); - expect(resultText(sandboxPs)).not.toContain(apiKeyPrefix); + await assertParameterPayloadStaysLiteral(host, env, redactions); + await assertHostProcessTableDoesNotExposeSecret(host, env, redactions); + await assertSandboxProcessTableDoesNotExposeSecret(host, env, redactions); const invalidNames = [ "foo;rm -rf /", @@ -171,7 +214,6 @@ test.skipIf(!shouldRunLiveE2EScenarios())( "`id`", "foo bar", "../etc/passwd", - "UPPERCASE", ]; for (const invalidName of invalidNames) { const validation = await host.command( From 539947ad24d41ea8576187a51c062b1c90973360 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Mon, 22 Jun 2026 09:20:01 -0400 Subject: [PATCH 05/13] test(e2e): restore stop/start assertions --- .../live/channels-stop-start-helpers.ts | 36 ++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/test/e2e-scenario/live/channels-stop-start-helpers.ts b/test/e2e-scenario/live/channels-stop-start-helpers.ts index d660803ba19..70cbe14319b 100644 --- a/test/e2e-scenario/live/channels-stop-start-helpers.ts +++ b/test/e2e-scenario/live/channels-stop-start-helpers.ts @@ -49,7 +49,8 @@ function planChannel(channelId: string) { function expectPlanChannelState(channelId: string, expected: ChannelState): void { const plan = messagingPlan(SANDBOX_NAME); - const channel = planChannel(channelId); + const channels = arrayRecords(plan.channels); + const channel = channels.find((entry) => entry.channelId === channelId); expect(channel, `${channelId} missing from messaging.plan.channels`).toBeTruthy(); expect(channel?.configured, `${channelId} configured`).toBe(true); expect(plan.sandboxName, "messaging.plan.sandboxName").toBe(SANDBOX_NAME); @@ -85,6 +86,10 @@ function expectPlanChannelState(channelId: string, expected: ChannelState): void expect(Object.hasOwn(plan, "agentRender"), "messaging.plan.agentRender should not persist").toBe( false, ); + expect( + channels.some((entry) => Object.hasOwn(entry, "hooks")), + "messaging.plan.channels hooks should not persist", + ).toBe(false); } function expectChannelInputs(): void { @@ -188,6 +193,34 @@ async function expectProvidersExist( } } +async function precleanProviders( + host: import("../fixtures/clients/host.ts").HostCliClient, + env: NodeJS.ProcessEnv, + redactions: string[], + context: string, +): Promise { + for (const channel of CHANNELS) { + for (const provider of PROVIDERS[channel](SANDBOX_NAME)) { + await host.command("openshell", ["provider", "delete", provider], { + artifactName: `provider-delete-${provider}-${context}`, + env, + redactionValues: redactions, + timeoutMs: 60_000, + }); + const result = await host.command("openshell", ["provider", "get", provider], { + artifactName: `provider-absent-${provider}-${context}`, + env, + redactionValues: redactions, + timeoutMs: 60_000, + }); + expect( + result.exitCode, + `${provider} absent after provider pre-clean\n${resultText(result)}`, + ).not.toBe(0); + } + } +} + async function policyPresetActive( host: import("../fixtures/clients/host.ts").HostCliClient, env: NodeJS.ProcessEnv, @@ -270,6 +303,7 @@ export async function runChannelsStopStartScenario({ redactions, `preclean-channels-stop-start-${AGENT}`, ); + await precleanProviders(host, env, redactions, `preclean-channels-stop-start-${AGENT}`); const docker = await dockerInfo(host, env); expect(docker.exitCode, resultText(docker)).toBe(0); From 8d489b39f497752dd1f5fd083a07a46791c19144 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Mon, 22 Jun 2026 09:45:00 -0400 Subject: [PATCH 06/13] test(e2e): restore schema assertions --- .../live/channels-stop-start-helpers.ts | 133 +++++++++++++++++- 1 file changed, 126 insertions(+), 7 deletions(-) diff --git a/test/e2e-scenario/live/channels-stop-start-helpers.ts b/test/e2e-scenario/live/channels-stop-start-helpers.ts index 70cbe14319b..48fa7b1556f 100644 --- a/test/e2e-scenario/live/channels-stop-start-helpers.ts +++ b/test/e2e-scenario/live/channels-stop-start-helpers.ts @@ -3,23 +3,22 @@ /** Live Vitest replacement for test/e2e/test-channels-stop-start.sh. */ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + import { expect } from "../fixtures/e2e-test.ts"; import { type AgentKind, - arrayRecords, + CLI, cleanupSandbox, dockerInfo, expectExitZero, expectSandboxReady, installSandboxOrSkipOnRateLimit, - messagingPlan, phase6Env, - phase6Tokens, - rebuildSandbox, - redactionValues, resultText, sandboxSh, - stringArray, } from "./phase6-messaging-helpers.ts"; const AGENT = (process.env.NEMOCLAW_CHANNELS_STOP_START_AGENT ?? @@ -29,6 +28,7 @@ if (AGENT !== "openclaw" && AGENT !== "hermes") { throw new Error(`NEMOCLAW_CHANNELS_STOP_START_AGENT must be openclaw or hermes, got ${AGENT}`); } const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? `e2e-channels-stop-start-${AGENT}`; +const REGISTRY_FILE = path.join(process.env.HOME ?? os.homedir(), ".nemoclaw", "sandboxes.json"); const CHANNELS = ["telegram", "discord", "wechat", "slack", "whatsapp"] as const; const PROVIDERS: Record string[]> = { telegram: (sandbox) => [`${sandbox}-telegram-bridge`], @@ -40,6 +40,105 @@ const PROVIDERS: Record string[]> = { export const LIVE_TIMEOUT_MS = 80 * 60_000; type ChannelState = "active" | "disabled"; +type JsonRecord = Record; +type Phase6Tokens = { + telegram: string; + discord: string; + slackBot: string; + slackApp: string; + wechat: string; +}; + +function phase6Tokens(suffix: string): Phase6Tokens { + return { + telegram: process.env.TELEGRAM_BOT_TOKEN ?? `test-fake-telegram-token-${suffix}`, + discord: process.env.DISCORD_BOT_TOKEN ?? `test-fake-discord-token-${suffix}`, + slackBot: process.env.SLACK_BOT_TOKEN ?? `xoxb-fake-slack-token-${suffix}`, + slackApp: process.env.SLACK_APP_TOKEN ?? `xapp-fake-slack-token-${suffix}`, + wechat: process.env.WECHAT_BOT_TOKEN ?? `test-fake-wechat-token-${suffix}`, + }; +} + +function phase6TokenEnv(tokens: Phase6Tokens): NodeJS.ProcessEnv { + const env: NodeJS.ProcessEnv = { + TELEGRAM_BOT_TOKEN: tokens.telegram, + TELEGRAM_ALLOWED_IDS: process.env.TELEGRAM_ALLOWED_IDS ?? "123456789,987654321", + TELEGRAM_REQUIRE_MENTION: process.env.TELEGRAM_REQUIRE_MENTION ?? "0", + DISCORD_BOT_TOKEN: tokens.discord, + DISCORD_SERVER_ID: process.env.DISCORD_SERVER_ID ?? "1491590992753590594", + DISCORD_SERVER_IDS: + process.env.DISCORD_SERVER_IDS ?? process.env.DISCORD_SERVER_ID ?? "1491590992753590594", + DISCORD_USER_ID: process.env.DISCORD_USER_ID ?? "1005536447329222676", + DISCORD_ALLOWED_IDS: + process.env.DISCORD_ALLOWED_IDS ?? process.env.DISCORD_USER_ID ?? "1005536447329222676", + DISCORD_REQUIRE_MENTION: process.env.DISCORD_REQUIRE_MENTION ?? "0", + SLACK_BOT_TOKEN: tokens.slackBot, + SLACK_APP_TOKEN: tokens.slackApp, + SLACK_ALLOWED_USERS: process.env.SLACK_ALLOWED_USERS ?? "U0123456789,U09ABCDEFGH", + WECHAT_BOT_TOKEN: tokens.wechat, + WECHAT_ACCOUNT_ID: process.env.WECHAT_ACCOUNT_ID ?? `e2e-fake-account-${SANDBOX_NAME}`, + WECHAT_BASE_URL: process.env.WECHAT_BASE_URL ?? "https://ilinkai.wechat.com", + WECHAT_USER_ID: process.env.WECHAT_USER_ID ?? "wxid_e2e_operator", + WECHAT_ALLOWED_IDS: + process.env.WECHAT_ALLOWED_IDS ?? process.env.WECHAT_USER_ID ?? "wxid_e2e_operator", + }; + if (tokens.telegram.includes("fake")) env.NEMOCLAW_SKIP_TELEGRAM_REACHABILITY = "1"; + if ( + /^(xoxb|xapp)-(fake|test)-/.test(tokens.slackBot) || + /^(xoxb|xapp)-(fake|test)-/.test(tokens.slackApp) + ) { + env.NEMOCLAW_SKIP_SLACK_AUTH_VALIDATION = "1"; + } + return env; +} + +function redactionValues(apiKey: string | undefined, tokens: Phase6Tokens): string[] { + return [apiKey, ...Object.values(tokens)].filter( + (value): value is string => typeof value === "string" && value.length > 0, + ); +} + +function arrayRecords(value: unknown): JsonRecord[] { + return Array.isArray(value) + ? value.filter((item): item is JsonRecord => Boolean(item) && typeof item === "object") + : []; +} + +function stringArray(value: unknown): string[] { + return Array.isArray(value) + ? value.filter((item): item is string => typeof item === "string") + : []; +} + +function readRegistryEntry(sandboxName: string): JsonRecord { + expect(fs.existsSync(REGISTRY_FILE), `${REGISTRY_FILE} missing`).toBe(true); + const registry = JSON.parse(fs.readFileSync(REGISTRY_FILE, "utf8")) as { + sandboxes?: Record; + }; + const entry = registry.sandboxes?.[sandboxName]; + expect(entry, `registry entry ${sandboxName} missing`).toBeTruthy(); + if (!entry) throw new Error(`registry entry ${sandboxName} missing`); + return entry; +} + +function messagingState(sandboxName: string): JsonRecord { + const messaging = readRegistryEntry(sandboxName).messaging; + expect(messaging && typeof messaging === "object", "registry messaging state missing").toBe(true); + if (!messaging || typeof messaging !== "object") + throw new Error("registry messaging state missing"); + const state = messaging as JsonRecord; + expect(state.schemaVersion, "messaging.schemaVersion").toBe(1); + return state; +} + +function messagingPlan(sandboxName: string): JsonRecord { + const plan = messagingState(sandboxName).plan; + expect(plan && typeof plan === "object", "registry messaging.plan missing").toBe(true); + if (!plan || typeof plan !== "object") throw new Error("registry messaging.plan missing"); + const record = plan as JsonRecord; + expect(record.schemaVersion, "messaging.plan.schemaVersion").toBe(1); + return record; +} function planChannel(channelId: string) { return arrayRecords(messagingPlan(SANDBOX_NAME).channels).find( @@ -221,6 +320,21 @@ async function precleanProviders( } } +async function rebuildSandbox( + host: import("../fixtures/clients/host.ts").HostCliClient, + sandboxName: string, + env: NodeJS.ProcessEnv, + redactions: string[], + artifactName: string, +) { + return host.command("node", [CLI, sandboxName, "rebuild", "--yes"], { + artifactName, + env, + redactionValues: redactions, + timeoutMs: 30 * 60_000, + }); +} + async function policyPresetActive( host: import("../fixtures/clients/host.ts").HostCliClient, env: NodeJS.ProcessEnv, @@ -280,7 +394,12 @@ export async function runChannelsStopStartScenario({ }): Promise { const apiKey = secrets.required("NVIDIA_INFERENCE_API_KEY"); const tokens = phase6Tokens(AGENT); - const env = phase6Env({ sandboxName: SANDBOX_NAME, agent: AGENT, apiKey, tokens }); + const env = phase6Env({ + sandboxName: SANDBOX_NAME, + agent: AGENT, + apiKey, + extra: phase6TokenEnv(tokens), + }); const redactions = redactionValues(apiKey, tokens); await artifacts.writeJson("scenario.json", { From e25c500f988cf90ab305af63623524aa380e0165 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Mon, 22 Jun 2026 09:59:20 -0400 Subject: [PATCH 07/13] ci(e2e): install OpenShell for stop-start --- .github/workflows/e2e-vitest-scenarios.yaml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index 1fe533d841a..19c57bea756 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -4221,6 +4221,13 @@ jobs: - name: Build CLI run: npm run build:cli + - name: Install OpenShell + env: + NEMOCLAW_NON_INTERACTIVE: "1" + run: | + set -euo pipefail + env -u DOCKER_CONFIG -u DOCKERHUB_USERNAME -u DOCKERHUB_TOKEN -u NVIDIA_API_KEY -u GITHUB_TOKEN bash scripts/install-openshell.sh + - name: Run channels stop/start live test # Migrated from test/e2e/test-channels-stop-start.sh. Preserves the # OpenClaw/Hermes messaging channel stop/start, rebuild, provider @@ -4234,6 +4241,18 @@ jobs: WECHAT_BOT_TOKEN: test-fake-wechat-token-stop-start-${{ matrix.agent }} run: | set -euo pipefail + export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH" + if command -v openshell >/dev/null 2>&1; then + OPENSHELL_BIN="$(command -v openshell)" + elif [ -x "$HOME/.local/bin/openshell" ]; then + OPENSHELL_BIN="$HOME/.local/bin/openshell" + else + echo "::error::OpenShell CLI not found after install" + ls -la /usr/local/bin/openshell "$HOME/.local/bin/openshell" 2>&1 || true + exit 1 + fi + export OPENSHELL_BIN + "$OPENSHELL_BIN" --version npx vitest run --project e2e-scenarios-live \ test/e2e-scenario/live/channels-stop-start.test.ts \ --silent=false --reporter=default From 9291b6a9d3368f176cc9a13e0774d7211851524f Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Mon, 22 Jun 2026 10:08:21 -0400 Subject: [PATCH 08/13] test(e2e): raise workflow boundary timeout --- .../e2e-scenarios-workflow.test.ts | 46 ++++++++++--------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts index 2da7d41c134..f4c24668210 100644 --- a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts +++ b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts @@ -685,27 +685,31 @@ jobs: } }); - it("keeps each free-standing scenario out of the registry matrix", { timeout: 120_000 }, () => { - const inventory = readFreeStandingJobsInventory(); - for (const job of inventory.allowedJobs) { - expect(generateMatrixForDispatch({ JOBS: job, SCENARIOS: "" })).toMatchObject({ - hermes_selected: job === "hermes-e2e-vitest" ? "true" : "false", - matrix: "[]", - }); - } - for (const [scenario, job] of inventory.scenarioToJob) { - expect(generateMatrixForDispatch({ JOBS: "", SCENARIOS: scenario })).toMatchObject({ - hermes_selected: scenario === "hermes-e2e" ? "true" : "false", - matrix: "[]", - }); - expect(evaluateE2eVitestWorkflowDispatchSelectors({ scenarios: scenario })).toMatchObject({ - valid: true, - liveScenariosRuns: false, - selectedFreeStandingJobs: [job], - registryScenarios: [], - }); - } - }); + it( + "keeps each free-standing scenario out of the registry matrix", + testTimeoutOptions(240_000), + () => { + const inventory = readFreeStandingJobsInventory(); + for (const job of inventory.allowedJobs) { + expect(generateMatrixForDispatch({ JOBS: job, SCENARIOS: "" })).toMatchObject({ + hermes_selected: job === "hermes-e2e-vitest" ? "true" : "false", + matrix: "[]", + }); + } + for (const [scenario, job] of inventory.scenarioToJob) { + expect(generateMatrixForDispatch({ JOBS: "", SCENARIOS: scenario })).toMatchObject({ + hermes_selected: scenario === "hermes-e2e" ? "true" : "false", + matrix: "[]", + }); + expect(evaluateE2eVitestWorkflowDispatchSelectors({ scenarios: scenario })).toMatchObject({ + valid: true, + liveScenariosRuns: false, + selectedFreeStandingJobs: [job], + registryScenarios: [], + }); + } + }, + ); it("flags direct dispatch-input interpolation and unsafe artifact upload", () => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-vitest-workflow-")); From 1f2f20b390c10db14a811edcae9c529d368f1679 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Mon, 22 Jun 2026 11:12:30 -0400 Subject: [PATCH 09/13] test(e2e): avoid multiline OpenShell exec arg --- test/e2e-scenario/live/channels-stop-start-helpers.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/e2e-scenario/live/channels-stop-start-helpers.ts b/test/e2e-scenario/live/channels-stop-start-helpers.ts index 48fa7b1556f..0080505997a 100644 --- a/test/e2e-scenario/live/channels-stop-start-helpers.ts +++ b/test/e2e-scenario/live/channels-stop-start-helpers.ts @@ -19,6 +19,7 @@ import { phase6Env, resultText, sandboxSh, + shellQuote, } from "./phase6-messaging-helpers.ts"; const AGENT = (process.env.NEMOCLAW_CHANNELS_STOP_START_AGENT ?? @@ -233,7 +234,11 @@ async function agentConfigContains( const result = await sandboxSh( sandbox, SANDBOX_NAME, - `python3 - <<'PY'\nimport json\nchannel=${JSON.stringify(openClawChannelKey(channel))}\ncfg=json.load(open('/sandbox/.openclaw/openclaw.json'))\nprint('yes' if channel in cfg.get('channels', {}) else 'no')\nPY`, + `python3 -c ${shellQuote( + `import json; channel=${JSON.stringify( + openClawChannelKey(channel), + )}; cfg=json.load(open('/sandbox/.openclaw/openclaw.json')); print('yes' if channel in cfg.get('channels', {}) else 'no')`, + )}`, { artifactName: `config-channel-${AGENT}-${channel}`, redactionValues: redactions }, ); expectExitZero(result, `read OpenClaw channel ${channel}`); From e528822fab883c550a7197bfd20dc5ed85ef0eb7 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Mon, 22 Jun 2026 11:28:29 -0400 Subject: [PATCH 10/13] test(e2e): guard stop-start sandbox cleanup --- .../live/channels-stop-start-helpers.ts | 2 ++ .../e2e-scenario/live/channels-stop-start-safety.ts | 12 ++++++++++++ .../support-tests/e2e-scenarios-workflow.test.ts | 13 +++++++++++++ 3 files changed, 27 insertions(+) create mode 100644 test/e2e-scenario/live/channels-stop-start-safety.ts diff --git a/test/e2e-scenario/live/channels-stop-start-helpers.ts b/test/e2e-scenario/live/channels-stop-start-helpers.ts index 0080505997a..df151adc5e3 100644 --- a/test/e2e-scenario/live/channels-stop-start-helpers.ts +++ b/test/e2e-scenario/live/channels-stop-start-helpers.ts @@ -8,6 +8,7 @@ import os from "node:os"; import path from "node:path"; import { expect } from "../fixtures/e2e-test.ts"; +import { assertChannelsStopStartSandboxName } from "./channels-stop-start-safety.ts"; import { type AgentKind, CLI, @@ -29,6 +30,7 @@ if (AGENT !== "openclaw" && AGENT !== "hermes") { throw new Error(`NEMOCLAW_CHANNELS_STOP_START_AGENT must be openclaw or hermes, got ${AGENT}`); } const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? `e2e-channels-stop-start-${AGENT}`; +assertChannelsStopStartSandboxName(SANDBOX_NAME); const REGISTRY_FILE = path.join(process.env.HOME ?? os.homedir(), ".nemoclaw", "sandboxes.json"); const CHANNELS = ["telegram", "discord", "wechat", "slack", "whatsapp"] as const; const PROVIDERS: Record string[]> = { diff --git a/test/e2e-scenario/live/channels-stop-start-safety.ts b/test/e2e-scenario/live/channels-stop-start-safety.ts new file mode 100644 index 00000000000..2e65c146528 --- /dev/null +++ b/test/e2e-scenario/live/channels-stop-start-safety.ts @@ -0,0 +1,12 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export const CHANNELS_STOP_START_SANDBOX_PREFIX = "e2e-channels-stop-start-"; + +export function assertChannelsStopStartSandboxName(sandboxName: string): void { + if (!sandboxName.startsWith(CHANNELS_STOP_START_SANDBOX_PREFIX)) { + throw new Error( + `channels-stop-start live test is destructive and only accepts sandbox names with prefix ${CHANNELS_STOP_START_SANDBOX_PREFIX}; got ${sandboxName}`, + ); + } +} diff --git a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts index f4c24668210..5d1527c1ea7 100644 --- a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts +++ b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts @@ -15,6 +15,7 @@ import { validateFreeStandingWorkflowInventory, } from "../../../tools/e2e-scenarios/workflow-boundary.mts"; import { testTimeoutOptions } from "../../helpers/timeouts"; +import { assertChannelsStopStartSandboxName } from "../live/channels-stop-start-safety.ts"; function readWorkflow(): Record { return YAML.parse( @@ -72,6 +73,18 @@ function generateMatrixForDispatch(env: { } describe("e2e-vitest-scenarios workflow boundary", () => { + it("guards channels-stop-start destructive cleanup to test-owned sandboxes", () => { + expect(() => assertChannelsStopStartSandboxName("personal-dev")).toThrow( + /only accepts sandbox names with prefix e2e-channels-stop-start-/, + ); + expect(() => + assertChannelsStopStartSandboxName("e2e-channels-stop-start-openclaw"), + ).not.toThrow(); + expect(() => + assertChannelsStopStartSandboxName("e2e-channels-stop-start-hermes"), + ).not.toThrow(); + }); + it("keeps the live Vitest scenario workflow manual, pinned, and artifact-safe", () => { expect(validateE2eVitestScenariosWorkflowBoundary()).toEqual([]); }); From 206e3256f1b2693b6c436ce8bc3bcad7dfb114ec Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Mon, 22 Jun 2026 11:46:14 -0400 Subject: [PATCH 11/13] test(e2e): validate stop-start workflow boundary --- .../e2e-scenarios-workflow.test.ts | 93 ++++++++ tools/e2e-scenarios/workflow-boundary.mts | 208 ++++++++++++++++++ 2 files changed, 301 insertions(+) diff --git a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts index 5d1527c1ea7..ff1d23d5278 100644 --- a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts +++ b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts @@ -1132,6 +1132,99 @@ jobs: } }); + it("rejects channels stop/start workflow-boundary drift for secret and artifact handling", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-vitest-workflow-")); + const workflowPath = path.join(tmp, "workflow.yaml"); + const workflow = readWorkflow() as { + jobs: Record< + string, + { + env: Record; + steps: Array>; + strategy: { matrix: { agent: string[] }; "fail-fast": boolean }; + "timeout-minutes"?: number; + } + >; + }; + const job = workflow.jobs["channels-stop-start-vitest"]; + expect(job).toBeDefined(); + job["timeout-minutes"] = 45; + job.strategy["fail-fast"] = true; + job.strategy.matrix.agent = ["openclaw"]; + job.env.NEMOCLAW_SANDBOX_NAME = "personal-dev-${{ matrix.agent }}"; + job.env.DOCKER_CONFIG = "${{ github.workspace }}/.docker-config-shared"; + job.env.NVIDIA_API_KEY = "${{ secrets.NVIDIA_API_KEY }}"; + for (const step of job.steps) { + if (typeof step.uses === "string" && step.uses.startsWith("actions/checkout@")) { + step.with = { ...(step.with as Record), "persist-credentials": true }; + } + if (step.name === "Authenticate to Docker Hub") { + step.run = + "docker login docker.io --username user --password ${{ secrets.DOCKERHUB_TOKEN }}"; + } + if (step.name === "Install root dependencies") { + step.run = "npm install"; + } + if (step.name === "Install OpenShell") { + step.run = "bash scripts/install-openshell.sh"; + } + if (step.name === "Run channels stop/start live test") { + step.env = { + NVIDIA_API_KEY: "${{ secrets.NVIDIA_API_KEY }}", + TELEGRAM_BOT_TOKEN: "real-token", + }; + step.run = String(step.run).replace( + "test/e2e-scenario/live/channels-stop-start.test.ts", + "test/e2e-scenario/live/channels-add-remove.test.ts", + ); + } + if (step.name === "Upload channels stop/start artifacts") { + step.uses = "actions/upload-artifact@v4"; + step.with = { + ...(step.with as Record), + name: "channels-stop-start", + path: "e2e-artifacts/vitest/channels-stop-start/", + "include-hidden-files": true, + "retention-days": 1, + }; + } + if (step.name === "Clean up Docker auth") { + delete step.if; + step.run = "docker logout docker.io"; + } + } + fs.writeFileSync(workflowPath, YAML.stringify(workflow)); + + try { + const errors = validateE2eVitestScenariosWorkflowBoundary(workflowPath); + expect(errors).toEqual( + expect.arrayContaining([ + "channels-stop-start-vitest job must keep the 90 minute timeout", + "channels-stop-start-vitest strategy.fail-fast must be false", + "channels-stop-start-vitest matrix.agent must be openclaw,hermes", + "channels-stop-start-vitest job must derive NEMOCLAW_SANDBOX_NAME from matrix.agent with the e2e-channels-stop-start- prefix", + "channels-stop-start-vitest job must isolate Docker auth by matrix agent", + "channels-stop-start-vitest job env must not include NVIDIA_API_KEY", + "channels-stop-start-vitest checkout step must set persist-credentials=false", + "step 'Install root dependencies' run script must include npm ci --ignore-scripts", + "step 'Install OpenShell' run script must include env -u DOCKER_CONFIG", + "channels-stop-start-vitest step 'Run channels stop/start live test' env must not include NVIDIA_API_KEY", + "channels-stop-start-vitest step must receive NVIDIA_INFERENCE_API_KEY from secrets", + "channels-stop-start-vitest step must set the fake Telegram token", + "step 'Run channels stop/start live test' run script must include test/e2e-scenario/live/channels-stop-start.test.ts", + "channels-stop-start-vitest upload-artifact action must be pinned to a full commit SHA", + "channels-stop-start-vitest artifact upload name must include matrix.agent", + "channels-stop-start-vitest artifact upload must set include-hidden-files: false", + "channels-stop-start-vitest artifact upload retention-days must be 14", + "channels-stop-start-vitest Docker auth cleanup must always run", + "step 'Clean up Docker auth' run script must include rm -rf \"${DOCKER_CONFIG}\"", + ]), + ); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); + it("requires messaging-compatible-endpoint workflow and report coverage", () => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-vitest-workflow-")); const renamedWorkflowPath = path.join(tmp, "renamed-workflow.yaml"); diff --git a/tools/e2e-scenarios/workflow-boundary.mts b/tools/e2e-scenarios/workflow-boundary.mts index 8decf10480e..0ec77fbb559 100644 --- a/tools/e2e-scenarios/workflow-boundary.mts +++ b/tools/e2e-scenarios/workflow-boundary.mts @@ -3474,6 +3474,213 @@ function validateChannelsAddRemoveVitestJob(errors: string[], jobs: WorkflowReco } } +function validateChannelsStopStartVitestJob(errors: string[], jobs: WorkflowRecord): void { + const jobName = "channels-stop-start-vitest"; + const scenarioName = "channels-stop-start"; + const job = asRecord(jobs[jobName]); + if (Object.keys(job).length === 0) { + errors.push("workflow missing channels-stop-start-vitest job"); + return; + } + + if (job["runs-on"] !== "ubuntu-latest") { + errors.push("channels-stop-start-vitest job must run on ubuntu-latest"); + } + validateFreeStandingJobSelector(errors, jobs, jobName, scenarioName); + if (job["timeout-minutes"] !== 90) { + errors.push("channels-stop-start-vitest job must keep the 90 minute timeout"); + } + const strategy = asRecord(job.strategy); + if (strategy["fail-fast"] !== false) { + errors.push("channels-stop-start-vitest strategy.fail-fast must be false"); + } + const matrix = asRecord(strategy.matrix); + if (!Array.isArray(matrix.agent) || matrix.agent.join(",") !== "openclaw,hermes") { + errors.push("channels-stop-start-vitest matrix.agent must be openclaw,hermes"); + } + + const jobEnv = asRecord(job.env); + if (jobEnv.NEMOCLAW_RUN_E2E_SCENARIOS !== "1") { + errors.push("channels-stop-start-vitest job must set NEMOCLAW_RUN_E2E_SCENARIOS=1"); + } + if ( + jobEnv.E2E_ARTIFACT_DIR !== + "${{ github.workspace }}/e2e-artifacts/vitest/channels-stop-start/${{ matrix.agent }}" + ) { + errors.push( + "channels-stop-start-vitest job must write artifacts under e2e-artifacts/vitest/channels-stop-start/${{ matrix.agent }}", + ); + } + if (jobEnv.NEMOCLAW_CLI_BIN !== "${{ github.workspace }}/bin/nemoclaw.js") { + errors.push("channels-stop-start-vitest job must point NEMOCLAW_CLI_BIN at the repo CLI"); + } + if (jobEnv.NEMOCLAW_SANDBOX_NAME !== "e2e-channels-stop-start-${{ matrix.agent }}") { + errors.push( + "channels-stop-start-vitest job must derive NEMOCLAW_SANDBOX_NAME from matrix.agent with the e2e-channels-stop-start- prefix", + ); + } + if (jobEnv.NEMOCLAW_AGENT !== "${{ matrix.agent }}") { + errors.push("channels-stop-start-vitest job must pass matrix.agent through NEMOCLAW_AGENT"); + } + if (jobEnv.NEMOCLAW_CHANNELS_STOP_START_AGENT !== "${{ matrix.agent }}") { + errors.push( + "channels-stop-start-vitest job must pass matrix.agent through NEMOCLAW_CHANNELS_STOP_START_AGENT", + ); + } + if (jobEnv.NEMOCLAW_NON_INTERACTIVE !== "1") { + errors.push("channels-stop-start-vitest job must set NEMOCLAW_NON_INTERACTIVE=1"); + } + if (jobEnv.NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE !== "1") { + errors.push("channels-stop-start-vitest job must set NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1"); + } + if (jobEnv.OPENSHELL_GATEWAY !== "nemoclaw") { + errors.push("channels-stop-start-vitest job must force OPENSHELL_GATEWAY=nemoclaw"); + } + if ( + jobEnv.DOCKER_CONFIG !== + "${{ github.workspace }}/.docker-config-channels-stop-start-${{ matrix.agent }}" + ) { + errors.push("channels-stop-start-vitest job must isolate Docker auth by matrix agent"); + } + for (const secret of [ + "NVIDIA_INFERENCE_API_KEY", + "NVIDIA_API_KEY", + "DOCKERHUB_USERNAME", + "DOCKERHUB_TOKEN", + "GITHUB_TOKEN", + ]) { + requireEnvDoesNotExposeSecret(errors, "channels-stop-start-vitest job", jobEnv, secret); + } + + const steps = asSteps(job.steps); + requireNoDispatchInputInterpolation(errors, steps); + for (const step of steps) { + const stepName = `channels-stop-start-vitest step '${step.name ?? step.uses ?? ""}'`; + const stepEnv = asRecord(step.env); + if (step.name !== "Run channels stop/start live test") { + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "NVIDIA_INFERENCE_API_KEY"); + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "NVIDIA_API_KEY"); + } + if (step.name !== "Authenticate to Docker Hub") { + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "DOCKERHUB_USERNAME"); + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "DOCKERHUB_TOKEN"); + requireNoDockerHubAuthInRun(errors, stepName, stringValue(step.run)); + } + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "GITHUB_TOKEN"); + } + + const checkout = steps.find((step) => stringValue(step.uses).startsWith("actions/checkout@")); + if (!checkout) errors.push("channels-stop-start-vitest job missing checkout step"); + requireFullShaAction(errors, checkout, "channels-stop-start-vitest checkout"); + if (asRecord(checkout?.with)["persist-credentials"] !== false) { + errors.push("channels-stop-start-vitest checkout step must set persist-credentials=false"); + } + + const dockerHubAuth = requireJobStep(errors, jobName, steps, "Authenticate to Docker Hub"); + const dockerHubEnv = asRecord(dockerHubAuth?.env); + if (dockerHubEnv.DOCKERHUB_USERNAME !== "${{ secrets.DOCKERHUB_USERNAME }}") { + errors.push( + "channels-stop-start-vitest Docker Hub auth must receive DOCKERHUB_USERNAME from secrets", + ); + } + if (dockerHubEnv.DOCKERHUB_TOKEN !== "${{ secrets.DOCKERHUB_TOKEN }}") { + errors.push( + "channels-stop-start-vitest Docker Hub auth must receive DOCKERHUB_TOKEN from secrets", + ); + } + requireRunContains(errors, dockerHubAuth, 'mkdir -p "${DOCKER_CONFIG}"'); + requireRunContains(errors, dockerHubAuth, 'chmod 700 "${DOCKER_CONFIG}"'); + requireRunContains(errors, dockerHubAuth, "docker login docker.io"); + requireRunContains(errors, dockerHubAuth, "--password-stdin"); + requireRunContains(errors, dockerHubAuth, "continuing with anonymous pulls"); + + const setupNode = namedStep(steps, "Set up Node"); + if (!setupNode) errors.push("channels-stop-start-vitest job missing step: Set up Node"); + requireFullShaAction(errors, setupNode, "channels-stop-start-vitest setup-node"); + + const installRootDependencies = requireJobStep( + errors, + jobName, + steps, + "Install root dependencies", + ); + requireRunContains(errors, installRootDependencies, "npm ci --ignore-scripts"); + + const buildCli = requireJobStep(errors, jobName, steps, "Build CLI"); + requireRunContains(errors, buildCli, "npm run build:cli"); + + const installOpenShell = requireJobStep(errors, jobName, steps, "Install OpenShell"); + requireRunContains(errors, installOpenShell, "bash scripts/install-openshell.sh"); + requireRunContains(errors, installOpenShell, "env -u DOCKER_CONFIG"); + requireRunContains(errors, installOpenShell, "-u DOCKERHUB_USERNAME"); + requireRunContains(errors, installOpenShell, "-u DOCKERHUB_TOKEN"); + requireRunContains(errors, installOpenShell, "-u NVIDIA_API_KEY"); + requireRunContains(errors, installOpenShell, "-u GITHUB_TOKEN"); + + const runVitest = requireJobStep(errors, jobName, steps, "Run channels stop/start live test"); + const runVitestEnv = asRecord(runVitest?.env); + requireEnvDoesNotExposeSecret( + errors, + "channels-stop-start-vitest step 'Run channels stop/start live test'", + runVitestEnv, + "NVIDIA_API_KEY", + ); + if (runVitestEnv.NVIDIA_INFERENCE_API_KEY !== "${{ secrets.NVIDIA_INFERENCE_API_KEY }}") { + errors.push( + "channels-stop-start-vitest step must receive NVIDIA_INFERENCE_API_KEY from secrets", + ); + } + if ( + runVitestEnv.TELEGRAM_BOT_TOKEN !== "test-fake-telegram-token-stop-start-${{ matrix.agent }}" + ) { + errors.push("channels-stop-start-vitest step must set the fake Telegram token"); + } + if (runVitestEnv.DISCORD_BOT_TOKEN !== "test-fake-discord-token-stop-start-${{ matrix.agent }}") { + errors.push("channels-stop-start-vitest step must set the fake Discord token"); + } + if (runVitestEnv.SLACK_BOT_TOKEN !== "xoxb-fake-slack-token-stop-start-${{ matrix.agent }}") { + errors.push("channels-stop-start-vitest step must set the fake Slack bot token"); + } + if (runVitestEnv.SLACK_APP_TOKEN !== "xapp-fake-slack-token-stop-start-${{ matrix.agent }}") { + errors.push("channels-stop-start-vitest step must set the fake Slack app token"); + } + if (runVitestEnv.WECHAT_BOT_TOKEN !== "test-fake-wechat-token-stop-start-${{ matrix.agent }}") { + errors.push("channels-stop-start-vitest step must set the fake WeChat token"); + } + requireRunContains(errors, runVitest, "OPENSHELL_BIN"); + requireRunContains(errors, runVitest, "npx vitest run --project e2e-scenarios-live"); + requireRunContains(errors, runVitest, "test/e2e-scenario/live/channels-stop-start.test.ts"); + + const upload = requireJobStep(errors, jobName, steps, "Upload channels stop/start artifacts"); + requireFullShaAction(errors, upload, "channels-stop-start-vitest upload-artifact"); + const uploadWith = asRecord(upload?.with); + if (uploadWith.name !== "e2e-vitest-scenarios-channels-stop-start-${{ matrix.agent }}") { + errors.push("channels-stop-start-vitest artifact upload name must include matrix.agent"); + } + const uploadPath = stringValue(uploadWith.path); + requireUploadPathContains( + errors, + uploadPath, + "e2e-artifacts/vitest/channels-stop-start/${{ matrix.agent }}/", + ); + if (uploadWith["include-hidden-files"] !== false) { + errors.push("channels-stop-start-vitest artifact upload must set include-hidden-files: false"); + } + if (uploadWith["if-no-files-found"] !== "ignore") { + errors.push("channels-stop-start-vitest artifact upload must ignore missing fixture artifacts"); + } + if (uploadWith["retention-days"] !== 14) { + errors.push("channels-stop-start-vitest artifact upload retention-days must be 14"); + } + + const cleanup = requireJobStep(errors, jobName, steps, "Clean up Docker auth"); + if (cleanup?.if !== "always()") { + errors.push("channels-stop-start-vitest Docker auth cleanup must always run"); + } + requireRunContains(errors, cleanup, "docker logout docker.io"); + requireRunContains(errors, cleanup, 'rm -rf "${DOCKER_CONFIG}"'); +} + function validateBedrockRuntimeCompatibleAnthropicVitestJob( errors: string[], jobs: WorkflowRecord, @@ -4022,6 +4229,7 @@ export function validateE2eVitestScenariosWorkflowBoundary( ); validateChannelsAddRemoveVitestJob(errors, jobs); + validateChannelsStopStartVitestJob(errors, jobs); const reportToPr = asRecord(jobs["report-to-pr"]); if (Object.keys(reportToPr).length === 0) { From e0aa5539ca575105b8d982a2e18a15602c94eb33 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Mon, 22 Jun 2026 11:58:57 -0400 Subject: [PATCH 12/13] test(e2e): keep workflow test linear --- .../e2e-scenarios-workflow.test.ts | 89 +++++++++++-------- 1 file changed, 50 insertions(+), 39 deletions(-) diff --git a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts index ff1d23d5278..95699519afa 100644 --- a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts +++ b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts @@ -1154,45 +1154,56 @@ jobs: job.env.NEMOCLAW_SANDBOX_NAME = "personal-dev-${{ matrix.agent }}"; job.env.DOCKER_CONFIG = "${{ github.workspace }}/.docker-config-shared"; job.env.NVIDIA_API_KEY = "${{ secrets.NVIDIA_API_KEY }}"; - for (const step of job.steps) { - if (typeof step.uses === "string" && step.uses.startsWith("actions/checkout@")) { - step.with = { ...(step.with as Record), "persist-credentials": true }; - } - if (step.name === "Authenticate to Docker Hub") { - step.run = - "docker login docker.io --username user --password ${{ secrets.DOCKERHUB_TOKEN }}"; - } - if (step.name === "Install root dependencies") { - step.run = "npm install"; - } - if (step.name === "Install OpenShell") { - step.run = "bash scripts/install-openshell.sh"; - } - if (step.name === "Run channels stop/start live test") { - step.env = { - NVIDIA_API_KEY: "${{ secrets.NVIDIA_API_KEY }}", - TELEGRAM_BOT_TOKEN: "real-token", - }; - step.run = String(step.run).replace( - "test/e2e-scenario/live/channels-stop-start.test.ts", - "test/e2e-scenario/live/channels-add-remove.test.ts", - ); - } - if (step.name === "Upload channels stop/start artifacts") { - step.uses = "actions/upload-artifact@v4"; - step.with = { - ...(step.with as Record), - name: "channels-stop-start", - path: "e2e-artifacts/vitest/channels-stop-start/", - "include-hidden-files": true, - "retention-days": 1, - }; - } - if (step.name === "Clean up Docker auth") { - delete step.if; - step.run = "docker logout docker.io"; - } - } + const checkoutStep = job.steps.find( + (step) => typeof step.uses === "string" && step.uses.startsWith("actions/checkout@"), + ); + expect(checkoutStep).toBeDefined(); + checkoutStep!.with = { + ...(checkoutStep!.with as Record), + "persist-credentials": true, + }; + + const dockerAuthStep = job.steps.find((step) => step.name === "Authenticate to Docker Hub"); + expect(dockerAuthStep).toBeDefined(); + dockerAuthStep!.run = + "docker login docker.io --username user --password ${{ secrets.DOCKERHUB_TOKEN }}"; + + const installRootStep = job.steps.find((step) => step.name === "Install root dependencies"); + expect(installRootStep).toBeDefined(); + installRootStep!.run = "npm install"; + + const installOpenShellStep = job.steps.find((step) => step.name === "Install OpenShell"); + expect(installOpenShellStep).toBeDefined(); + installOpenShellStep!.run = "bash scripts/install-openshell.sh"; + + const runStep = job.steps.find((step) => step.name === "Run channels stop/start live test"); + expect(runStep).toBeDefined(); + runStep!.env = { + NVIDIA_API_KEY: "${{ secrets.NVIDIA_API_KEY }}", + TELEGRAM_BOT_TOKEN: "real-token", + }; + runStep!.run = String(runStep!.run).replace( + "test/e2e-scenario/live/channels-stop-start.test.ts", + "test/e2e-scenario/live/channels-add-remove.test.ts", + ); + + const uploadStep = job.steps.find( + (step) => step.name === "Upload channels stop/start artifacts", + ); + expect(uploadStep).toBeDefined(); + uploadStep!.uses = "actions/upload-artifact@v4"; + uploadStep!.with = { + ...(uploadStep!.with as Record), + name: "channels-stop-start", + path: "e2e-artifacts/vitest/channels-stop-start/", + "include-hidden-files": true, + "retention-days": 1, + }; + + const cleanupStep = job.steps.find((step) => step.name === "Clean up Docker auth"); + expect(cleanupStep).toBeDefined(); + delete cleanupStep!.if; + cleanupStep!.run = "docker logout docker.io"; fs.writeFileSync(workflowPath, YAML.stringify(workflow)); try { From 74709a7d0365f213a819b5f4fbfbfc2b5109fd32 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Mon, 22 Jun 2026 12:22:48 -0400 Subject: [PATCH 13/13] test(e2e): restore stop-start gateway cleanup --- .../live/channels-stop-start-helpers.ts | 41 +++++++++++++++++-- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/test/e2e-scenario/live/channels-stop-start-helpers.ts b/test/e2e-scenario/live/channels-stop-start-helpers.ts index df151adc5e3..bbf95b2c0d9 100644 --- a/test/e2e-scenario/live/channels-stop-start-helpers.ts +++ b/test/e2e-scenario/live/channels-stop-start-helpers.ts @@ -11,6 +11,7 @@ import { expect } from "../fixtures/e2e-test.ts"; import { assertChannelsStopStartSandboxName } from "./channels-stop-start-safety.ts"; import { type AgentKind, + bestEffort, CLI, cleanupSandbox, dockerInfo, @@ -327,6 +328,22 @@ async function precleanProviders( } } +async function destroyNemoclawGateway( + host: import("../fixtures/clients/host.ts").HostCliClient, + env: NodeJS.ProcessEnv, + redactions: string[], + artifactName: string, +): Promise { + await bestEffort(() => + host.command("openshell", ["gateway", "destroy", "-g", "nemoclaw"], { + artifactName, + env, + redactionValues: redactions, + timeoutMs: 60_000, + }), + ); +} + async function rebuildSandbox( host: import("../fixtures/clients/host.ts").HostCliClient, sandboxName: string, @@ -419,9 +436,21 @@ export async function runChannelsStopStartScenario({ channels: CHANNELS, }); - cleanup.add(`destroy channels stop/start sandbox ${SANDBOX_NAME}`, () => - cleanupSandbox(host, SANDBOX_NAME, env, redactions, `cleanup-channels-stop-start-${AGENT}`), - ); + cleanup.add(`destroy channels stop/start sandbox ${SANDBOX_NAME}`, async () => { + await cleanupSandbox( + host, + SANDBOX_NAME, + env, + redactions, + `cleanup-channels-stop-start-${AGENT}`, + ); + await destroyNemoclawGateway( + host, + env, + redactions, + `cleanup-openshell-gateway-destroy-${AGENT}`, + ); + }); await cleanupSandbox( host, SANDBOX_NAME, @@ -429,6 +458,12 @@ export async function runChannelsStopStartScenario({ redactions, `preclean-channels-stop-start-${AGENT}`, ); + await destroyNemoclawGateway( + host, + env, + redactions, + `preclean-openshell-gateway-destroy-${AGENT}`, + ); await precleanProviders(host, env, redactions, `preclean-channels-stop-start-${AGENT}`); const docker = await dockerInfo(host, env);