diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index 65e06c7eda..dcb96163ff 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -40,7 +40,7 @@ jobs: SCENARIOS: ${{ inputs.scenarios }} run: | set -euo pipefail - allowed_jobs="openshell-version-pin-vitest,onboard-negative-paths-vitest,openclaw-tui-chat-correlation-vitest,gateway-guard-recovery" + allowed_jobs="openshell-version-pin-vitest,onboard-negative-paths-vitest,token-rotation-vitest,openclaw-tui-chat-correlation-vitest,gateway-guard-recovery" if [ -n "${JOBS}" ] && [ -n "${SCENARIOS}" ]; then echo "::error::Use either scenarios or jobs, not both." >&2 exit 1 @@ -85,6 +85,7 @@ jobs: name: Generate Vitest scenario matrix env: SCENARIOS: ${{ inputs.scenarios }} + JOBS: ${{ inputs.jobs }} run: | set -euo pipefail args=(--emit-live-matrix) @@ -95,6 +96,10 @@ jobs: fi args+=(--scenarios "${SCENARIOS}") fi + if [ -n "${JOBS}" ] && [[ ! "${JOBS}" =~ ^[A-Za-z0-9_-]+(,[A-Za-z0-9_-]+)*$ ]]; then + echo "::error::Invalid jobs input; use comma-separated job ids" >&2 + exit 1 + fi matrix="$(npx tsx test/e2e-scenario/scenarios/run.ts "${args[@]}")" echo "matrix=${matrix}" >> "$GITHUB_OUTPUT" MATRIX_JSON="${matrix}" python - <<'PY' >> "$GITHUB_STEP_SUMMARY" @@ -298,6 +303,89 @@ jobs: if-no-files-found: ignore retention-days: 14 + token-rotation-vitest: + needs: validate-jobs + if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',token-rotation-vitest,') }} + runs-on: ubuntu-latest + timeout-minutes: 45 + env: + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/token-rotation + NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js + NEMOCLAW_RUN_E2E_SCENARIOS: "1" + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Authenticate to Docker Hub + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + shell: bash + run: | + set -euo pipefail + if [[ -z "${DOCKERHUB_USERNAME}" || -z "${DOCKERHUB_TOKEN}" ]]; then + echo "::notice::Docker Hub credentials not configured; continuing with anonymous pulls." + exit 0 + fi + login_succeeded=0 + for attempt in 1 2 3; do + if echo "${DOCKERHUB_TOKEN}" | timeout 30s docker login docker.io --username "${DOCKERHUB_USERNAME}" --password-stdin; then + login_succeeded=1 + break + fi + if [[ "$attempt" -lt 3 ]]; then + echo "::warning::Docker Hub login attempt ${attempt} failed; retrying." + sleep 5 + fi + done + if [[ "$login_succeeded" -ne 1 ]]; then + echo "::warning::Docker Hub login failed after 3 attempts; continuing with anonymous pulls." + fi + + - name: Set up Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 + with: + node-version: 22 + cache: npm + + - name: Install root dependencies + run: npm ci --ignore-scripts + + - name: Build CLI + run: npm run build:cli + + - name: Run token rotation live test + # Migrated from nightly-e2e.yaml token-rotation-e2e. Preserve the + # original runner class: ubuntu-latest with Docker/OpenShell plus the + # legacy-supported fake OpenAI-compatible endpoint path and fake + # Telegram/Discord/Slack token boundary. + env: + GITHUB_TOKEN: ${{ github.token }} + TELEGRAM_BOT_TOKEN_A: "test-fake-token-A-rotation-e2e" + TELEGRAM_BOT_TOKEN_B: "test-fake-token-B-rotation-e2e" + DISCORD_BOT_TOKEN_A: "dc-a-rotation-e2e" + DISCORD_BOT_TOKEN_B: "dc-b-rotation-e2e" + SLACK_BOT_TOKEN_A: "xoxb-fake-A-rotation-e2e" + SLACK_BOT_TOKEN_B: "xoxb-fake-B-rotation-e2e" + SLACK_APP_TOKEN_A: "xapp-fake-A-rotation-e2e" + SLACK_APP_TOKEN_B: "xapp-fake-B-rotation-e2e" + run: | + set -euo pipefail + npx vitest run --project e2e-scenarios-live \ + test/e2e-scenario/live/token-rotation.test.ts \ + --silent=false --reporter=default + + - name: Upload token rotation artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: e2e-vitest-scenarios-token-rotation + path: e2e-artifacts/vitest/token-rotation/ + include-hidden-files: false + if-no-files-found: ignore + retention-days: 14 + # Focused coverage slice for the #2603/#3145 OpenClaw websocket # protocol/history contract. The retained legacy bash lane remains the # source for full closeout until a later PR proves replacement and deletes it. @@ -481,6 +569,7 @@ jobs: live-scenarios, openshell-version-pin-vitest, onboard-negative-paths-vitest, + token-rotation-vitest, openclaw-tui-chat-correlation-vitest, gateway-guard-recovery, ] diff --git a/test/e2e-scenario/live/token-rotation.test.ts b/test/e2e-scenario/live/token-rotation.test.ts new file mode 100644 index 0000000000..de2aa75c6b --- /dev/null +++ b/test/e2e-scenario/live/token-rotation.test.ts @@ -0,0 +1,586 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import http from "node:http"; +import type { AddressInfo } from "node:net"; +import path from "node:path"; + +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { validateSandboxName } from "../fixtures/clients/sandbox.ts"; +import { expect, test } from "../fixtures/e2e-test.ts"; +import { shouldRunLiveE2EScenarios } from "../fixtures/live-project-gate.ts"; +import { testTimeoutOptions } from "../../helpers/timeouts"; + +// Focused Vitest replacement coverage for test/e2e/test-token-rotation.sh. +// Keep this free-standing and direct: the legacy contract is the real CLI + +// OpenShell/provider boundary for messaging credential reuse/rotation, not the +// typed registry scenario steady-state probe path. The test drives the real +// `nemoclaw onboard` CLI with fake provider tokens, preserving the provider +// upsert, registry credential-hash, sandbox rebuild, and reuse assertions. + +const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); +const CLI_ENTRYPOINT = path.join(REPO_ROOT, "bin", "nemoclaw.js"); +const REGISTRY_FILE = path.join(process.env.HOME ?? "/tmp", ".nemoclaw", "sandboxes.json"); +const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? `e2e-token-rotation-${process.pid}`; +validateSandboxName(SANDBOX_NAME); + +const ONBOARD_TIMEOUT_MS = 25 * 60_000; +const PHASE_TIMEOUT_MS = 40 * 60_000; + +interface TokenSet { + telegram: string; + discord: string; + slackBot: string; + slackApp: string; +} + +const TOKEN_A: TokenSet = { + telegram: process.env.TELEGRAM_BOT_TOKEN_A ?? "test-fake-token-A-rotation-e2e", + discord: process.env.DISCORD_BOT_TOKEN_A ?? "dc-a-rotation-e2e", + slackBot: process.env.SLACK_BOT_TOKEN_A ?? "xoxb-fake-A-rotation-e2e", + slackApp: process.env.SLACK_APP_TOKEN_A ?? "xapp-fake-A-rotation-e2e", +}; + +const TOKEN_B: TokenSet = { + telegram: process.env.TELEGRAM_BOT_TOKEN_B ?? "test-fake-token-B-rotation-e2e", + discord: process.env.DISCORD_BOT_TOKEN_B ?? "dc-b-rotation-e2e", + slackBot: process.env.SLACK_BOT_TOKEN_B ?? "xoxb-fake-B-rotation-e2e", + slackApp: process.env.SLACK_APP_TOKEN_B ?? "xapp-fake-B-rotation-e2e", +}; + +type RegistryCredentialBinding = { + providerEnvKey?: unknown; + credentialHash?: unknown; +}; + +type FakeOpenAIRequest = { + method: string; + path: string; + bodyBytes: number; +}; + +type FakeOpenAIEndpoint = { + baseUrl: string; + requests: FakeOpenAIRequest[]; + close(): Promise; +}; + +type RegistrySandboxEntry = { + messaging?: { + plan?: { + credentialBindings?: RegistryCredentialBinding[]; + }; + }; +}; + +function resultText(result: { stdout: string; stderr: string }): string { + return [result.stdout, result.stderr].filter(Boolean).join("\n"); +} + +function stripAnsi(value: string): string { + return value.replace(/\u001B\[[0-?]*[ -/]*[@-~]/g, ""); +} + +function jsonResponse(res: http.ServerResponse, status: number, payload: unknown): void { + const body = JSON.stringify(payload); + res.writeHead(status, { + "Content-Type": "application/json", + "Content-Length": Buffer.byteLength(body), + }); + res.end(body); +} + +async function startFakeOpenAIEndpoint(): Promise { + const requests: FakeOpenAIRequest[] = []; + const server = http.createServer((req, res) => { + let bodyBytes = 0; + req.on("data", (chunk: Buffer) => { + bodyBytes += chunk.length; + }); + req.on("end", () => { + const requestPath = new URL(req.url ?? "/", "http://fake.local").pathname; + requests.push({ + method: req.method ?? "GET", + path: requestPath, + bodyBytes, + }); + if (req.method === "GET" && ["/v1/models", "/models"].includes(requestPath)) { + jsonResponse(res, 200, { + data: [{ id: "test-model", object: "model" }], + }); + return; + } + if ( + req.method === "POST" && + ["/v1/chat/completions", "/chat/completions"].includes(requestPath) + ) { + jsonResponse(res, 200, { + id: "chatcmpl-token-rotation-e2e", + object: "chat.completion", + choices: [ + { + index: 0, + message: { role: "assistant", content: "OK" }, + finish_reason: "stop", + }, + ], + }); + return; + } + if (req.method === "POST" && ["/v1/responses", "/responses"].includes(requestPath)) { + jsonResponse(res, 200, { + id: "resp-token-rotation-e2e", + object: "response", + output: [ + { + type: "message", + role: "assistant", + content: [{ type: "output_text", text: "OK" }], + }, + ], + }); + return; + } + jsonResponse(res, 404, { error: { message: "not found" } }); + }); + }); + + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "0.0.0.0", () => { + server.off("error", reject); + resolve(); + }); + }); + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error("fake OpenAI-compatible endpoint did not bind to a TCP port"); + } + const port = (address as AddressInfo).port; + return { + baseUrl: `http://host.openshell.internal:${port}/v1`, + requests, + close: () => + new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }), + }; +} + +function onboardEnv(endpointUrl: string, tokens: TokenSet): NodeJS.ProcessEnv { + return { + ...buildAvailabilityProbeEnv(), + COMPATIBLE_API_KEY: "token-rotation-compatible-e2e", + TELEGRAM_BOT_TOKEN: tokens.telegram, + DISCORD_BOT_TOKEN: tokens.discord, + SLACK_BOT_TOKEN: tokens.slackBot, + SLACK_APP_TOKEN: tokens.slackApp, + NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_YES: "1", + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", + NEMOCLAW_PROVIDER: "custom", + NEMOCLAW_ENDPOINT_URL: endpointUrl, + NEMOCLAW_MODEL: "test-model", + NEMOCLAW_POLICY_TIER: "open", + NEMOCLAW_SKIP_TELEGRAM_REACHABILITY: "1", + NEMOCLAW_SKIP_SLACK_AUTH_VALIDATION: "1", + NEMOCLAW_RECREATE_WITHOUT_BACKUP: "1", + GITHUB_TOKEN: process.env.GITHUB_TOKEN, + }; +} + +function readSandboxRegistryEntry(): RegistrySandboxEntry { + expect(fs.existsSync(REGISTRY_FILE), `${REGISTRY_FILE} missing`).toBe(true); + const registry = JSON.parse(fs.readFileSync(REGISTRY_FILE, "utf8")) as { + sandboxes?: Record; + }; + const entry = registry.sandboxes?.[SANDBOX_NAME]; + expect(entry, `registry entry ${SANDBOX_NAME} missing`).toBeTruthy(); + if (!entry) throw new Error(`registry entry ${SANDBOX_NAME} missing`); + return entry; +} + +function credentialBindings(): RegistryCredentialBinding[] { + const bindings = readSandboxRegistryEntry().messaging?.plan?.credentialBindings; + expect(Array.isArray(bindings), "messaging.plan.credentialBindings missing").toBe(true); + return Array.isArray(bindings) ? bindings : []; +} + +function expectCredentialHash(envKey: string): void { + const binding = credentialBindings().find((entry) => entry.providerEnvKey === envKey); + expect(binding, `${envKey} credential binding missing`).toBeTruthy(); + expect(typeof binding?.credentialHash, `${envKey} credential hash missing`).toBe("string"); + expect( + String(binding?.credentialHash ?? "").length, + `${envKey} credential hash empty`, + ).toBeGreaterThan(0); +} + +function expectRotationOutput( + output: string, + expectedProviders: readonly string[], + forbiddenProviders: readonly string[], +): void { + const rotationLine = output + .split(/\r?\n/) + .find((line) => line.includes("Messaging credential(s) rotated:")); + expect(rotationLine, output).toBeTruthy(); + for (const provider of expectedProviders) { + expect(rotationLine, `rotation line should name ${provider}: ${rotationLine}`).toContain( + provider, + ); + } + for (const provider of forbiddenProviders) { + expect( + rotationLine, + `rotation line should not name ${provider}: ${rotationLine}`, + ).not.toContain(provider); + } + expect(output).toContain("Rebuilding sandbox to propagate new credentials"); +} + +function assertTokenPairsDiffer(): void { + for (const [label, a, b] of [ + ["TELEGRAM_BOT_TOKEN", TOKEN_A.telegram, TOKEN_B.telegram], + ["DISCORD_BOT_TOKEN", TOKEN_A.discord, TOKEN_B.discord], + ["SLACK_BOT_TOKEN", TOKEN_A.slackBot, TOKEN_B.slackBot], + ["SLACK_APP_TOKEN", TOKEN_A.slackApp, TOKEN_B.slackApp], + ] as const) { + expect(a, `${label}_A and ${label}_B must be different`).not.toBe(b); + } +} + +function redactionValues(): string[] { + return [ + "token-rotation-compatible-e2e", + process.env.NVIDIA_API_KEY, + process.env.GITHUB_TOKEN, + ...Object.values(TOKEN_A), + ...Object.values(TOKEN_B), + ].filter((value): value is string => typeof value === "string" && value.length > 0); +} + +async function runInstall( + host: import("../fixtures/clients/host.ts").HostCliClient, + endpointUrl: string, + tokens: TokenSet, + extraEnv: NodeJS.ProcessEnv = {}, +) { + return host.command("bash", ["install.sh", "--non-interactive"], { + artifactName: "phase-0-install-token-a", + cwd: REPO_ROOT, + env: { + ...onboardEnv(endpointUrl, tokens), + ...extraEnv, + }, + redactionValues: redactionValues(), + timeoutMs: ONBOARD_TIMEOUT_MS, + }); +} + +async function runOnboard( + host: import("../fixtures/clients/host.ts").HostCliClient, + endpointUrl: string, + tokens: TokenSet, + artifactName: string, + extraEnv: NodeJS.ProcessEnv = {}, +) { + return host.command("node", [CLI_ENTRYPOINT, "onboard", "--non-interactive"], { + artifactName, + env: { + ...onboardEnv(endpointUrl, tokens), + ...extraEnv, + }, + redactionValues: redactionValues(), + timeoutMs: ONBOARD_TIMEOUT_MS, + }); +} + +async function assertSandboxRunning( + host: import("../fixtures/clients/host.ts").HostCliClient, + artifactName: string, +): Promise { + const sandboxList = await host.command( + "bash", + ["-lc", 'openshell sandbox list 2>/dev/null | grep -F -- "$1"', "_", SANDBOX_NAME], + { + artifactName, + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }, + ); + const output = resultText(sandboxList); + const plainStdout = stripAnsi(sandboxList.stdout); + expect(sandboxList.exitCode, output).toBe(0); + expect(plainStdout, output).toContain(SANDBOX_NAME); + expect(plainStdout, output).toMatch(/\b(?:Ready|Running)\b/i); +} + +async function deleteSandboxIfOpenshellExists( + host: import("../fixtures/clients/host.ts").HostCliClient, + artifactName: string, +): Promise { + await host.command( + "bash", + [ + "-lc", + 'if command -v openshell >/dev/null 2>&1; then openshell sandbox delete "$1"; fi', + "_", + SANDBOX_NAME, + ], + { + artifactName, + env: buildAvailabilityProbeEnv(), + timeoutMs: 60_000, + }, + ); +} + +async function destroyGatewayIfOpenshellExists( + host: import("../fixtures/clients/host.ts").HostCliClient, + artifactName: string, +): Promise { + await host.command( + "bash", + [ + "-lc", + "if command -v openshell >/dev/null 2>&1; then openshell gateway destroy -g nemoclaw; fi", + ], + { + artifactName, + env: buildAvailabilityProbeEnv(), + timeoutMs: 60_000, + }, + ); +} + +const liveTest = shouldRunLiveE2EScenarios() ? test : test.skip; + +liveTest( + "messaging token rotation rebuilds only the changed provider and reuses unchanged credentials", + testTimeoutOptions(PHASE_TIMEOUT_MS), + async ({ artifacts, cleanup, host, skip }) => { + expect( + fs.existsSync(CLI_ENTRYPOINT), + "run `npm run build:cli` before live repo CLI scenarios", + ).toBe(true); + + assertTokenPairsDiffer(); + + const docker = await host.command("docker", ["info"], { + artifactName: "prereq-docker-info-token-rotation", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }); + if (docker.exitCode !== 0) { + if (process.env.GITHUB_ACTIONS === "true") { + throw new Error(`Docker is required for token rotation live E2E: ${resultText(docker)}`); + } + skip("Docker is required for token rotation live E2E"); + } + + const fakeOpenAI = await startFakeOpenAIEndpoint(); + cleanup.add("stop fake OpenAI-compatible endpoint for token rotation", async () => { + await artifacts.writeJson("fake-openai-compatible-requests.json", fakeOpenAI.requests); + await fakeOpenAI.close(); + }); + + await artifacts.writeJson("scenario.json", { + id: "token-rotation", + runner: "vitest", + boundary: "direct-cli-onboard-openshell", + legacySource: "test/e2e/test-token-rotation.sh", + originalRunner: { + workflow: "nightly-e2e.yaml", + job: "token-rotation-e2e", + runsOn: "ubuntu-latest", + resources: [ + "Docker", + "install.sh/OpenShell", + "NVIDIA_API_KEY or fake OpenAI-compatible endpoint", + "fake messaging tokens", + ], + }, + replacementRunner: { + workflow: "e2e-vitest-scenarios.yaml", + job: "token-rotation-vitest", + runsOn: "ubuntu-latest", + resources: [ + "Docker", + "install.sh/OpenShell", + "hermetic fake OpenAI-compatible endpoint", + "fake messaging tokens", + ], + }, + documentedException: + "The replacement uses the legacy-supported fake OpenAI-compatible endpoint path so the messaging credential-rotation guard is not blocked by unrelated NVIDIA endpoint 429 rate limits.", + contract: [ + "first onboard stores messaging credential hashes and creates provider attachments", + "rotating Telegram rebuilds and names only telegram-bridge", + "unchanged tokens reuse the sandbox", + "rotating Discord rebuilds and names only discord-bridge", + "rotating Slack bot/app credentials rebuilds and names slack-bridge and slack-app only", + ], + }); + + const cleanupEnv = buildAvailabilityProbeEnv(); + cleanup.add(`destroy token-rotation sandbox ${SANDBOX_NAME}`, async () => { + await host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { + artifactName: "cleanup-nemoclaw-destroy-token-rotation", + env: cleanupEnv, + timeoutMs: 120_000, + }); + await deleteSandboxIfOpenshellExists(host, "cleanup-openshell-sandbox-delete-token-rotation"); + await destroyGatewayIfOpenshellExists( + host, + "cleanup-openshell-gateway-destroy-token-rotation", + ); + }); + + await host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { + artifactName: "pre-cleanup-nemoclaw-destroy-token-rotation", + env: cleanupEnv, + timeoutMs: 120_000, + }); + await deleteSandboxIfOpenshellExists( + host, + "pre-cleanup-openshell-sandbox-delete-token-rotation", + ); + await destroyGatewayIfOpenshellExists( + host, + "pre-cleanup-openshell-gateway-destroy-token-rotation", + ); + + const first = await runInstall(host, fakeOpenAI.baseUrl, TOKEN_A, { + NEMOCLAW_RECREATE_SANDBOX: "1", + }); + expect(first.exitCode, resultText(first)).toBe(0); + + const openshellVersion = await host.command("openshell", ["--version"], { + artifactName: "phase-0-openshell-version-token-rotation", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }); + expect(openshellVersion.exitCode, resultText(openshellVersion)).toBe(0); + + for (const providerName of [ + `${SANDBOX_NAME}-telegram-bridge`, + `${SANDBOX_NAME}-discord-bridge`, + `${SANDBOX_NAME}-slack-bridge`, + `${SANDBOX_NAME}-slack-app`, + ]) { + const provider = await host.command("openshell", ["provider", "get", providerName], { + artifactName: `phase-1-provider-get-${providerName}`, + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }); + expect(provider.exitCode, resultText(provider)).toBe(0); + } + + for (const envKey of [ + "TELEGRAM_BOT_TOKEN", + "DISCORD_BOT_TOKEN", + "SLACK_BOT_TOKEN", + "SLACK_APP_TOKEN", + ]) { + expectCredentialHash(envKey); + } + await assertSandboxRunning(host, "phase-1-sandbox-running-after-install"); + + const telegram = await runOnboard( + host, + fakeOpenAI.baseUrl, + { ...TOKEN_A, telegram: TOKEN_B.telegram }, + "phase-2-rotate-telegram", + ); + const telegramText = resultText(telegram); + expect(telegram.exitCode, telegramText).toBe(0); + expectRotationOutput( + telegramText, + [`${SANDBOX_NAME}-telegram-bridge`], + [ + `${SANDBOX_NAME}-discord-bridge`, + `${SANDBOX_NAME}-slack-bridge`, + `${SANDBOX_NAME}-slack-app`, + ], + ); + await assertSandboxRunning(host, "phase-2-sandbox-running-after-telegram-rotation"); + + const afterTelegramSame = await runOnboard( + host, + fakeOpenAI.baseUrl, + { ...TOKEN_A, telegram: TOKEN_B.telegram }, + "phase-3-same-after-telegram", + ); + const afterTelegramSameText = resultText(afterTelegramSame); + expect(afterTelegramSame.exitCode, afterTelegramSameText).toBe(0); + expect(afterTelegramSameText).toContain(`Sandbox '${SANDBOX_NAME}' exists and is ready`); + expect(afterTelegramSameText).toContain("reusing it"); + + const discord = await runOnboard( + host, + fakeOpenAI.baseUrl, + { ...TOKEN_A, telegram: TOKEN_B.telegram, discord: TOKEN_B.discord }, + "phase-4-rotate-discord", + ); + const discordText = resultText(discord); + expect(discord.exitCode, discordText).toBe(0); + expectRotationOutput( + discordText, + [`${SANDBOX_NAME}-discord-bridge`], + [ + `${SANDBOX_NAME}-telegram-bridge`, + `${SANDBOX_NAME}-slack-bridge`, + `${SANDBOX_NAME}-slack-app`, + ], + ); + await assertSandboxRunning(host, "phase-4-sandbox-running-after-discord-rotation"); + + const afterDiscordSame = await runOnboard( + host, + fakeOpenAI.baseUrl, + { ...TOKEN_A, telegram: TOKEN_B.telegram, discord: TOKEN_B.discord }, + "phase-5-same-after-discord", + ); + const afterDiscordSameText = resultText(afterDiscordSame); + expect(afterDiscordSame.exitCode, afterDiscordSameText).toBe(0); + expect(afterDiscordSameText).toContain(`Sandbox '${SANDBOX_NAME}' exists and is ready`); + expect(afterDiscordSameText).toContain("reusing it"); + + const slack = await runOnboard(host, fakeOpenAI.baseUrl, TOKEN_B, "phase-6-rotate-slack"); + const slackText = resultText(slack); + expect(slack.exitCode, slackText).toBe(0); + expectRotationOutput( + slackText, + [`${SANDBOX_NAME}-slack-bridge`, `${SANDBOX_NAME}-slack-app`], + [`${SANDBOX_NAME}-telegram-bridge`, `${SANDBOX_NAME}-discord-bridge`], + ); + await assertSandboxRunning(host, "phase-6-sandbox-running-after-slack-rotation"); + + const afterSlackSame = await runOnboard( + host, + fakeOpenAI.baseUrl, + TOKEN_B, + "phase-7-same-after-slack", + ); + const afterSlackSameText = resultText(afterSlackSame); + expect(afterSlackSame.exitCode, afterSlackSameText).toBe(0); + expect(afterSlackSameText).toContain(`Sandbox '${SANDBOX_NAME}' exists and is ready`); + expect(afterSlackSameText).toContain("reusing it"); + + await artifacts.writeJson("scenario-result.json", { + id: "token-rotation", + sandboxName: SANDBOX_NAME, + assertions: { + providersCreated: true, + credentialHashesStored: true, + telegramRotationIsolated: true, + discordRotationIsolated: true, + slackRotationIsolated: true, + unchangedTokensReuseSandbox: true, + }, + }); + }, +); diff --git a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts index 056f8b1dc0..f0d6b8ac4b 100644 --- a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts +++ b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts @@ -128,6 +128,42 @@ jobs: path: .e2e/onboard-negative-paths/ include-hidden-files: true if-no-files-found: error + token-rotation-vitest: + runs-on: macos-latest + needs: generate-matrix + if: \${{ inputs.scenarios != '' }} + timeout-minutes: 15 + env: + E2E_ARTIFACT_DIR: \${{ github.workspace }}/.e2e/token-rotation + NEMOCLAW_RUN_E2E_SCENARIOS: "0" + NEMOCLAW_CLI_BIN: /tmp/nemoclaw + NVIDIA_API_KEY: \${{ secrets.NVIDIA_API_KEY }} + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: true + - name: Authenticate to Docker Hub + env: + DOCKERHUB_USERNAME: bad + DOCKERHUB_TOKEN: bad + run: echo no-docker-login + - name: Set up Node + uses: actions/setup-node@v4 + - name: Install root dependencies + run: npm install + - name: Run token rotation live test + env: + NVIDIA_API_KEY: bad + GITHUB_TOKEN: bad + TELEGRAM_BOT_TOKEN_A: "" + run: npx vitest run --project e2e-scenarios-live "\${{ inputs.test_filter }}" + - name: Upload token rotation artifacts + uses: actions/upload-artifact@v4 + with: + name: token-rotation + path: .e2e/token-rotation/ + include-hidden-files: true + if-no-files-found: error `, ); @@ -143,11 +179,14 @@ jobs: "validate-jobs step must pass scenarios through SCENARIOS env", "step 'Validate free-standing job selector' run script must include Use either scenarios or jobs, not both", "step 'Validate free-standing job selector' run script must include allowed_jobs=", + "step 'Validate free-standing job selector' run script must include token-rotation-vitest", "step 'Validate free-standing job selector' run script must include Invalid jobs input; use comma-separated job ids", "step 'Validate free-standing job selector' run script must not include Invalid jobs input: ${JOBS}", "step 'Validate free-standing job selector' run script must include Unknown free-standing Vitest job", "workflow missing generate-matrix job", "generate-matrix job must run on ubuntu-latest", + "matrix generation step must pass scenarios through SCENARIOS env", + "matrix generation step must pass jobs through JOBS env", "live-scenarios job must run on the matrix runner", "live-scenarios job must depend on generate-matrix", "live-scenarios job must not run when a free-standing jobs selector is supplied", @@ -217,12 +256,44 @@ jobs: "onboard-negative-paths-vitest artifact upload must set include-hidden-files: false", "onboard-negative-paths-vitest artifact upload must ignore missing fixture artifacts", "onboard-negative-paths-vitest artifact upload retention-days must be 14", + "token-rotation-vitest job must run on ubuntu-latest", + "token-rotation-vitest job must depend on validate-jobs", + "token-rotation-vitest job must use the shared jobs selector condition", + "token-rotation-vitest job must keep the legacy 45 minute timeout", + "token-rotation-vitest job must set NEMOCLAW_RUN_E2E_SCENARIOS=1", + "token-rotation-vitest job must write artifacts under e2e-artifacts/vitest/token-rotation", + "token-rotation-vitest job must point NEMOCLAW_CLI_BIN at the repo CLI", + "token-rotation-vitest job env must not include NVIDIA_API_KEY", + "token-rotation-vitest checkout action must be pinned to a full commit SHA", + "token-rotation-vitest checkout step must set persist-credentials=false", + "token-rotation-vitest Docker Hub auth must receive DOCKERHUB_USERNAME from secrets", + "token-rotation-vitest Docker Hub auth must receive DOCKERHUB_TOKEN from secrets", + "step 'Authenticate to Docker Hub' run script must include docker login docker.io", + "token-rotation-vitest setup-node action must be pinned to a full commit SHA", + "token-rotation-vitest job missing step: Build CLI", + "token-rotation-vitest step must receive GITHUB_TOKEN from github.token", + "token-rotation-vitest step must set TELEGRAM_BOT_TOKEN_A", + "token-rotation-vitest step must set TELEGRAM_BOT_TOKEN_B", + "token-rotation-vitest step must set DISCORD_BOT_TOKEN_A", + "token-rotation-vitest step must set DISCORD_BOT_TOKEN_B", + "token-rotation-vitest step must set SLACK_BOT_TOKEN_A", + "token-rotation-vitest step must set SLACK_BOT_TOKEN_B", + "token-rotation-vitest step must set SLACK_APP_TOKEN_A", + "token-rotation-vitest step must set SLACK_APP_TOKEN_B", + "step 'Run token rotation live test' run script must include test/e2e-scenario/live/token-rotation.test.ts", + "token-rotation-vitest upload-artifact action must be pinned to a full commit SHA", + "token-rotation-vitest artifact upload name must be stable", + "artifact upload path must include e2e-artifacts/vitest/token-rotation/", + "token-rotation-vitest artifact upload must set include-hidden-files: false", + "token-rotation-vitest artifact upload must ignore missing fixture artifacts", + "token-rotation-vitest artifact upload retention-days must be 14", "openclaw-tui-chat-correlation-vitest job must depend on validate-jobs", "openclaw-tui-chat-correlation-vitest job must use the shared jobs selector condition", "gateway-guard-recovery job must depend on validate-jobs", "gateway-guard-recovery job must use the shared jobs selector condition", "report-to-pr job must wait for validate-jobs", "report-to-pr job must wait for live-scenarios", + "report-to-pr job must wait for token-rotation-vitest", "report-to-pr step must pass pr_number through JOB_PR_NUMBER env", "report-to-pr step must pass scenarios through JOB_SCENARIOS env", "step 'Post Vitest scenario results to PR' run script must include process.env.JOBS", @@ -235,4 +306,32 @@ jobs: fs.rmSync(tmp, { recursive: true, force: true }); } }); + + it("rejects raw jobs selector echo from matrix generation", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-vitest-workflow-")); + const workflowPath = path.join(tmp, "workflow.yaml"); + const workflow = fs.readFileSync( + path.join(process.cwd(), ".github/workflows/e2e-vitest-scenarios.yaml"), + "utf8", + ); + fs.writeFileSync( + workflowPath, + workflow.replace( + 'echo "::error::Invalid jobs input; use comma-separated job ids" >&2\n exit 1\n fi\n matrix=', + 'echo "::error::Invalid jobs input: ${JOBS}" >&2\n exit 1\n fi\n matrix=', + ), + ); + + try { + const errors = validateE2eVitestScenariosWorkflowBoundary(workflowPath); + expect(errors).toEqual( + expect.arrayContaining([ + "step 'Generate Vitest scenario matrix' run script must include Invalid jobs input; use comma-separated job ids", + "step 'Generate Vitest scenario matrix' run script must not include Invalid jobs input: ${JOBS}", + ]), + ); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); }); diff --git a/tools/e2e-scenarios/workflow-boundary.mts b/tools/e2e-scenarios/workflow-boundary.mts index 1189a0dd07..63f590e61d 100644 --- a/tools/e2e-scenarios/workflow-boundary.mts +++ b/tools/e2e-scenarios/workflow-boundary.mts @@ -163,6 +163,7 @@ function validateJobsSelector(errors: string[], jobs: WorkflowRecord): void { requireRunContains(errors, validate, "allowed_jobs="); requireRunContains(errors, validate, "openshell-version-pin-vitest"); requireRunContains(errors, validate, "onboard-negative-paths-vitest"); + requireRunContains(errors, validate, "token-rotation-vitest"); requireRunContains(errors, validate, "openclaw-tui-chat-correlation-vitest"); requireRunContains(errors, validate, "gateway-guard-recovery"); requireRunContains(errors, validate, "^[A-Za-z0-9_-]+(,[A-Za-z0-9_-]+)*$"); @@ -252,6 +253,125 @@ function validateOpenShellVersionPinVitestJob(errors: string[], jobs: WorkflowRe } +function validateTokenRotationVitestJob(errors: string[], jobs: WorkflowRecord): void { + const jobName = "token-rotation-vitest"; + const job = asRecord(jobs[jobName]); + if (Object.keys(job).length === 0) { + errors.push("workflow missing token-rotation-vitest job"); + return; + } + + if (job["runs-on"] !== "ubuntu-latest") { + errors.push("token-rotation-vitest job must run on ubuntu-latest"); + } + validateFreeStandingJobSelector(errors, jobs, jobName); + if (job["timeout-minutes"] !== 45) { + errors.push("token-rotation-vitest job must keep the legacy 45 minute timeout"); + } + const jobEnv = asRecord(job.env); + if (jobEnv.NEMOCLAW_RUN_E2E_SCENARIOS !== "1") { + errors.push("token-rotation-vitest job must set NEMOCLAW_RUN_E2E_SCENARIOS=1"); + } + if (jobEnv.E2E_ARTIFACT_DIR !== "${{ github.workspace }}/e2e-artifacts/vitest/token-rotation") { + errors.push("token-rotation-vitest job must write artifacts under e2e-artifacts/vitest/token-rotation"); + } + if (!stringValue(jobEnv.NEMOCLAW_CLI_BIN).includes("bin/nemoclaw.js")) { + errors.push("token-rotation-vitest job must point NEMOCLAW_CLI_BIN at the repo CLI"); + } + requireEnvDoesNotExposeSecret(errors, "token-rotation-vitest job", jobEnv, "NVIDIA_API_KEY"); + + const steps = asSteps(job.steps); + requireNoDispatchInputInterpolation(errors, steps); + for (const step of steps) { + if (step.name !== "Run token rotation live test") { + requireEnvDoesNotExposeSecret( + errors, + `token-rotation-vitest step '${step.name ?? step.uses ?? ""}'`, + asRecord(step.env), + "NVIDIA_API_KEY", + ); + } + } + + const checkout = steps.find((step) => stringValue(step.uses).startsWith("actions/checkout@")); + if (!checkout) errors.push("token-rotation-vitest job missing checkout step"); + requireFullShaAction(errors, checkout, "token-rotation-vitest checkout"); + if (asRecord(checkout?.with)["persist-credentials"] !== false) { + errors.push("token-rotation-vitest checkout step must set persist-credentials=false"); + } + + const dockerHubAuth = requireJobStep(errors, jobName, steps, "Authenticate to Docker Hub"); + const dockerHubEnv = asRecord(dockerHubAuth?.env); + if (dockerHubEnv.DOCKERHUB_USERNAME !== "${{ secrets.DOCKERHUB_USERNAME }}") { + errors.push("token-rotation-vitest Docker Hub auth must receive DOCKERHUB_USERNAME from secrets"); + } + if (dockerHubEnv.DOCKERHUB_TOKEN !== "${{ secrets.DOCKERHUB_TOKEN }}") { + errors.push("token-rotation-vitest Docker Hub auth must receive DOCKERHUB_TOKEN from secrets"); + } + requireRunContains(errors, dockerHubAuth, "docker login docker.io"); + + const setupNode = namedStep(steps, "Set up Node"); + if (!setupNode) errors.push("token-rotation-vitest job missing step: Set up Node"); + requireFullShaAction(errors, setupNode, "token-rotation-vitest setup-node"); + + const installRootDependencies = requireJobStep(errors, jobName, steps, "Install root dependencies"); + requireRunContains(errors, installRootDependencies, "npm ci --ignore-scripts"); + + const buildCli = requireJobStep(errors, jobName, steps, "Build CLI"); + requireRunContains(errors, buildCli, "npm run build:cli"); + + const runVitest = requireJobStep(errors, jobName, steps, "Run token rotation live test"); + const runVitestEnv = asRecord(runVitest?.env); + requireEnvDoesNotExposeSecret( + errors, + "token-rotation-vitest step", + runVitestEnv, + "NVIDIA_API_KEY", + ); + if (runVitestEnv.GITHUB_TOKEN !== "${{ github.token }}") { + errors.push("token-rotation-vitest step must receive GITHUB_TOKEN from github.token"); + } + for (const tokenName of [ + "TELEGRAM_BOT_TOKEN_A", + "TELEGRAM_BOT_TOKEN_B", + "DISCORD_BOT_TOKEN_A", + "DISCORD_BOT_TOKEN_B", + "SLACK_BOT_TOKEN_A", + "SLACK_BOT_TOKEN_B", + "SLACK_APP_TOKEN_A", + "SLACK_APP_TOKEN_B", + ]) { + const tokenValue = stringValue(runVitestEnv[tokenName]); + if ( + tokenValue.length === 0 || + tokenValue.includes("${{") || + !/^(test-fake-token-|dc-|xoxb-fake-|xapp-fake-)/.test(tokenValue) + ) { + errors.push(`token-rotation-vitest step must set ${tokenName}`); + } + } + requireRunContains(errors, runVitest, "npx vitest run --project e2e-scenarios-live"); + requireRunContains(errors, runVitest, "test/e2e-scenario/live/token-rotation.test.ts"); + + const upload = requireJobStep(errors, jobName, steps, "Upload token rotation artifacts"); + requireFullShaAction(errors, upload, "token-rotation-vitest upload-artifact"); + const uploadWith = asRecord(upload?.with); + if (uploadWith.name !== "e2e-vitest-scenarios-token-rotation") { + errors.push("token-rotation-vitest artifact upload name must be stable"); + } + const uploadPath = stringValue(uploadWith.path); + requireUploadPathContains(errors, uploadPath, "e2e-artifacts/vitest/token-rotation/"); + if (uploadWith["include-hidden-files"] !== false) { + errors.push("token-rotation-vitest artifact upload must set include-hidden-files: false"); + } + if (uploadWith["if-no-files-found"] !== "ignore") { + errors.push("token-rotation-vitest artifact upload must ignore missing fixture artifacts"); + } + if (uploadWith["retention-days"] !== 14) { + errors.push("token-rotation-vitest artifact upload retention-days must be 14"); + } +} + function validateOnboardNegativePathsVitestJob(errors: string[], jobs: WorkflowRecord): void { const jobName = "onboard-negative-paths-vitest"; const job = asRecord(jobs[jobName]); @@ -379,10 +499,15 @@ export function validateE2eVitestScenariosWorkflowBoundary( if (generateEnv.SCENARIOS !== "${{ inputs.scenarios }}") { errors.push("matrix generation step must pass scenarios through SCENARIOS env"); } + if (generateEnv.JOBS !== "${{ inputs.jobs }}") { + errors.push("matrix generation step must pass jobs through JOBS env"); + } requireRunContains(errors, generate, "npx tsx test/e2e-scenario/scenarios/run.ts"); requireRunContains(errors, generate, "--emit-live-matrix"); requireRunContains(errors, generate, "--scenarios"); requireRunContains(errors, generate, "^[A-Za-z0-9_-]+(,[A-Za-z0-9_-]+)*$"); + requireRunContains(errors, generate, "Invalid jobs input; use comma-separated job ids"); + requireRunDoesNotContain(errors, generate, "Invalid jobs input: ${JOBS}"); requireRunDoesNotContain(errors, generate, "^[A-Za-z0-9._-]+"); requireRunContains(errors, generate, "## Vitest E2E Scenario Matrix"); requireRunContains(errors, generate, "| Scenario | Runner | Label |"); @@ -523,6 +648,7 @@ export function validateE2eVitestScenariosWorkflowBoundary( validateOpenShellVersionPinVitestJob(errors, jobs); validateOnboardNegativePathsVitestJob(errors, jobs); + validateTokenRotationVitestJob(errors, jobs); validateFreeStandingJobSelector(errors, jobs, "openclaw-tui-chat-correlation-vitest"); validateFreeStandingJobSelector(errors, jobs, "gateway-guard-recovery"); @@ -537,6 +663,7 @@ export function validateE2eVitestScenariosWorkflowBoundary( "live-scenarios", "openshell-version-pin-vitest", "onboard-negative-paths-vitest", + "token-rotation-vitest", "openclaw-tui-chat-correlation-vitest", "gateway-guard-recovery", ]) {