From f001c41b84c0af4c305910de800dd0a925bc4edd Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Fri, 12 Jun 2026 12:15:32 -0700 Subject: [PATCH 1/2] test(e2e): migrate sessions agents cli scenario Signed-off-by: Carlos Villela --- .github/workflows/e2e-vitest-scenarios.yaml | 83 +++ .../live/sessions-agents-cli.test.ts | 564 ++++++++++++++++++ .../e2e-scenarios-workflow.test.ts | 18 +- tools/e2e-scenarios/free-standing-jobs.env | 6 +- tools/e2e-scenarios/workflow-boundary.mts | 1 + 5 files changed, 668 insertions(+), 4 deletions(-) create mode 100644 test/e2e-scenario/live/sessions-agents-cli.test.ts diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index f199050c83d..9fe52ce7230 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -853,7 +853,89 @@ jobs: if-no-files-found: ignore retention-days: 14 + sessions-agents-cli-vitest: + needs: generate-matrix + if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',sessions-agents-cli-vitest,') || contains(format(',{0},', inputs.scenarios), ',sessions-agents-cli,') }} + runs-on: ubuntu-latest + timeout-minutes: 70 + env: + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/sessions-agents-cli + NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js + NEMOCLAW_RUN_E2E_SCENARIOS: "1" + NEMOCLAW_SANDBOX_NAME: "e2e-sessions-agents-cli" + OPENSHELL_GATEWAY: "nemoclaw" + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + - name: Authenticate to Docker Hub + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + shell: bash + run: | + set -euo pipefail + if [[ -z "${DOCKERHUB_USERNAME}" || -z "${DOCKERHUB_TOKEN}" ]]; then + echo "::notice::Docker Hub credentials not configured; continuing with anonymous pulls." + exit 0 + fi + login_succeeded=0 + for attempt in 1 2 3; do + if echo "${DOCKERHUB_TOKEN}" | timeout 30s docker login docker.io --username "${DOCKERHUB_USERNAME}" --password-stdin; then + login_succeeded=1 + break + fi + if [[ "$attempt" -lt 3 ]]; then + echo "::warning::Docker Hub login attempt ${attempt} failed; retrying." + sleep 5 + fi + done + if [[ "$login_succeeded" -ne 1 ]]; then + echo "::warning::Docker Hub login failed after 3 attempts; continuing with anonymous pulls." + fi + + - name: Set up Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 + with: + node-version: 22 + cache: npm + + - name: Install root dependencies + run: npm ci --ignore-scripts + + - name: Build CLI + run: npm run build:cli + + - name: Install OpenShell CLI + run: | + set -euo pipefail + env -u DOCKER_CONFIG -u DOCKERHUB_USERNAME -u DOCKERHUB_TOKEN -u NVIDIA_API_KEY -u GITHUB_TOKEN \ + bash scripts/install-openshell.sh + + - name: Run sessions/agents CLI live test + # Migrated from test/e2e/test-sessions-agents-cli.sh. This stays as a + # host-side CLI live test because the contract is argv translation, + # OpenClaw gateway dispatch, pending pairing/scope approval, and JSON + # envelope handling for sessions/agents commands. + env: + NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} + run: | + set -euo pipefail + export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH" + npx vitest run --project e2e-scenarios-live \ + test/e2e-scenario/live/sessions-agents-cli.test.ts \ + --silent=false --reporter=default + + - name: Upload sessions/agents CLI artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: e2e-vitest-scenarios-sessions-agents-cli + path: e2e-artifacts/vitest/sessions-agents-cli/ + include-hidden-files: false + if-no-files-found: ignore + retention-days: 14 runtime-overrides-vitest: needs: generate-matrix @@ -1878,6 +1960,7 @@ jobs: inference-routing-vitest, credential-sanitization-vitest, credential-migration-vitest, + sessions-agents-cli-vitest, runtime-overrides-vitest, hermes-e2e-vitest, hermes-root-entrypoint-smoke-vitest, diff --git a/test/e2e-scenario/live/sessions-agents-cli.test.ts b/test/e2e-scenario/live/sessions-agents-cli.test.ts new file mode 100644 index 00000000000..4970e9ffe73 --- /dev/null +++ b/test/e2e-scenario/live/sessions-agents-cli.test.ts @@ -0,0 +1,564 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Live Vitest replacement for test/e2e/test-sessions-agents-cli.sh. + * + * Preserves the legacy host-side CLI contract for `nemoclaw sessions` + * and `nemoclaw agents`: real Docker/OpenShell onboarding, live NVIDIA + * credential gating, OpenClaw pairing/scope approval draining, JSON envelope + * parsing, and cleanup. OpenClaw still owns the in-sandbox session-store + * recovery semantics; this test stays at NemoClaw's argv translation, + * gateway dispatch, and JSON-envelope boundary. + */ + +import fs from "node:fs"; +import path from "node:path"; + +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { resultText } from "../fixtures/clients/command.ts"; +import type { HostCliClient } from "../fixtures/clients/host.ts"; +import { validateSandboxName } from "../fixtures/clients/sandbox.ts"; +import { expect, test } from "../fixtures/e2e-test.ts"; +import { shouldRunLiveE2EScenarios } from "../fixtures/live-project-gate.ts"; +import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; + +const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); +const CLI_ENTRYPOINT = path.join(REPO_ROOT, "bin", "nemoclaw.js"); +const CLI_DIST_ENTRYPOINT = path.join(REPO_ROOT, "dist", "nemoclaw.js"); +const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-sessions-agents-cli"; +const TEST_AGENT_ID = process.env.NEMOCLAW_E2E_AGENT_ID ?? "work"; +const ONBOARD_TIMEOUT_MS = 40 * 60_000; +const AGENT_TURN_TIMEOUT_MS = 5 * 60_000; +const GATEWAY_RPC_TIMEOUT_MS = 120_000; +const TEST_TIMEOUT_MS = 60 * 60_000; +const SCOPE_RETRY_ATTEMPTS = 5; +const SCOPE_RETRY_DELAY_MS = 4_000; +const SCOPE_PENDING_PATTERN = + /scope upgrade pending|Failed to reach the OpenClaw gateway|pairing required/i; + +process.env.NEMOCLAW_CLI_BIN ??= CLI_ENTRYPOINT; +validateSandboxName(SANDBOX_NAME); + +type JsonRecord = Record; +type CommandOptions = { + artifactName: string; + env?: NodeJS.ProcessEnv; + timeoutMs?: number; + redactionValues?: string[]; +}; + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function stripAnsi(value: string): string { + return value.replace(/\u001B\[[0-?]*[ -/]*[@-~]/g, ""); +} + +function commandEnv(extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { + const base = buildAvailabilityProbeEnv(); + return { + ...base, + NODE_NO_WARNINGS: "1", + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", + NEMOCLAW_AGENT: "openclaw", + NEMOCLAW_PROVIDER: "cloud", + NEMOCLAW_POLICY_TIER: "open", + NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, + OPENSHELL_GATEWAY: process.env.OPENSHELL_GATEWAY ?? "nemoclaw", + PATH: [path.join(process.env.HOME ?? "", ".local", "bin"), base.PATH].filter(Boolean).join(":"), + ...extra, + }; +} + +async function bestEffort(run: () => Promise): Promise { + try { + await run(); + } catch { + // Cleanup is best effort so an early setup failure keeps the original error. + } +} + +async function runNemoclaw( + host: HostCliClient, + args: string[], + apiKey: string, + options: CommandOptions, +): Promise { + return await host.command("node", [CLI_ENTRYPOINT, ...args], { + artifactName: options.artifactName, + env: commandEnv({ + NVIDIA_API_KEY: apiKey, + ...(options.env ?? {}), + }), + redactionValues: [apiKey, ...(options.redactionValues ?? [])], + timeoutMs: options.timeoutMs ?? GATEWAY_RPC_TIMEOUT_MS, + }); +} + +async function ensureOpenshellAvailable(host: HostCliClient): Promise { + const current = await host.command( + "bash", + ["-lc", "command -v openshell && openshell --version"], + { + artifactName: "prereq-openshell-version", + env: commandEnv(), + timeoutMs: 30_000, + }, + ); + if (current.exitCode === 0) return; + + const install = await host.command( + "bash", + [path.join(REPO_ROOT, "scripts", "install-openshell.sh")], + { + artifactName: "prereq-install-openshell", + cwd: REPO_ROOT, + env: commandEnv(), + timeoutMs: 10 * 60_000, + }, + ); + expect(install.exitCode, `install-openshell.sh failed\n${resultText(install)}`).toBe(0); + + const afterInstall = await host.command( + "bash", + ["-lc", "command -v openshell && openshell --version"], + { + artifactName: "prereq-openshell-version-after-install", + env: commandEnv(), + timeoutMs: 30_000, + }, + ); + expect( + afterInstall.exitCode, + `openshell missing after install\n${resultText(afterInstall)}`, + ).toBe(0); +} + +async function cleanupSandbox(host: HostCliClient, apiKey: string): Promise { + if (process.env.NEMOCLAW_E2E_KEEP_SANDBOX === "1") return; + + // Match the legacy teardown helper's tolerance: a failed preflight or + // provider-validation skip may happen before a sandbox exists. + await bestEffort(() => + runNemoclaw(host, [SANDBOX_NAME, "destroy", "--yes"], apiKey, { + artifactName: "cleanup-nemoclaw-destroy-sessions-agents-cli", + timeoutMs: 5 * 60_000, + }), + ); + await bestEffort(() => + host.command("openshell", ["sandbox", "delete", SANDBOX_NAME], { + artifactName: "cleanup-openshell-sandbox-delete-sessions-agents-cli", + env: commandEnv(), + timeoutMs: 60_000, + }), + ); +} + +function isPreContractEndpointValidationRateLimit(result: ShellProbeResult): boolean { + const text = resultText(result); + return ( + /NVIDIA Endpoints endpoint validation failed|endpoint validation failed/i.test(text) && + /HTTP 429|\b429\b|rate[- ]?limit|quota|temporarily unavailable|Validation details were omitted to avoid exposing credentials/i.test( + text, + ) + ); +} + +function tailEvidence(text: string, maxLength = 2_000): string { + return text.length > maxLength ? text.slice(-maxLength) : text; +} + +function parseJsonFromText(raw: string): unknown { + const text = stripAnsi(raw); + let cursor = 0; + for (const lineWithBreak of text.match(/^.*(?:\r?\n|$)/gm) ?? []) { + const line = lineWithBreak.replace(/\r?\n$/, ""); + const trimmed = line.trimStart(); + if (trimmed.startsWith("{") || trimmed.startsWith("[")) { + const offset = cursor + line.length - trimmed.length; + return JSON.parse(text.slice(offset)); + } + cursor += lineWithBreak.length; + } + throw new Error("no JSON object or array found"); +} + +function parseJsonEnvelope(result: ShellProbeResult, label: string): unknown { + const candidates = [result.stdout, resultText(result)].filter((candidate) => candidate.trim()); + for (const candidate of candidates) { + try { + const parsed = parseJsonFromText(candidate); + expect( + typeof parsed === "object" && parsed !== null, + `${label} JSON envelope must be an object or array`, + ).toBe(true); + return parsed; + } catch { + // Try the next candidate. stdout is preferred because stderr can carry + // process warnings after an otherwise valid JSON payload. + } + } + throw new Error(`${label} did not contain parseable JSON:\n${resultText(result)}`); +} + +function asRecord(value: unknown): JsonRecord | undefined { + return value && typeof value === "object" && !Array.isArray(value) + ? (value as JsonRecord) + : undefined; +} + +function sessionEntries(envelope: unknown): JsonRecord[] { + if (Array.isArray(envelope)) + return envelope.filter((entry): entry is JsonRecord => !!asRecord(entry)); + const sessions = asRecord(envelope)?.sessions; + return Array.isArray(sessions) + ? sessions.filter((entry): entry is JsonRecord => !!asRecord(entry)) + : []; +} + +function agentEntries(envelope: unknown): JsonRecord[] { + if (Array.isArray(envelope)) + return envelope.filter((entry): entry is JsonRecord => !!asRecord(entry)); + const agents = asRecord(envelope)?.agents; + return Array.isArray(agents) + ? agents.filter((entry): entry is JsonRecord => !!asRecord(entry)) + : []; +} + +function firstSessionKey(envelope: unknown): string | undefined { + for (const entry of sessionEntries(envelope)) { + if (typeof entry.key === "string" && entry.key.length > 0) return entry.key; + } + return undefined; +} + +function pendingRequestIds(envelope: unknown): string[] { + const pending = asRecord(envelope)?.pending; + if (!Array.isArray(pending)) return []; + return pending + .map((entry) => asRecord(entry)?.requestId ?? asRecord(entry)?.id) + .filter((id): id is string => typeof id === "string" && id.length > 0); +} + +async function approvePendingPairingRequests( + host: HostCliClient, + apiKey: string, + artifactPrefix: string, +): Promise { + for (let attempt = 1; attempt <= 10; attempt += 1) { + const state = await runNemoclaw( + host, + [SANDBOX_NAME, "exec", "--", "openclaw", "devices", "list", "--json"], + apiKey, + { + artifactName: `${artifactPrefix}-devices-list-${attempt}`, + timeoutMs: 60_000, + }, + ); + if (state.exitCode !== 0 || !state.stdout.trim()) { + await sleep(3_000); + continue; + } + + let ids: string[] = []; + try { + ids = pendingRequestIds(parseJsonEnvelope(state, "openclaw devices list --json")); + } catch { + await sleep(3_000); + continue; + } + if (ids.length === 0) return true; + + for (const id of ids) { + await runNemoclaw( + host, + [SANDBOX_NAME, "exec", "--", "openclaw", "devices", "approve", id, "--json"], + apiKey, + { + artifactName: `${artifactPrefix}-devices-approve-${id}`, + timeoutMs: 60_000, + }, + ); + } + await sleep(3_000); + } + return false; +} + +async function runGatewayRpcWithScopeRetry( + host: HostCliClient, + args: string[], + apiKey: string, + artifactName: string, +): Promise { + let lastResult: ShellProbeResult | undefined; + for (let attempt = 1; attempt <= SCOPE_RETRY_ATTEMPTS; attempt += 1) { + lastResult = await runNemoclaw(host, args, apiKey, { + artifactName: `${artifactName}-attempt-${attempt}`, + timeoutMs: GATEWAY_RPC_TIMEOUT_MS, + }); + if (lastResult.exitCode === 0) return lastResult; + if (!SCOPE_PENDING_PATTERN.test(resultText(lastResult))) break; + await approvePendingPairingRequests(host, apiKey, `${artifactName}-scope-${attempt}`); + await sleep(SCOPE_RETRY_DELAY_MS); + } + return lastResult!; +} + +async function expectJsonCommand( + host: HostCliClient, + args: string[], + apiKey: string, + artifactName: string, +): Promise { + const result = await runNemoclaw(host, args, apiKey, { artifactName }); + expect(result.exitCode, `${args.join(" ")} failed\n${resultText(result)}`).toBe(0); + return parseJsonEnvelope(result, args.join(" ")); +} + +const runSessionsAgentsCliTest = shouldRunLiveE2EScenarios() ? test : test.skip; + +runSessionsAgentsCliTest( + "sessions/agents host CLI routes to OpenClaw and preserves JSON envelopes", + { timeout: TEST_TIMEOUT_MS }, + async ({ artifacts, cleanup, host, secrets, skip }) => { + expect(fs.existsSync(CLI_ENTRYPOINT), "bin/nemoclaw.js missing").toBe(true); + expect( + fs.existsSync(CLI_DIST_ENTRYPOINT), + "run `npm run build:cli` before live repo CLI scenarios", + ).toBe(true); + + await artifacts.writeJson("scenario.json", { + id: "sessions-agents-cli", + runner: "vitest", + boundary: "host-cli-openclaw-sessions-agents-gateway", + migratedFrom: "test/e2e/test-sessions-agents-cli.sh", + sandboxName: SANDBOX_NAME, + contracts: [ + "NVIDIA_API_KEY absence skips the live credential-gated scenario", + "nemoclaw sessions --json defaults to OpenClaw sessions list", + "nemoclaw sessions list --json returns a parseable JSON envelope", + "sessions reset/delete gateway RPCs retry through pending pairing/scope approval", + "nemoclaw agents add/list/delete pass through to the in-sandbox OpenClaw CLI", + "the secondary agent session key is removed through sessions delete --json", + "cleanup destroys the named sandbox unless NEMOCLAW_E2E_KEEP_SANDBOX=1", + ], + }); + + const docker = await host.command("docker", ["info"], { + artifactName: "prereq-docker-info-sessions-agents-cli", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }); + expect( + docker.exitCode, + `Docker is required for sessions/agents E2E\n${resultText(docker)}`, + ).toBe(0); + + const apiKey = secrets.required("NVIDIA_API_KEY"); + expect(apiKey.startsWith("nvapi-"), "NVIDIA_API_KEY must start with nvapi-").toBe(true); + + await ensureOpenshellAvailable(host); + cleanup.add(`destroy sessions/agents sandbox ${SANDBOX_NAME}`, async () => + cleanupSandbox(host, apiKey), + ); + await cleanupSandbox(host, apiKey); + fs.rmSync(path.join(process.env.HOME ?? "", ".nemoclaw", "onboard.lock"), { force: true }); + + const onboard = await runNemoclaw( + host, + ["onboard", "--non-interactive", "--yes-i-accept-third-party-software"], + apiKey, + { + artifactName: "onboard-sessions-agents-cli", + timeoutMs: ONBOARD_TIMEOUT_MS, + }, + ); + if (onboard.exitCode !== 0 && isPreContractEndpointValidationRateLimit(onboard)) { + await artifacts.writeJson("onboard-endpoint-validation-skip.json", { + reason: + "NVIDIA endpoint validation was externally rate-limited or sanitized before the sessions/agents CLI contract could run.", + exitCode: onboard.exitCode, + stdoutTail: tailEvidence(onboard.stdout), + stderrTail: tailEvidence(onboard.stderr), + }); + skip( + "NVIDIA endpoint validation hit HTTP 429/sanitized failure before sessions/agents CLI contract could run", + ); + } + expect(onboard.exitCode, `onboard failed\n${resultText(onboard)}`).toBe(0); + + await approvePendingPairingRequests(host, apiKey, "post-onboard-scope"); + + const mainSeed = await runNemoclaw( + host, + [SANDBOX_NAME, "exec", "--", "openclaw", "agent", "--agent", "main", "-m", "ping"], + apiKey, + { + artifactName: "seed-main-session", + timeoutMs: AGENT_TURN_TIMEOUT_MS, + }, + ); + + if (mainSeed.exitCode === 0) { + await approvePendingPairingRequests(host, apiKey, "post-main-seed-scope"); + await expectJsonCommand( + host, + [SANDBOX_NAME, "sessions", "--json"], + apiKey, + "tc-sess-01-sessions-default-json", + ); + await expectJsonCommand( + host, + [SANDBOX_NAME, "sessions", "list", "--json"], + apiKey, + "tc-sess-02-sessions-list-json", + ); + + const reset = await runGatewayRpcWithScopeRetry( + host, + [SANDBOX_NAME, "sessions", "reset", "agent:main:main", "--json"], + apiKey, + "tc-sess-03-sessions-reset-main-json", + ); + expect(reset.exitCode, `sessions reset failed\n${resultText(reset)}`).toBe(0); + const resetEnvelope = parseJsonEnvelope(reset, "sessions reset --json"); + expect(asRecord(resetEnvelope)?.key, "sessions reset JSON must include key").toBe( + "agent:main:main", + ); + + await expectJsonCommand( + host, + [SANDBOX_NAME, "sessions", "list", "--json"], + apiKey, + "tc-sess-04-sessions-list-after-reset-json", + ); + } else { + await artifacts.writeJson("main-session-cases-skipped.json", { + reason: "main agent seed failed; preserving legacy TC-SESS-01..04 skip behavior", + exitCode: mainSeed.exitCode, + stderr: mainSeed.stderr, + }); + } + + const addAgent = await runNemoclaw( + host, + [ + SANDBOX_NAME, + "agents", + "add", + TEST_AGENT_ID, + "--workspace", + `/sandbox/.openclaw/workspace-${TEST_AGENT_ID}`, + "--non-interactive", + ], + apiKey, + { + artifactName: "tc-agent-01-agents-add-passthrough", + timeoutMs: 120_000, + }, + ); + expect(addAgent.exitCode, `agents add failed\n${resultText(addAgent)}`).toBe(0); + + await expectJsonCommand( + host, + [SANDBOX_NAME, "sessions", "list", "--agent", TEST_AGENT_ID, "--json"], + apiKey, + "tc-agent-01-sessions-list-agent-after-add-json", + ); + + const agentsList = await expectJsonCommand( + host, + [SANDBOX_NAME, "agents", "list", "--json"], + apiKey, + "tc-agent-03-agents-list-json", + ); + expect( + agentEntries(agentsList).some((entry) => entry.id === TEST_AGENT_ID), + `agents list --json must include '${TEST_AGENT_ID}'`, + ).toBe(true); + + const workSeed = await runNemoclaw( + host, + [SANDBOX_NAME, "exec", "--", "openclaw", "agent", "--agent", TEST_AGENT_ID, "-m", "ping"], + apiKey, + { + artifactName: "seed-work-agent-session", + timeoutMs: AGENT_TURN_TIMEOUT_MS, + }, + ); + expect(workSeed.exitCode, `work-agent seed failed\n${resultText(workSeed)}`).toBe(0); + + await approvePendingPairingRequests(host, apiKey, "post-work-seed-scope"); + const workSessions = await expectJsonCommand( + host, + [SANDBOX_NAME, "sessions", "list", "--agent", TEST_AGENT_ID, "--json"], + apiKey, + "tc-sess-05-work-agent-sessions-json", + ); + const sessionKey = firstSessionKey(workSessions); + expect( + sessionKey, + `expected a session key for agent '${TEST_AGENT_ID}' after seed prompt`, + ).toBeTruthy(); + + const deleteSession = await runGatewayRpcWithScopeRetry( + host, + [SANDBOX_NAME, "sessions", "delete", sessionKey!, "--json"], + apiKey, + "tc-sess-05-sessions-delete-json", + ); + expect(deleteSession.exitCode, `sessions delete failed\n${resultText(deleteSession)}`).toBe(0); + const deleteEnvelope = parseJsonEnvelope(deleteSession, "sessions delete --json"); + expect(asRecord(deleteEnvelope)?.key, "sessions delete JSON must include deleted key").toBe( + sessionKey, + ); + + const workSessionsAfterDelete = await expectJsonCommand( + host, + [SANDBOX_NAME, "sessions", "list", "--agent", TEST_AGENT_ID, "--json"], + apiKey, + "tc-sess-05-work-agent-sessions-after-delete-json", + ); + expect( + sessionEntries(workSessionsAfterDelete).some((entry) => entry.key === sessionKey), + `session key '${sessionKey}' must be absent after delete`, + ).toBe(false); + + const deleteAgent = await runNemoclaw( + host, + [SANDBOX_NAME, "agents", "delete", TEST_AGENT_ID, "--force", "--json"], + apiKey, + { + artifactName: "tc-agent-02-agents-delete-json", + timeoutMs: 120_000, + }, + ); + expect(deleteAgent.exitCode, `agents delete failed\n${resultText(deleteAgent)}`).toBe(0); + + const deletedAgentSessions = await runNemoclaw( + host, + [SANDBOX_NAME, "sessions", "list", "--agent", TEST_AGENT_ID, "--json"], + apiKey, + { + artifactName: "tc-agent-02-sessions-list-after-agent-delete", + timeoutMs: 60_000, + }, + ); + let deletedAgentStillVisible = false; + if (deletedAgentSessions.exitCode === 0) { + try { + parseJsonEnvelope(deletedAgentSessions, "sessions list deleted agent --json"); + deletedAgentStillVisible = true; + } catch { + deletedAgentStillVisible = false; + } + } + expect(deletedAgentStillVisible, `agent '${TEST_AGENT_ID}' still visible after delete`).toBe( + false, + ); + }, +); diff --git a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts index 354712ecde2..e23937abdd8 100644 --- a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts +++ b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts @@ -149,6 +149,22 @@ describe("e2e-vitest-scenarios workflow boundary", () => { selectedFreeStandingJobs: ["credential-sanitization-vitest"], registryScenarios: [], }); + expect( + evaluateE2eVitestWorkflowDispatchSelectors({ scenarios: "sessions-agents-cli" }), + ).toMatchObject({ + valid: true, + liveScenariosRuns: false, + selectedFreeStandingJobs: ["sessions-agents-cli-vitest"], + registryScenarios: [], + }); + expect( + evaluateE2eVitestWorkflowDispatchSelectors({ jobs: "sessions-agents-cli-vitest" }), + ).toMatchObject({ + valid: true, + liveScenariosRuns: false, + selectedFreeStandingJobs: ["sessions-agents-cli-vitest"], + registryScenarios: [], + }); expect( evaluateE2eVitestWorkflowDispatchSelectors({ jobs: "runtime-overrides-vitest" }), ).toMatchObject({ @@ -376,7 +392,7 @@ describe("e2e-vitest-scenarios workflow boundary", () => { registryScenarios: [], }); } - }); + }, 15_000); it("flags direct dispatch-input interpolation and unsafe artifact upload", () => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-vitest-workflow-")); diff --git a/tools/e2e-scenarios/free-standing-jobs.env b/tools/e2e-scenarios/free-standing-jobs.env index 68ce23e36df..07d7e9962ce 100644 --- a/tools/e2e-scenarios/free-standing-jobs.env +++ b/tools/e2e-scenarios/free-standing-jobs.env @@ -1,5 +1,5 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -allowed_jobs=openshell-version-pin-vitest,onboard-negative-paths-vitest,skill-agent-vitest,inference-routing-vitest,credential-migration-vitest,runtime-overrides-vitest,hermes-e2e-vitest,hermes-root-entrypoint-smoke-vitest,network-policy-vitest,shields-config-vitest,rebuild-openclaw-vitest,sandbox-rebuild-vitest,token-rotation-vitest,launchable-smoke-vitest,openclaw-tui-chat-correlation-vitest,gateway-guard-recovery,double-onboard-vitest,issue-4434-tui-unreachable-inference-vitest,model-router-provider-routed-inference-vitest,credential-sanitization-vitest,sandbox-survival-vitest -free_standing_scenarios_csv=openshell-version-pin,onboard-negative-paths,skill-agent,inference-routing,runtime-overrides,hermes-e2e,hermes-root-entrypoint-smoke,network-policy,shields-config,rebuild-openclaw,sandbox-rebuild,token-rotation,openclaw-tui-chat-correlation,double-onboard,issue-4434-tui-unreachable-inference,model-router-provider-routed-inference,credential-sanitization,sandbox-survival -free_standing_scenario_jobs_csv=openshell-version-pin:openshell-version-pin-vitest,onboard-negative-paths:onboard-negative-paths-vitest,skill-agent:skill-agent-vitest,inference-routing:inference-routing-vitest,runtime-overrides:runtime-overrides-vitest,hermes-e2e:hermes-e2e-vitest,hermes-root-entrypoint-smoke:hermes-root-entrypoint-smoke-vitest,network-policy:network-policy-vitest,shields-config:shields-config-vitest,rebuild-openclaw:rebuild-openclaw-vitest,sandbox-rebuild:sandbox-rebuild-vitest,token-rotation:token-rotation-vitest,openclaw-tui-chat-correlation:openclaw-tui-chat-correlation-vitest,double-onboard:double-onboard-vitest,issue-4434-tui-unreachable-inference:issue-4434-tui-unreachable-inference-vitest,model-router-provider-routed-inference:model-router-provider-routed-inference-vitest,credential-sanitization:credential-sanitization-vitest,sandbox-survival:sandbox-survival-vitest +allowed_jobs=openshell-version-pin-vitest,onboard-negative-paths-vitest,skill-agent-vitest,inference-routing-vitest,credential-migration-vitest,runtime-overrides-vitest,hermes-e2e-vitest,hermes-root-entrypoint-smoke-vitest,network-policy-vitest,shields-config-vitest,rebuild-openclaw-vitest,sandbox-rebuild-vitest,token-rotation-vitest,launchable-smoke-vitest,openclaw-tui-chat-correlation-vitest,gateway-guard-recovery,double-onboard-vitest,issue-4434-tui-unreachable-inference-vitest,model-router-provider-routed-inference-vitest,credential-sanitization-vitest,sandbox-survival-vitest,sessions-agents-cli-vitest +free_standing_scenarios_csv=openshell-version-pin,onboard-negative-paths,skill-agent,inference-routing,runtime-overrides,hermes-e2e,hermes-root-entrypoint-smoke,network-policy,shields-config,rebuild-openclaw,sandbox-rebuild,token-rotation,openclaw-tui-chat-correlation,double-onboard,issue-4434-tui-unreachable-inference,model-router-provider-routed-inference,credential-sanitization,sandbox-survival,sessions-agents-cli +free_standing_scenario_jobs_csv=openshell-version-pin:openshell-version-pin-vitest,onboard-negative-paths:onboard-negative-paths-vitest,skill-agent:skill-agent-vitest,inference-routing:inference-routing-vitest,runtime-overrides:runtime-overrides-vitest,hermes-e2e:hermes-e2e-vitest,hermes-root-entrypoint-smoke:hermes-root-entrypoint-smoke-vitest,network-policy:network-policy-vitest,shields-config:shields-config-vitest,rebuild-openclaw:rebuild-openclaw-vitest,sandbox-rebuild:sandbox-rebuild-vitest,token-rotation:token-rotation-vitest,openclaw-tui-chat-correlation:openclaw-tui-chat-correlation-vitest,double-onboard:double-onboard-vitest,issue-4434-tui-unreachable-inference:issue-4434-tui-unreachable-inference-vitest,model-router-provider-routed-inference:model-router-provider-routed-inference-vitest,credential-sanitization:credential-sanitization-vitest,sandbox-survival:sandbox-survival-vitest,sessions-agents-cli:sessions-agents-cli-vitest diff --git a/tools/e2e-scenarios/workflow-boundary.mts b/tools/e2e-scenarios/workflow-boundary.mts index 9e55fc3ff31..dd80d98a4c0 100644 --- a/tools/e2e-scenarios/workflow-boundary.mts +++ b/tools/e2e-scenarios/workflow-boundary.mts @@ -2109,6 +2109,7 @@ export function validateE2eVitestScenariosWorkflowBoundary( validateOnboardNegativePathsVitestJob(errors, jobs); validateSkillAgentVitestJob(errors, jobs); validateFreeStandingJobSelector(errors, jobs, "credential-migration-vitest"); + validateFreeStandingJobSelector(errors, jobs, "sessions-agents-cli-vitest", "sessions-agents-cli"); validateFreeStandingJobSelector(errors, jobs, "inference-routing-vitest", "inference-routing"); validateRuntimeOverridesVitestJob(errors, jobs); validateDoubleOnboardVitestJob(errors, jobs); From 1fa9b335f42fa78ae35c4721d509859b3b73a617 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Fri, 12 Jun 2026 14:30:27 -0700 Subject: [PATCH 2/2] test(e2e): tighten sessions agents assertions Signed-off-by: Carlos Villela --- .../live/sessions-agents-cli.test.ts | 64 +++++++++---------- 1 file changed, 30 insertions(+), 34 deletions(-) diff --git a/test/e2e-scenario/live/sessions-agents-cli.test.ts b/test/e2e-scenario/live/sessions-agents-cli.test.ts index 4970e9ffe73..3744d367a37 100644 --- a/test/e2e-scenario/live/sessions-agents-cli.test.ts +++ b/test/e2e-scenario/live/sessions-agents-cli.test.ts @@ -140,21 +140,28 @@ async function ensureOpenshellAvailable(host: HostCliClient): Promise { async function cleanupSandbox(host: HostCliClient, apiKey: string): Promise { if (process.env.NEMOCLAW_E2E_KEEP_SANDBOX === "1") return; - // Match the legacy teardown helper's tolerance: a failed preflight or - // provider-validation skip may happen before a sandbox exists. - await bestEffort(() => - runNemoclaw(host, [SANDBOX_NAME, "destroy", "--yes"], apiKey, { - artifactName: "cleanup-nemoclaw-destroy-sessions-agents-cli", - timeoutMs: 5 * 60_000, - }), - ); - await bestEffort(() => - host.command("openshell", ["sandbox", "delete", SANDBOX_NAME], { - artifactName: "cleanup-openshell-sandbox-delete-sessions-agents-cli", - env: commandEnv(), - timeoutMs: 60_000, - }), - ); + const destroy = await runNemoclaw(host, [SANDBOX_NAME, "destroy", "--yes"], apiKey, { + artifactName: "cleanup-nemoclaw-destroy-sessions-agents-cli", + timeoutMs: 5 * 60_000, + }); + expect( + destroy.exitCode === 0 || isMissingSandboxCleanupOutput(resultText(destroy)), + `cleanup NemoClaw destroy failed\n${resultText(destroy)}`, + ).toBe(true); + + const openshellDelete = await host.command("openshell", ["sandbox", "delete", SANDBOX_NAME], { + artifactName: "cleanup-openshell-sandbox-delete-sessions-agents-cli", + env: commandEnv(), + timeoutMs: 60_000, + }); + expect( + openshellDelete.exitCode === 0 || isMissingSandboxCleanupOutput(resultText(openshellDelete)), + `cleanup OpenShell sandbox delete failed\n${resultText(openshellDelete)}`, + ).toBe(true); +} + +function isMissingSandboxCleanupOutput(text: string): boolean { + return /does not exist|not found|Run 'nemoclaw onboard'|no such sandbox/i.test(text); } function isPreContractEndpointValidationRateLimit(result: ShellProbeResult): boolean { @@ -363,7 +370,7 @@ runSessionsAgentsCliTest( await ensureOpenshellAvailable(host); cleanup.add(`destroy sessions/agents sandbox ${SANDBOX_NAME}`, async () => - cleanupSandbox(host, apiKey), + bestEffort(() => cleanupSandbox(host, apiKey)), ); await cleanupSandbox(host, apiKey); fs.rmSync(path.join(process.env.HOME ?? "", ".nemoclaw", "onboard.lock"), { force: true }); @@ -539,26 +546,15 @@ runSessionsAgentsCliTest( ); expect(deleteAgent.exitCode, `agents delete failed\n${resultText(deleteAgent)}`).toBe(0); - const deletedAgentSessions = await runNemoclaw( + const agentsAfterDelete = await expectJsonCommand( host, - [SANDBOX_NAME, "sessions", "list", "--agent", TEST_AGENT_ID, "--json"], + [SANDBOX_NAME, "agents", "list", "--json"], apiKey, - { - artifactName: "tc-agent-02-sessions-list-after-agent-delete", - timeoutMs: 60_000, - }, - ); - let deletedAgentStillVisible = false; - if (deletedAgentSessions.exitCode === 0) { - try { - parseJsonEnvelope(deletedAgentSessions, "sessions list deleted agent --json"); - deletedAgentStillVisible = true; - } catch { - deletedAgentStillVisible = false; - } - } - expect(deletedAgentStillVisible, `agent '${TEST_AGENT_ID}' still visible after delete`).toBe( - false, + "tc-agent-02-agents-list-after-delete-json", ); + expect( + agentEntries(agentsAfterDelete).some((entry) => entry.id === TEST_AGENT_ID), + `agent '${TEST_AGENT_ID}' still visible after delete`, + ).toBe(false); }, );