diff --git a/test/dcode-wrapper-empty-prompt.test.ts b/test/dcode-wrapper-empty-prompt.test.ts index f5920982e34..2dc7c7c8969 100644 --- a/test/dcode-wrapper-empty-prompt.test.ts +++ b/test/dcode-wrapper-empty-prompt.test.ts @@ -48,12 +48,17 @@ type WrapperRun = { function runWrapper(args: string[]): WrapperRun { const dir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-dcode-wrapper-")); try { - fs.copyFileSync(WRAPPER, path.join(dir, "dcode")); - fs.chmodSync(path.join(dir, "dcode"), 0o755); - const marker = path.join(dir, "launched.txt"); const bin = path.join(dir, "bin"); fs.mkdirSync(bin); + const wrapperFixture = fs + .readFileSync(WRAPPER, "utf8") + .replace( + 'export PATH="/usr/local/bin:/opt/venv/bin:/usr/local/sbin:/usr/sbin:/usr/bin:/sbin:/bin"', + `export PATH="${bin}:/usr/local/bin:/opt/venv/bin:/usr/local/sbin:/usr/sbin:/usr/bin:/sbin:/bin"`, + ); + fs.writeFileSync(path.join(dir, "dcode"), wrapperFixture, { mode: 0o755 }); + fs.writeFileSync( path.join(bin, "python3"), `#!/usr/bin/env bash\nprintf '%s' "$*" > ${JSON.stringify(marker)}\nexit 0\n`, diff --git a/test/e2e-scenario/fixtures/phases/onboarding.ts b/test/e2e-scenario/fixtures/phases/onboarding.ts index 3efa991d7ca..4d8d583144b 100644 --- a/test/e2e-scenario/fixtures/phases/onboarding.ts +++ b/test/e2e-scenario/fixtures/phases/onboarding.ts @@ -70,7 +70,7 @@ export interface OnboardingExpectedFailure { export interface NemoClawInstance { onboarding: string; sandboxName: string; - agent: "openclaw" | "hermes"; + agent: "openclaw" | "hermes" | "langchain-deepagents-code"; provider: "nvidia" | "ollama"; providerEnv: "cloud" | "local"; platformOs?: "ubuntu" | "macos" | "windows"; @@ -180,6 +180,9 @@ export class OnboardingPhaseFixture { case "cloud-openclaw-no-docker": result = await this.cloudOpenClawNoDocker(environment, options); break; + case "cloud-langchain-deepagents-code": + result = await this.cloudLangchainDeepAgentsCode(environment, options); + break; default: throw new Error(`Unsupported onboarding profile '${environment.onboarding}'.`); } @@ -219,6 +222,53 @@ export class OnboardingPhaseFixture { }; } + async cloudLangchainDeepAgentsCode( + environment: EnvironmentReady, + options: OnboardingOptions = {}, + ): Promise { + if (!environment.docker.available) { + throw new Error( + "cloud-langchain-deepagents-code onboarding requires an available Docker runtime.", + ); + } + const sandboxName = sandboxNameFromOptions(environment.onboarding, options); + const apiKey = this.secrets.required("NVIDIA_INFERENCE_API_KEY"); + this.registerSandboxCleanup(sandboxName); + const result = await this.host.nemoclaw(ONBOARD_ARGS, { + artifactName: "onboard-cloud-langchain-deepagents-code", + env: commandEnv(sandboxName, { + NEMOCLAW_AGENT: "langchain-deepagents-code", + NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1", + NEMOCLAW_PROVIDER: HOSTED_INFERENCE_PROVIDER, + NEMOCLAW_ENDPOINT_URL: + process.env.NEMOCLAW_ENDPOINT_URL || DEFAULT_HOSTED_INFERENCE_BASE_URL, + NEMOCLAW_MODEL: + process.env.NEMOCLAW_MODEL || + process.env.NEMOCLAW_COMPAT_MODEL || + DEFAULT_HOSTED_INFERENCE_MODEL, + NEMOCLAW_COMPAT_MODEL: + process.env.NEMOCLAW_MODEL || + process.env.NEMOCLAW_COMPAT_MODEL || + DEFAULT_HOSTED_INFERENCE_MODEL, + NEMOCLAW_PREFERRED_API: process.env.NEMOCLAW_PREFERRED_API || "openai-completions", + NVIDIA_INFERENCE_API_KEY: apiKey, + [HOSTED_INFERENCE_CREDENTIAL_ENV]: apiKey, + }), + redactionValues: [apiKey], + timeoutMs: options.timeoutMs ?? DEFAULT_TIMEOUT_MS, + }); + assertExitZero(result, "cloud-langchain-deepagents-code onboarding"); + return { + onboarding: environment.onboarding, + sandboxName, + agent: "langchain-deepagents-code", + provider: "nvidia", + providerEnv: "cloud", + gatewayUrl: OPENCLAW_GATEWAY_URL, + result, + }; + } + async cloudOpenClawNoDocker( environment: EnvironmentReady, options: OnboardingOptions = {}, diff --git a/test/e2e-scenario/live/cloud-experimental-check-list.ts b/test/e2e-scenario/live/cloud-experimental-check-list.ts new file mode 100644 index 00000000000..6846364f802 --- /dev/null +++ b/test/e2e-scenario/live/cloud-experimental-check-list.ts @@ -0,0 +1,15 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export const DEEPAGENTS_CLOUD_EXPERIMENTAL_CHECKS = [ + "test/e2e/e2e-cloud-experimental/checks/05-deepagents-code-landlock-readonly.sh", + "test/e2e/e2e-cloud-experimental/checks/06-deepagents-code-python-egress.sh", +] as const; + +export function cloudExperimentalChecksForOnboarding( + onboarding: string | undefined, +): readonly string[] { + return onboarding === "cloud-langchain-deepagents-code" + ? DEEPAGENTS_CLOUD_EXPERIMENTAL_CHECKS + : []; +} diff --git a/test/e2e-scenario/live/cloud-experimental-checks.ts b/test/e2e-scenario/live/cloud-experimental-checks.ts new file mode 100644 index 00000000000..c5b1b20cf6a --- /dev/null +++ b/test/e2e-scenario/live/cloud-experimental-checks.ts @@ -0,0 +1,95 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import path from "node:path"; + +import { expect } from "vitest"; +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { resultText } from "../fixtures/clients/command.ts"; +import type { E2EScenarioFixtures } from "../fixtures/e2e-test.ts"; +import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; + +const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); +const REQUIRED_CHECK_SKIP_PATTERN = /(^|\n).*\bSKIP\b/i; + +export function buildCloudExperimentalCommandEnv( + sandboxName: string, + apiKey: string, + base: NodeJS.ProcessEnv = process.env, +): NodeJS.ProcessEnv { + return { + ...buildAvailabilityProbeEnv(base), + CLOUD_EXPERIMENTAL_MODEL: base.NEMOCLAW_MODEL, + COMPATIBLE_API_KEY: apiKey, + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", + NEMOCLAW_E2E_CLOUD_API_KEY_ENV: "COMPATIBLE_API_KEY", + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_SANDBOX_NAME: sandboxName, + OPENSHELL_GATEWAY: "nemoclaw", + REPO: REPO_ROOT, + SANDBOX_NAME: sandboxName, + }; +} + +export function assertRequiredCloudExperimentalResult( + scriptPath: string, + result: ShellProbeResult, +): void { + const output = resultText(result); + expect(result.exitCode, `${scriptPath}: ${output}`).toBe(0); + expect(output, `${scriptPath}: required cloud-experimental check must not skip`).not.toMatch( + REQUIRED_CHECK_SKIP_PATTERN, + ); +} + +async function assertDeepAgentsRuntimeObserved( + sandboxName: string, + context: Pick, +): Promise { + const result = await context.host.command( + "openshell", + [ + "sandbox", + "exec", + "--name", + sandboxName, + "--", + "bash", + "-c", + "test -d /sandbox/.deepagents && command -v dcode >/dev/null", + ], + { + artifactName: "cloud-experimental-deepagents-runtime", + env: buildCloudExperimentalCommandEnv(sandboxName, ""), + timeoutMs: 30_000, + }, + ); + expect(result.exitCode, `Deep Agents Code runtime marker missing: ${resultText(result)}`).toBe(0); +} + +export async function runE2eCloudExperimentalChecks( + scenarioId: string, + sandboxName: string, + checkScripts: readonly string[], + context: Pick, +): Promise { + const apiKey = context.secrets.optional("NVIDIA_INFERENCE_API_KEY") ?? ""; + await context.artifacts.writeJson("e2e-cloud-experimental-checks.json", { + scenarioId, + sandboxName, + checkScripts, + }); + await Promise.resolve( + checkScripts.length > 0 ? assertDeepAgentsRuntimeObserved(sandboxName, context) : undefined, + ); + for (const scriptPath of checkScripts) { + const result = await context.host.command("bash", [path.join(REPO_ROOT, scriptPath)], { + artifactName: `cloud-experimental-${path.basename(scriptPath, ".sh")}`, + cwd: REPO_ROOT, + env: buildCloudExperimentalCommandEnv(sandboxName, apiKey), + redactionValues: [apiKey], + timeoutMs: 180_000, + }); + assertRequiredCloudExperimentalResult(scriptPath, result); + } +} diff --git a/test/e2e-scenario/live/gpu-e2e.test.ts b/test/e2e-scenario/live/gpu-e2e.test.ts index 527fe09123c..b479cc9cc87 100644 --- a/test/e2e-scenario/live/gpu-e2e.test.ts +++ b/test/e2e-scenario/live/gpu-e2e.test.ts @@ -29,6 +29,68 @@ import { const TIMEOUT_MS = 75 * 60_000; +function asRecord(value: unknown): Record | undefined { + return value && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : undefined; +} + +function modelIdentifier(value: Record, key: string): string | undefined { + return typeof value[key] === "string" ? (value[key] as string) : undefined; +} + +function assertSmallContextCompactionPolicy(configText: string): void { + const config = asRecord(JSON.parse(configText)); + const agents = asRecord(config?.agents); + const defaults = asRecord(agents?.defaults); + const modelDefaults = asRecord(defaults?.model); + const primary = modelIdentifier(modelDefaults ?? {}, "primary"); + const compaction = asRecord(defaults?.compaction); + const modelsRoot = asRecord(config?.models); + const providers = asRecord(modelsRoot?.providers); + const primaryWithoutProvider = primary?.startsWith("inference/") + ? primary.slice("inference/".length) + : primary; + const model = Object.values(providers ?? {}) + .flatMap((provider) => { + const models = asRecord(provider)?.models; + return Array.isArray(models) ? models : []; + }) + .map(asRecord) + .find((candidate) => { + const identifiers = + candidate && primary && primaryWithoutProvider + ? ["id", "name", "label"].flatMap((key) => { + const value = modelIdentifier(candidate, key); + return value ? [value] : []; + }) + : []; + return identifiers.some( + (identifier) => + identifier === primary || + identifier === primaryWithoutProvider || + identifier === `inference/${primaryWithoutProvider}`, + ); + }); + + expect(primary, "OpenClaw config must declare the active model").toBeTruthy(); + expect(model, `OpenClaw config must include active Ollama model ${primary}`).toBeDefined(); + expect(typeof model?.contextWindow).toBe("number"); + expect(typeof model?.maxTokens).toBe("number"); + const contextWindow = model?.contextWindow as number; + const maxTokens = model?.maxTokens as number; + expect( + contextWindow, + `active Ollama model ${primary} must stay on the small-context lane`, + ).toBeLessThanOrEqual(28_000); + const expectedReserve = Math.min(maxTokens, Math.max(0, contextWindow - 8_000)); + + expect(compaction).toEqual({ + reserveTokens: expectedReserve, + reserveTokensFloor: expectedReserve, + }); +} + test.skipIf(!shouldRunLiveE2EScenarios())( "GPU Ollama onboard enables CUDA, auth proxy, and sandbox inference", { timeout: TIMEOUT_MS }, @@ -43,7 +105,7 @@ test.skipIf(!shouldRunLiveE2EScenarios())( sandboxName: SANDBOX_NAME, delegatedLegacyContracts: [ "Phase 11 shell retirement decides whether uninstall --delete-models remains a separate cleanup lane", - "The #5468 OpenClaw TUI compaction guard remains in the retained legacy shell until a TUI fixture exists", + "The #5468 interactive TUI first-turn smoke remains waived until a TUI fixture exists; this Vitest asserts the baked compaction budget directly", ], }); @@ -75,6 +137,15 @@ test.skipIf(!shouldRunLiveE2EScenarios())( expect(install.exitCode, resultText(install)).toBe(0); await artifacts.writeText("install-gpu-ollama.log", resultText(install)); + const config = await sandbox.execShell( + SANDBOX_NAME, + trustedSandboxShellScript("cat /sandbox/.openclaw/openclaw.json"), + { artifactName: "sandbox-openclaw-config", env: env(), timeoutMs: 30_000 }, + ); + expect(config.exitCode, resultText(config)).toBe(0); + await artifacts.writeText("openclaw-config.json", config.stdout); + assertSmallContextCompactionPolicy(config.stdout); + const status = await host.command("node", [CLI, SANDBOX_NAME, "status"], { artifactName: "status-gpu-ollama", env: env(), diff --git a/test/e2e-scenario/live/registry-scenarios.test.ts b/test/e2e-scenario/live/registry-scenarios.test.ts index 0e94f7e1575..f3da5ad1408 100644 --- a/test/e2e-scenario/live/registry-scenarios.test.ts +++ b/test/e2e-scenario/live/registry-scenarios.test.ts @@ -8,6 +8,8 @@ import { expect, test } from "../fixtures/e2e-test.ts"; import type { LifecycleProfile } from "../fixtures/phases/index.ts"; import { listScenarios } from "../scenarios/registry.ts"; import { liveScenarioSupport, liveScenarioTestName } from "../scenarios/runtime-support.ts"; +import { cloudExperimentalChecksForOnboarding } from "./cloud-experimental-check-list.ts"; +import { runE2eCloudExperimentalChecks } from "./cloud-experimental-checks.ts"; import { buildLiveScenarioRunPlan } from "./run-plan.ts"; const LIFECYCLE_PROFILES: ReadonlySet = new Set(["post-reboot-recovery"]); @@ -18,6 +20,10 @@ function isLifecycleProfile(value: string | undefined): value is LifecycleProfil const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); const CLI_DIST_ENTRYPOINT = path.join(REPO_ROOT, "dist", "nemoclaw.js"); +const E2E_CLOUD_EXPERIMENTAL_CHECKS_DIR = path.join( + REPO_ROOT, + "test/e2e/e2e-cloud-experimental/checks", +); process.env.NEMOCLAW_CLI_BIN ??= path.join(REPO_ROOT, "bin", "nemoclaw.js"); // The workflow filters by exact scenario id via `-t "^${SCENARIO_ID}$"`. @@ -38,7 +44,7 @@ for (const scenario of listScenarios()) { test( liveScenarioTestName(scenario), - async ({ artifacts, environment, lifecycle, onboard, secrets, stateValidation }) => { + async ({ artifacts, environment, host, lifecycle, onboard, secrets, stateValidation }) => { for (const secret of scenario.requiredSecrets ?? []) { secrets.required(secret); } @@ -61,7 +67,8 @@ for (const scenario of listScenarios()) { pendingRuntimeSuites: support.pendingRuntimeSuites, }); - await artifacts.writeJson("run-plan.json", buildLiveScenarioRunPlan(scenario)); + const runPlan = buildLiveScenarioRunPlan(scenario); + await artifacts.writeJson("run-plan.json", runPlan); const ready = await environment.assertReady(scenario.environment); const instance = await onboard.from(ready, { sandboxName: `e2e-${scenario.id}` }); @@ -92,6 +99,20 @@ for (const scenario of listScenarios()) { const validation = await stateValidation.from(scenario.expectedStateId, instance); + const checkScripts = runPlan.e2eCloudExperimentalChecks ?? []; + expect(checkScripts).toEqual( + cloudExperimentalChecksForOnboarding(scenario.environment.onboarding), + ); + for (const scriptPath of checkScripts) { + expect(fs.existsSync(path.join(REPO_ROOT, scriptPath))).toBe(true); + } + expect(fs.existsSync(E2E_CLOUD_EXPERIMENTAL_CHECKS_DIR)).toBe(true); + await runE2eCloudExperimentalChecks(scenario.id, instance.sandboxName, checkScripts, { + artifacts, + host, + secrets, + }); + await artifacts.writeJson("scenario-result.json", { id: scenario.id, expectedStateId: validation.state.id, diff --git a/test/e2e-scenario/live/run-plan.ts b/test/e2e-scenario/live/run-plan.ts index 3e0c659ff39..df0d9481a4b 100644 --- a/test/e2e-scenario/live/run-plan.ts +++ b/test/e2e-scenario/live/run-plan.ts @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { cloudExperimentalChecksForOnboarding } from "./cloud-experimental-check-list.ts"; import type { ScenarioDefinition } from "../scenarios/types.ts"; export interface LiveScenarioRunPlan { @@ -9,10 +10,11 @@ export interface LiveScenarioRunPlan { expectedStateId: string | undefined; suiteIds: string[]; phases: string[]; + e2eCloudExperimentalChecks?: string[]; } export function buildLiveScenarioRunPlan(scenario: ScenarioDefinition): LiveScenarioRunPlan { - return { + const plan: LiveScenarioRunPlan = { scenarioId: scenario.id, manifestPath: scenario.manifestPath ?? null, expectedStateId: scenario.expectedStateId, @@ -24,4 +26,11 @@ export function buildLiveScenarioRunPlan(scenario: ScenarioDefinition): LiveScen "state-validation", ], }; + const cloudExperimentalChecks = cloudExperimentalChecksForOnboarding( + scenario.environment?.onboarding, + ); + if (cloudExperimentalChecks.length > 0) { + plan.e2eCloudExperimentalChecks = [...cloudExperimentalChecks]; + } + return plan; } diff --git a/test/e2e-scenario/scenarios/expected-states.ts b/test/e2e-scenario/scenarios/expected-states.ts index 7da362975fd..6095890630f 100644 --- a/test/e2e-scenario/scenarios/expected-states.ts +++ b/test/e2e-scenario/scenarios/expected-states.ts @@ -31,10 +31,14 @@ const cloudHermesReady: ExpectedState = { credentials: { expected: "present" }, }; +// Deep Agents Code is a terminal-agent runtime, not an OpenClaw dashboard +// runtime. The P0-E parity target is sandbox policy/egress behavior, so the +// live typed scenario must not require a host dashboard forward on 18789 before +// running the in-sandbox cloud-experimental checks. const cloudDeepAgentsCodeReady: ExpectedState = { id: "cloud-deepagents-code-ready", cli: { installed: true }, - gateway: { expected: "present", health: "healthy" }, + gateway: { expected: "optional", health: "optional" }, sandbox: { expected: "present", status: "running", agent: "langchain-deepagents-code" }, inference: { expected: "available", provider: "nvidia" }, credentials: { expected: "present" }, diff --git a/test/e2e-scenario/scenarios/runtime-support.ts b/test/e2e-scenario/scenarios/runtime-support.ts index 6a9f874264d..ac16ce8f54f 100644 --- a/test/e2e-scenario/scenarios/runtime-support.ts +++ b/test/e2e-scenario/scenarios/runtime-support.ts @@ -6,7 +6,7 @@ import type { ScenarioDefinition } from "./types.ts"; const SUPPORTED_PLATFORMS = new Set(["ubuntu-local"]); const SUPPORTED_INSTALLS = new Set(["repo-current"]); const SUPPORTED_RUNTIMES = new Set(["docker-running"]); -const SUPPORTED_ONBOARDING = new Set(["cloud-openclaw"]); +const SUPPORTED_ONBOARDING = new Set(["cloud-openclaw", "cloud-langchain-deepagents-code"]); // Lifecycle profiles wired into the live Vitest driver. A profile is // supported only after both (a) `LifecyclePhaseFixture.simulate(profile)` // dispatches it, and (b) at least one expected-state declares the post- diff --git a/test/e2e-scenario/scenarios/scenarios/baseline.ts b/test/e2e-scenario/scenarios/scenarios/baseline.ts index 270768791a7..795f91184ad 100644 --- a/test/e2e-scenario/scenarios/scenarios/baseline.ts +++ b/test/e2e-scenario/scenarios/scenarios/baseline.ts @@ -83,7 +83,7 @@ const canonicalScenarioInputs: CanonicalScenarioInput[] = [ manifestName: "langchain-deepagents-code-nvidia", environment: ubuntuRepoDocker("cloud-langchain-deepagents-code"), expectedStateId: "cloud-deepagents-code-ready", - suiteIds: ["smoke", "inference", "terminal-agent"], + suiteIds: ["smoke", "inference", "terminal-agent", "deepagents-code-policy"], description: "Ubuntu repo checkout with Docker and LangChain Deep Agents Code onboarding.", requiredSecrets: ["NVIDIA_INFERENCE_API_KEY"], }, diff --git a/test/e2e-scenario/support-tests/e2e-expected-state.test.ts b/test/e2e-scenario/support-tests/e2e-expected-state.test.ts index 8d255b484c6..76b84d5ec41 100644 --- a/test/e2e-scenario/support-tests/e2e-expected-state.test.ts +++ b/test/e2e-scenario/support-tests/e2e-expected-state.test.ts @@ -39,6 +39,13 @@ describe("probesForState maps typed expected-state into probe ids", () => { ]); }); + it("Deep Agents Code ready state omits host dashboard health for terminal-agent parity", () => { + expect(probesForState(requireExpectedState("cloud-deepagents-code-ready"))).toEqual([ + "cli-installed", + "sandbox-running", + ]); + }); + it("preflight-failure state emits cli-installed, gateway-absent, sandbox-absent", () => { expect(probesForState(requireExpectedState("preflight-failure-no-sandbox"))).toEqual([ "cli-installed", diff --git a/test/e2e-scenario/support-tests/e2e-scenario-matrix.test.ts b/test/e2e-scenario/support-tests/e2e-scenario-matrix.test.ts index e9d3d1aaf7d..791d02f8f0d 100644 --- a/test/e2e-scenario/support-tests/e2e-scenario-matrix.test.ts +++ b/test/e2e-scenario/support-tests/e2e-scenario-matrix.test.ts @@ -77,10 +77,24 @@ describe("live Vitest scenario matrix", () => { it("builds the default live Vitest matrix from fixture-supported scenarios only", () => { expect(buildLiveScenarioMatrix().map((entry) => entry.id)).toEqual([ + "ubuntu-repo-cloud-langchain-deepagents-code", "ubuntu-repo-cloud-openclaw", "ubuntu-repo-docker-post-reboot-recovery", ]); expect(buildLiveScenarioMatrix()[0]).toMatchObject({ + id: "ubuntu-repo-cloud-langchain-deepagents-code", + runner: "ubuntu-latest", + platform: "ubuntu-local", + install: "repo-current", + runtime: "docker-running", + onboarding: "cloud-langchain-deepagents-code", + expectedStateId: "cloud-deepagents-code-ready", + requiredSecrets: ["NVIDIA_INFERENCE_API_KEY"], + supported: true, + supportReasons: [], + pendingRuntimeSuites: ["smoke", "inference", "terminal-agent", "deepagents-code-policy"], + }); + expect(buildLiveScenarioMatrix()[1]).toMatchObject({ id: "ubuntu-repo-cloud-openclaw", runner: "ubuntu-latest", platform: "ubuntu-local", @@ -97,7 +111,7 @@ describe("live Vitest scenario matrix", () => { // confirm the lifecycle whitelist + post-reboot-recovery scenario // are wired together; the actual RED/GREEN behavior is exercised // by the live runner (gates on the fix landing in src/lib/). - expect(buildLiveScenarioMatrix()[1]).toMatchObject({ + expect(buildLiveScenarioMatrix()[2]).toMatchObject({ id: "ubuntu-repo-docker-post-reboot-recovery", runner: "ubuntu-latest", platform: "ubuntu-local", @@ -128,6 +142,7 @@ describe("live Vitest scenario matrix", () => { expect(lines.length, "live matrix output must be a single line").toBe(1); const parsed = JSON.parse(lines[0]); expect(parsed.map((entry: { id: string }) => entry.id)).toEqual([ + "ubuntu-repo-cloud-langchain-deepagents-code", "ubuntu-repo-cloud-openclaw", "ubuntu-repo-docker-post-reboot-recovery", ]); diff --git a/test/e2e-scenario/support-tests/platform-parity-cloud-experimental.test.ts b/test/e2e-scenario/support-tests/platform-parity-cloud-experimental.test.ts new file mode 100644 index 00000000000..7a97979be86 --- /dev/null +++ b/test/e2e-scenario/support-tests/platform-parity-cloud-experimental.test.ts @@ -0,0 +1,86 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; +import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; +import { + assertRequiredCloudExperimentalResult, + buildCloudExperimentalCommandEnv, +} from "../live/cloud-experimental-checks.ts"; + +function shellResult(exitCode: number, stdout: string, stderr = ""): ShellProbeResult { + return { + command: [], + exitCode, + signal: null, + timedOut: false, + stdout, + stderr, + artifacts: { + stdout: "stdout.txt", + stderr: "stderr.txt", + result: "result.json", + }, + }; +} + +describe("P0-E cloud-experimental parity guardrails", () => { + it("fails required Deep Agents cloud-experimental checks when scripts print SKIP", () => { + expect(() => + assertRequiredCloudExperimentalResult( + "test/e2e/e2e-cloud-experimental/checks/05-deepagents-code-landlock-readonly.sh", + shellResult(0, "05-deepagents-code-landlock-readonly: SKIP: not a Deep Agents sandbox\n"), + ), + ).toThrow(/must not skip/); + }); + + it("fails Deep Agents Python egress blocked-host assertions without denial evidence", () => { + const result = spawnSync( + "bash", + [ + path.join( + process.cwd(), + "test/e2e/e2e-cloud-experimental/checks/06-deepagents-code-python-egress.sh", + ), + ], + { + encoding: "utf8", + env: { + NEMOCLAW_E2E_PYTHON_EGRESS_SELF_TEST: "blocked-no-marker", + NEMOCLAW_E2E_PYTHON_PROBE_FIXTURE: "OpenShell runtime error without denial marker", + PATH: process.env.PATH ?? "/usr/bin:/bin", + }, + }, + ); + + expect(result.status).not.toBe(0); + expect(`${result.stdout}\n${result.stderr}`).toContain( + "self-test Python probe for fixture host lacked denial evidence", + ); + }); + + it("builds a minimal cloud-experimental child environment", () => { + const env = buildCloudExperimentalCommandEnv("deepagents-sandbox", "secret-key", { + HOME: "/home/runner", + PATH: "/usr/bin", + AWS_SECRET_ACCESS_KEY: "do-not-copy", + GITHUB_TOKEN: "do-not-copy", + NEMOCLAW_MODEL: "model-a", + RANDOM_RUNNER_SECRET: "do-not-copy", + }); + + expect(env).toMatchObject({ + COMPATIBLE_API_KEY: "secret-key", + CLOUD_EXPERIMENTAL_MODEL: "model-a", + NEMOCLAW_SANDBOX_NAME: "deepagents-sandbox", + SANDBOX_NAME: "deepagents-sandbox", + OPENSHELL_GATEWAY: "nemoclaw", + }); + expect(env.AWS_SECRET_ACCESS_KEY).toBeUndefined(); + expect(env.GITHUB_TOKEN).toBeUndefined(); + expect(env.RANDOM_RUNNER_SECRET).toBeUndefined(); + }); +}); diff --git a/test/e2e/e2e-cloud-experimental/checks/06-deepagents-code-python-egress.sh b/test/e2e/e2e-cloud-experimental/checks/06-deepagents-code-python-egress.sh index 970e1090d10..0fa79ff9f75 100755 --- a/test/e2e/e2e-cloud-experimental/checks/06-deepagents-code-python-egress.sh +++ b/test/e2e/e2e-cloud-experimental/checks/06-deepagents-code-python-egress.sh @@ -37,18 +37,63 @@ sandbox_exec() { python_probe() { local python_bin="$1" local url="$2" + if [ -n "${NEMOCLAW_E2E_PYTHON_PROBE_FIXTURE+x}" ]; then + printf '%s\n' "$NEMOCLAW_E2E_PYTHON_PROBE_FIXTURE" + return 0 + fi sandbox_exec "${python_bin@Q} - ${url@Q} <<'PY' import sys import urllib.error import urllib.request + +DENIAL_MARKERS = ( + 'access denied', + 'blocked by', + 'connection forbidden', + 'egress denied', + 'network is unreachable', + 'network policy', + 'operation not permitted', + 'permission denied', + 'policy denied', + 'tunnel connection failed', +) + + +def is_policy_denial(text): + lowered = text.lower() + return any(marker in lowered for marker in DENIAL_MARKERS) + + url = sys.argv[1] try: with urllib.request.urlopen(url, timeout=8) as response: print(f'REACHED:{response.status}') except urllib.error.HTTPError as exc: - print(f'REACHED:{exc.code}') + body = '' + try: + body = exc.read(512).decode('utf-8', 'replace') + except Exception: + body = '' + details = f'{exc} {body}'.strip() + if is_policy_denial(details): + print(f'BLOCKED:HTTPError:{details}') + else: + print(f'REACHED:{exc.code}') +except urllib.error.URLError as exc: + details = str(exc.reason if getattr(exc, 'reason', None) is not None else exc) + if is_policy_denial(details): + print(f'BLOCKED:URLError:{details}') + else: + print(f'ERROR:URLError:{details}') +except OSError as exc: + details = str(exc) + if is_policy_denial(details): + print(f'BLOCKED:{type(exc).__name__}:{details}') + else: + print(f'ERROR:{type(exc).__name__}:{details}') except Exception as exc: - print(f'BLOCKED:{type(exc).__name__}:{exc}') + print(f'ERROR:{type(exc).__name__}:{exc}') PY " } @@ -76,18 +121,27 @@ expect_blocked() { output="$(python_probe "$python_bin" "$url")" if echo "$output" | grep -q "BLOCKED:" && ! echo "$output" | grep -q "REACHED:"; then pass "${actor} cannot reach ${label} without explicit policy" - else + elif echo "$output" | grep -q "REACHED:"; then fail_test "${actor} reached ${label} unexpectedly: $output" + else + fail_test "${actor} probe for ${label} lacked denial evidence: $output" fi } +PASSED=0 +FAILED=0 + +if [ "${NEMOCLAW_E2E_PYTHON_EGRESS_SELF_TEST:-}" = "blocked-no-marker" ]; then + expect_blocked "self-test Python" "fixture host" "https://blocked.example/" + printf '%s\n' "${PREFIX}: $PASSED passed, $FAILED failed" + [ "$FAILED" -eq 0 ] || exit 1 + exit 0 +fi + cleanup_project_venv() { sandbox_exec "rm -rf ${PROJECT_VENV@Q}" >/dev/null || true } -PASSED=0 -FAILED=0 - if ! sandbox_exec "test -d /sandbox/.deepagents && command -v dcode >/dev/null 2>&1" >/dev/null; then info "SKIP: sandbox '${SANDBOX_NAME}' is not a Deep Agents Code sandbox" exit 0 diff --git a/test/langchain-deepagents-code-image.test.ts b/test/langchain-deepagents-code-image.test.ts index 0ff2add83f5..d4348697dc6 100644 --- a/test/langchain-deepagents-code-image.test.ts +++ b/test/langchain-deepagents-code-image.test.ts @@ -5,8 +5,10 @@ import { execFileSync } from "node:child_process"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; - import { describe, expect, it } from "vitest"; +import YAML from "yaml"; + +import { cloudExperimentalChecksForOnboarding } from "./e2e-scenario/live/cloud-experimental-check-list.ts"; const agentDir = path.join(process.cwd(), "agents", "langchain-deepagents-code"); const DCODE_CANONICAL_PATH = @@ -16,6 +18,17 @@ function readAgentFile(name: string): string { return fs.readFileSync(path.join(agentDir, name), "utf8"); } +function policyBinaryPaths(policyText: string, policyName: string): string[] { + const parsed = YAML.parse(policyText) as { + network_policies?: Record }>; + }; + const binaries = parsed.network_policies?.[policyName]?.binaries; + expect(Array.isArray(binaries), `${policyName} policy must declare binary-scoped egress`).toBe( + true, + ); + return (binaries ?? []).map((entry) => (typeof entry.path === "string" ? entry.path : "")); +} + function makeStartScriptFixture(tempDir: string): { envFile: string; messagingEnvFile: string; @@ -247,12 +260,29 @@ describe("LangChain Deep Agents Code image contracts", () => { expect(policy).toContain( "Tavily, LangSmith, MCP, and arbitrary hosts are intentionally absent", ); - expect(policy).toContain("- { path: /opt/venv/bin/python3* }"); - expect(policy).toContain("- { path: /opt/venv/bin/pip3 }"); - expect(policy).toContain("- { path: /sandbox/**/bin/python3* }"); - expect(policy).toContain("- { path: /sandbox/**/bin/pip3 }"); - expect(policy).not.toContain("- { path: /usr/bin/python3* }"); - expect(policy).not.toContain("- { path: /usr/local/bin/pip3 }"); + + const githubBinaries = policyBinaryPaths(policy, "github"); + expect(githubBinaries).toEqual( + expect.arrayContaining(["/usr/bin/git", "/usr/local/bin/dcode", "/opt/venv/bin/python3*"]), + ); + expect(githubBinaries).not.toEqual(expect.arrayContaining(["/usr/bin/python3*"])); + expect(githubBinaries).not.toEqual(expect.arrayContaining(["/usr/local/bin/python3*"])); + expect(githubBinaries).not.toEqual(expect.arrayContaining(["/usr/local/lib/python3.13/**"])); + + const pypiBinaries = policyBinaryPaths(policy, "pypi"); + expect(pypiBinaries).toEqual( + expect.arrayContaining([ + "/opt/venv/bin/pip3", + "/sandbox/**/bin/pip3", + "/opt/venv/bin/python3*", + "/sandbox/**/bin/python3*", + "/usr/local/bin/dcode", + ]), + ); + expect(pypiBinaries).not.toEqual(expect.arrayContaining(["/usr/bin/python3*"])); + expect(pypiBinaries).not.toEqual(expect.arrayContaining(["/usr/local/bin/python3*"])); + expect(pypiBinaries).not.toEqual(expect.arrayContaining(["/usr/local/bin/pip3"])); + expect(pypiBinaries).not.toEqual(expect.arrayContaining(["/usr/local/lib/python3.13/**"])); }); it("ships live policy behavior checks for Deep Agents Code", () => { @@ -296,6 +326,9 @@ describe("LangChain Deep Agents Code image contracts", () => { expect(pythonEgressCheck).toContain("^USRLOCAL_COUNT=1$"); expect(pythonEgressCheck).toContain("import urllib.error"); expect(pythonEgressCheck).toContain("except urllib.error.HTTPError as exc:"); + expect(pythonEgressCheck).toContain("except urllib.error.URLError as exc:"); + expect(pythonEgressCheck).toContain("ERROR:URLError"); + expect(pythonEgressCheck).toContain("lacked denial evidence"); expect(pythonEgressCheck).toContain("${python_bin@Q} - ${url@Q} <<'PY'"); expect(pythonEgressCheck).toContain( 'expect_reached "arbitrary Python" "GitHub" "https://api.github.com/"', @@ -319,6 +352,10 @@ describe("LangChain Deep Agents Code image contracts", () => { expect(pythonEgressCheck).toContain("https://modelcontextprotocol.io/"); expect(pythonEgressCheck).toContain("https://example.com/"); expect(pythonEgressCheck).toContain("${actor} cannot reach ${label} without explicit policy"); + expect(cloudExperimentalChecksForOnboarding("cloud-langchain-deepagents-code")).toEqual([ + "test/e2e/e2e-cloud-experimental/checks/05-deepagents-code-landlock-readonly.sh", + "test/e2e/e2e-cloud-experimental/checks/06-deepagents-code-python-egress.sh", + ]); }); it("hash-locks Deep Agents Code base image PyPI installs", () => {