From 0d9560ef0b1de538d310ba3486463f1f4d638e44 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Thu, 25 Jun 2026 18:02:56 -0400 Subject: [PATCH 1/8] test(e2e): close platform parity decisions --- .../fixtures/phases/onboarding.ts | 38 ++++++++++- test/e2e-scenario/live/gpu-e2e.test.ts | 46 ++++++++++++- .../live/registry-scenarios.test.ts | 67 ++++++++++++++++++- test/e2e-scenario/live/run-plan.ts | 10 ++- .../e2e-scenario/scenarios/runtime-support.ts | 2 +- .../scenarios/scenarios/baseline.ts | 2 +- .../support-tests/e2e-scenario-matrix.test.ts | 17 ++++- 7 files changed, 174 insertions(+), 8 deletions(-) diff --git a/test/e2e-scenario/fixtures/phases/onboarding.ts b/test/e2e-scenario/fixtures/phases/onboarding.ts index 3efa991d7ca..e057bd7c917 100644 --- a/test/e2e-scenario/fixtures/phases/onboarding.ts +++ b/test/e2e-scenario/fixtures/phases/onboarding.ts @@ -70,7 +70,7 @@ export interface OnboardingExpectedFailure { export interface NemoClawInstance { onboarding: string; sandboxName: string; - agent: "openclaw" | "hermes"; + agent: "openclaw" | "hermes" | "langchain-deepagents-code"; provider: "nvidia" | "ollama"; providerEnv: "cloud" | "local"; platformOs?: "ubuntu" | "macos" | "windows"; @@ -180,6 +180,9 @@ export class OnboardingPhaseFixture { case "cloud-openclaw-no-docker": result = await this.cloudOpenClawNoDocker(environment, options); break; + case "cloud-langchain-deepagents-code": + result = await this.cloudLangchainDeepAgentsCode(environment, options); + break; default: throw new Error(`Unsupported onboarding profile '${environment.onboarding}'.`); } @@ -219,6 +222,39 @@ export class OnboardingPhaseFixture { }; } + async cloudLangchainDeepAgentsCode( + environment: EnvironmentReady, + options: OnboardingOptions = {}, + ): Promise { + if (!environment.docker.available) { + throw new Error( + "cloud-langchain-deepagents-code onboarding requires an available Docker runtime.", + ); + } + const sandboxName = sandboxNameFromOptions(environment.onboarding, options); + const apiKey = this.secrets.required("NVIDIA_INFERENCE_API_KEY"); + this.registerSandboxCleanup(sandboxName); + const result = await this.host.nemoclaw(ONBOARD_ARGS, { + artifactName: "onboard-cloud-langchain-deepagents-code", + env: commandEnv(sandboxName, { + NEMOCLAW_AGENT: "langchain-deepagents-code", + NVIDIA_INFERENCE_API_KEY: apiKey, + }), + redactionValues: [apiKey], + timeoutMs: options.timeoutMs ?? DEFAULT_TIMEOUT_MS, + }); + assertExitZero(result, "cloud-langchain-deepagents-code onboarding"); + return { + onboarding: environment.onboarding, + sandboxName, + agent: "langchain-deepagents-code", + provider: "nvidia", + providerEnv: "cloud", + gatewayUrl: OPENCLAW_GATEWAY_URL, + result, + }; + } + async cloudOpenClawNoDocker( environment: EnvironmentReady, options: OnboardingOptions = {}, diff --git a/test/e2e-scenario/live/gpu-e2e.test.ts b/test/e2e-scenario/live/gpu-e2e.test.ts index 527fe09123c..dfbce45af36 100644 --- a/test/e2e-scenario/live/gpu-e2e.test.ts +++ b/test/e2e-scenario/live/gpu-e2e.test.ts @@ -29,6 +29,41 @@ import { const TIMEOUT_MS = 75 * 60_000; +function asRecord(value: unknown): Record | undefined { + return value && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : undefined; +} + +function assertSmallContextCompactionPolicy(configText: string): void { + const config = asRecord(JSON.parse(configText)); + const agents = asRecord(config?.agents); + const defaults = asRecord(agents?.defaults); + const compaction = asRecord(defaults?.compaction); + const modelsRoot = asRecord(config?.models); + const providers = asRecord(modelsRoot?.providers); + const model = Object.values(providers ?? {}) + .flatMap((provider) => { + const models = asRecord(provider)?.models; + return Array.isArray(models) ? models : []; + }) + .map(asRecord) + .find(Boolean); + + expect(typeof model?.contextWindow).toBe("number"); + expect(typeof model?.maxTokens).toBe("number"); + const contextWindow = model?.contextWindow; + const maxTokens = model?.maxTokens; + if (typeof contextWindow !== "number" || typeof maxTokens !== "number") return; + if (contextWindow > 28_000) return; + + const expectedReserve = Math.min(maxTokens, Math.max(0, contextWindow - 8_000)); + expect(compaction).toMatchObject({ + reserveTokens: expectedReserve, + reserveTokensFloor: expectedReserve, + }); +} + test.skipIf(!shouldRunLiveE2EScenarios())( "GPU Ollama onboard enables CUDA, auth proxy, and sandbox inference", { timeout: TIMEOUT_MS }, @@ -43,7 +78,7 @@ test.skipIf(!shouldRunLiveE2EScenarios())( sandboxName: SANDBOX_NAME, delegatedLegacyContracts: [ "Phase 11 shell retirement decides whether uninstall --delete-models remains a separate cleanup lane", - "The #5468 OpenClaw TUI compaction guard remains in the retained legacy shell until a TUI fixture exists", + "The #5468 interactive TUI first-turn smoke remains waived until a TUI fixture exists; this Vitest asserts the baked compaction budget directly", ], }); @@ -75,6 +110,15 @@ test.skipIf(!shouldRunLiveE2EScenarios())( expect(install.exitCode, resultText(install)).toBe(0); await artifacts.writeText("install-gpu-ollama.log", resultText(install)); + const config = await sandbox.execShell( + SANDBOX_NAME, + trustedSandboxShellScript("cat /sandbox/.openclaw/openclaw.json"), + { artifactName: "sandbox-openclaw-config", env: env(), timeoutMs: 30_000 }, + ); + expect(config.exitCode, resultText(config)).toBe(0); + await artifacts.writeText("openclaw-config.json", config.stdout); + assertSmallContextCompactionPolicy(config.stdout); + const status = await host.command("node", [CLI, SANDBOX_NAME, "status"], { artifactName: "status-gpu-ollama", env: env(), diff --git a/test/e2e-scenario/live/registry-scenarios.test.ts b/test/e2e-scenario/live/registry-scenarios.test.ts index 0e94f7e1575..f423b9b8070 100644 --- a/test/e2e-scenario/live/registry-scenarios.test.ts +++ b/test/e2e-scenario/live/registry-scenarios.test.ts @@ -2,9 +2,10 @@ // SPDX-License-Identifier: Apache-2.0 import fs from "node:fs"; +import os from "node:os"; import path from "node:path"; -import { expect, test } from "../fixtures/e2e-test.ts"; +import { expect, test, type E2EScenarioFixtures } from "../fixtures/e2e-test.ts"; import type { LifecycleProfile } from "../fixtures/phases/index.ts"; import { listScenarios } from "../scenarios/registry.ts"; import { liveScenarioSupport, liveScenarioTestName } from "../scenarios/runtime-support.ts"; @@ -16,8 +17,54 @@ function isLifecycleProfile(value: string | undefined): value is LifecycleProfil return value !== undefined && LIFECYCLE_PROFILES.has(value as LifecycleProfile); } +function commandEnv(sandboxName: string, extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { + return { + ...process.env, + PATH: `${os.homedir()}/.local/bin:${os.homedir()}/.npm-global/bin:${process.env.PATH ?? ""}`, + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_SANDBOX_NAME: sandboxName, + OPENSHELL_GATEWAY: "nemoclaw", + ...extra, + }; +} + +async function runE2eCloudExperimentalChecks( + scenarioId: string, + sandboxName: string, + checkScripts: readonly string[], + context: Pick, +): Promise { + const apiKey = context.secrets.required("NVIDIA_INFERENCE_API_KEY"); + await context.artifacts.writeJson("e2e-cloud-experimental-checks.json", { + scenarioId, + sandboxName, + checkScripts, + }); + for (const scriptPath of checkScripts) { + const result = await context.host.command("bash", [path.join(REPO_ROOT, scriptPath)], { + artifactName: `cloud-experimental-${path.basename(scriptPath, ".sh")}`, + cwd: REPO_ROOT, + env: commandEnv(sandboxName, { + CLOUD_EXPERIMENTAL_MODEL: process.env.NEMOCLAW_MODEL, + COMPATIBLE_API_KEY: apiKey, + NEMOCLAW_E2E_CLOUD_API_KEY_ENV: "COMPATIBLE_API_KEY", + REPO: REPO_ROOT, + SANDBOX_NAME: sandboxName, + }), + redactionValues: [apiKey], + timeoutMs: 180_000, + }); + expect(result.exitCode, `${scriptPath}: ${result.stdout}\n${result.stderr}`).toBe(0); + } +} + const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); const CLI_DIST_ENTRYPOINT = path.join(REPO_ROOT, "dist", "nemoclaw.js"); +const E2E_CLOUD_EXPERIMENTAL_CHECKS_DIR = path.join( + REPO_ROOT, + "test/e2e/e2e-cloud-experimental/checks", +); process.env.NEMOCLAW_CLI_BIN ??= path.join(REPO_ROOT, "bin", "nemoclaw.js"); // The workflow filters by exact scenario id via `-t "^${SCENARIO_ID}$"`. @@ -38,7 +85,7 @@ for (const scenario of listScenarios()) { test( liveScenarioTestName(scenario), - async ({ artifacts, environment, lifecycle, onboard, secrets, stateValidation }) => { + async ({ artifacts, environment, host, lifecycle, onboard, secrets, stateValidation }) => { for (const secret of scenario.requiredSecrets ?? []) { secrets.required(secret); } @@ -92,6 +139,22 @@ for (const scenario of listScenarios()) { const validation = await stateValidation.from(scenario.expectedStateId, instance); + if (scenario.environment.onboarding === "cloud-langchain-deepagents-code") { + const checkScripts = [ + "test/e2e/e2e-cloud-experimental/checks/05-deepagents-code-landlock-readonly.sh", + "test/e2e/e2e-cloud-experimental/checks/06-deepagents-code-python-egress.sh", + ]; + for (const scriptPath of checkScripts) { + expect(fs.existsSync(path.join(REPO_ROOT, scriptPath))).toBe(true); + } + expect(fs.existsSync(E2E_CLOUD_EXPERIMENTAL_CHECKS_DIR)).toBe(true); + await runE2eCloudExperimentalChecks(scenario.id, instance.sandboxName, checkScripts, { + artifacts, + host, + secrets, + }); + } + await artifacts.writeJson("scenario-result.json", { id: scenario.id, expectedStateId: validation.state.id, diff --git a/test/e2e-scenario/live/run-plan.ts b/test/e2e-scenario/live/run-plan.ts index 3e0c659ff39..0ce195b93fd 100644 --- a/test/e2e-scenario/live/run-plan.ts +++ b/test/e2e-scenario/live/run-plan.ts @@ -9,10 +9,11 @@ export interface LiveScenarioRunPlan { expectedStateId: string | undefined; suiteIds: string[]; phases: string[]; + e2eCloudExperimentalChecks?: string[]; } export function buildLiveScenarioRunPlan(scenario: ScenarioDefinition): LiveScenarioRunPlan { - return { + const plan: LiveScenarioRunPlan = { scenarioId: scenario.id, manifestPath: scenario.manifestPath ?? null, expectedStateId: scenario.expectedStateId, @@ -24,4 +25,11 @@ export function buildLiveScenarioRunPlan(scenario: ScenarioDefinition): LiveScen "state-validation", ], }; + if (scenario.environment?.onboarding === "cloud-langchain-deepagents-code") { + plan.e2eCloudExperimentalChecks = [ + "test/e2e/e2e-cloud-experimental/checks/05-deepagents-code-landlock-readonly.sh", + "test/e2e/e2e-cloud-experimental/checks/06-deepagents-code-python-egress.sh", + ]; + } + return plan; } diff --git a/test/e2e-scenario/scenarios/runtime-support.ts b/test/e2e-scenario/scenarios/runtime-support.ts index 6a9f874264d..ac16ce8f54f 100644 --- a/test/e2e-scenario/scenarios/runtime-support.ts +++ b/test/e2e-scenario/scenarios/runtime-support.ts @@ -6,7 +6,7 @@ import type { ScenarioDefinition } from "./types.ts"; const SUPPORTED_PLATFORMS = new Set(["ubuntu-local"]); const SUPPORTED_INSTALLS = new Set(["repo-current"]); const SUPPORTED_RUNTIMES = new Set(["docker-running"]); -const SUPPORTED_ONBOARDING = new Set(["cloud-openclaw"]); +const SUPPORTED_ONBOARDING = new Set(["cloud-openclaw", "cloud-langchain-deepagents-code"]); // Lifecycle profiles wired into the live Vitest driver. A profile is // supported only after both (a) `LifecyclePhaseFixture.simulate(profile)` // dispatches it, and (b) at least one expected-state declares the post- diff --git a/test/e2e-scenario/scenarios/scenarios/baseline.ts b/test/e2e-scenario/scenarios/scenarios/baseline.ts index 270768791a7..795f91184ad 100644 --- a/test/e2e-scenario/scenarios/scenarios/baseline.ts +++ b/test/e2e-scenario/scenarios/scenarios/baseline.ts @@ -83,7 +83,7 @@ const canonicalScenarioInputs: CanonicalScenarioInput[] = [ manifestName: "langchain-deepagents-code-nvidia", environment: ubuntuRepoDocker("cloud-langchain-deepagents-code"), expectedStateId: "cloud-deepagents-code-ready", - suiteIds: ["smoke", "inference", "terminal-agent"], + suiteIds: ["smoke", "inference", "terminal-agent", "deepagents-code-policy"], description: "Ubuntu repo checkout with Docker and LangChain Deep Agents Code onboarding.", requiredSecrets: ["NVIDIA_INFERENCE_API_KEY"], }, diff --git a/test/e2e-scenario/support-tests/e2e-scenario-matrix.test.ts b/test/e2e-scenario/support-tests/e2e-scenario-matrix.test.ts index e9d3d1aaf7d..791d02f8f0d 100644 --- a/test/e2e-scenario/support-tests/e2e-scenario-matrix.test.ts +++ b/test/e2e-scenario/support-tests/e2e-scenario-matrix.test.ts @@ -77,10 +77,24 @@ describe("live Vitest scenario matrix", () => { it("builds the default live Vitest matrix from fixture-supported scenarios only", () => { expect(buildLiveScenarioMatrix().map((entry) => entry.id)).toEqual([ + "ubuntu-repo-cloud-langchain-deepagents-code", "ubuntu-repo-cloud-openclaw", "ubuntu-repo-docker-post-reboot-recovery", ]); expect(buildLiveScenarioMatrix()[0]).toMatchObject({ + id: "ubuntu-repo-cloud-langchain-deepagents-code", + runner: "ubuntu-latest", + platform: "ubuntu-local", + install: "repo-current", + runtime: "docker-running", + onboarding: "cloud-langchain-deepagents-code", + expectedStateId: "cloud-deepagents-code-ready", + requiredSecrets: ["NVIDIA_INFERENCE_API_KEY"], + supported: true, + supportReasons: [], + pendingRuntimeSuites: ["smoke", "inference", "terminal-agent", "deepagents-code-policy"], + }); + expect(buildLiveScenarioMatrix()[1]).toMatchObject({ id: "ubuntu-repo-cloud-openclaw", runner: "ubuntu-latest", platform: "ubuntu-local", @@ -97,7 +111,7 @@ describe("live Vitest scenario matrix", () => { // confirm the lifecycle whitelist + post-reboot-recovery scenario // are wired together; the actual RED/GREEN behavior is exercised // by the live runner (gates on the fix landing in src/lib/). - expect(buildLiveScenarioMatrix()[1]).toMatchObject({ + expect(buildLiveScenarioMatrix()[2]).toMatchObject({ id: "ubuntu-repo-docker-post-reboot-recovery", runner: "ubuntu-latest", platform: "ubuntu-local", @@ -128,6 +142,7 @@ describe("live Vitest scenario matrix", () => { expect(lines.length, "live matrix output must be a single line").toBe(1); const parsed = JSON.parse(lines[0]); expect(parsed.map((entry: { id: string }) => entry.id)).toEqual([ + "ubuntu-repo-cloud-langchain-deepagents-code", "ubuntu-repo-cloud-openclaw", "ubuntu-repo-docker-post-reboot-recovery", ]); From c25c0727f5945b00319f51d6ded1416516593950 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Thu, 25 Jun 2026 18:09:48 -0400 Subject: [PATCH 2/8] test(e2e): fix platform parity guardrails --- .../fixtures/phases/onboarding.ts | 14 +++++++++ test/e2e-scenario/live/gpu-e2e.test.ts | 17 +++++----- .../live/registry-scenarios.test.ts | 31 ++++++++++--------- 3 files changed, 38 insertions(+), 24 deletions(-) diff --git a/test/e2e-scenario/fixtures/phases/onboarding.ts b/test/e2e-scenario/fixtures/phases/onboarding.ts index e057bd7c917..4d8d583144b 100644 --- a/test/e2e-scenario/fixtures/phases/onboarding.ts +++ b/test/e2e-scenario/fixtures/phases/onboarding.ts @@ -238,7 +238,21 @@ export class OnboardingPhaseFixture { artifactName: "onboard-cloud-langchain-deepagents-code", env: commandEnv(sandboxName, { NEMOCLAW_AGENT: "langchain-deepagents-code", + NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1", + NEMOCLAW_PROVIDER: HOSTED_INFERENCE_PROVIDER, + NEMOCLAW_ENDPOINT_URL: + process.env.NEMOCLAW_ENDPOINT_URL || DEFAULT_HOSTED_INFERENCE_BASE_URL, + NEMOCLAW_MODEL: + process.env.NEMOCLAW_MODEL || + process.env.NEMOCLAW_COMPAT_MODEL || + DEFAULT_HOSTED_INFERENCE_MODEL, + NEMOCLAW_COMPAT_MODEL: + process.env.NEMOCLAW_MODEL || + process.env.NEMOCLAW_COMPAT_MODEL || + DEFAULT_HOSTED_INFERENCE_MODEL, + NEMOCLAW_PREFERRED_API: process.env.NEMOCLAW_PREFERRED_API || "openai-completions", NVIDIA_INFERENCE_API_KEY: apiKey, + [HOSTED_INFERENCE_CREDENTIAL_ENV]: apiKey, }), redactionValues: [apiKey], timeoutMs: options.timeoutMs ?? DEFAULT_TIMEOUT_MS, diff --git a/test/e2e-scenario/live/gpu-e2e.test.ts b/test/e2e-scenario/live/gpu-e2e.test.ts index dfbce45af36..b7d536fcb68 100644 --- a/test/e2e-scenario/live/gpu-e2e.test.ts +++ b/test/e2e-scenario/live/gpu-e2e.test.ts @@ -52,16 +52,15 @@ function assertSmallContextCompactionPolicy(configText: string): void { expect(typeof model?.contextWindow).toBe("number"); expect(typeof model?.maxTokens).toBe("number"); - const contextWindow = model?.contextWindow; - const maxTokens = model?.maxTokens; - if (typeof contextWindow !== "number" || typeof maxTokens !== "number") return; - if (contextWindow > 28_000) return; - + const contextWindow = model?.contextWindow as number; + const maxTokens = model?.maxTokens as number; const expectedReserve = Math.min(maxTokens, Math.max(0, contextWindow - 8_000)); - expect(compaction).toMatchObject({ - reserveTokens: expectedReserve, - reserveTokensFloor: expectedReserve, - }); + const expectedCompaction = + contextWindow <= 28_000 + ? { reserveTokens: expectedReserve, reserveTokensFloor: expectedReserve } + : undefined; + + expect(compaction).toEqual(expectedCompaction); } test.skipIf(!shouldRunLiveE2EScenarios())( diff --git a/test/e2e-scenario/live/registry-scenarios.test.ts b/test/e2e-scenario/live/registry-scenarios.test.ts index f423b9b8070..1e1df4edf9f 100644 --- a/test/e2e-scenario/live/registry-scenarios.test.ts +++ b/test/e2e-scenario/live/registry-scenarios.test.ts @@ -35,7 +35,7 @@ async function runE2eCloudExperimentalChecks( checkScripts: readonly string[], context: Pick, ): Promise { - const apiKey = context.secrets.required("NVIDIA_INFERENCE_API_KEY"); + const apiKey = context.secrets.optional("NVIDIA_INFERENCE_API_KEY") ?? ""; await context.artifacts.writeJson("e2e-cloud-experimental-checks.json", { scenarioId, sandboxName, @@ -139,21 +139,22 @@ for (const scenario of listScenarios()) { const validation = await stateValidation.from(scenario.expectedStateId, instance); - if (scenario.environment.onboarding === "cloud-langchain-deepagents-code") { - const checkScripts = [ - "test/e2e/e2e-cloud-experimental/checks/05-deepagents-code-landlock-readonly.sh", - "test/e2e/e2e-cloud-experimental/checks/06-deepagents-code-python-egress.sh", - ]; - for (const scriptPath of checkScripts) { - expect(fs.existsSync(path.join(REPO_ROOT, scriptPath))).toBe(true); - } - expect(fs.existsSync(E2E_CLOUD_EXPERIMENTAL_CHECKS_DIR)).toBe(true); - await runE2eCloudExperimentalChecks(scenario.id, instance.sandboxName, checkScripts, { - artifacts, - host, - secrets, - }); + const checkScripts = + scenario.environment.onboarding === "cloud-langchain-deepagents-code" + ? [ + "test/e2e/e2e-cloud-experimental/checks/05-deepagents-code-landlock-readonly.sh", + "test/e2e/e2e-cloud-experimental/checks/06-deepagents-code-python-egress.sh", + ] + : []; + for (const scriptPath of checkScripts) { + expect(fs.existsSync(path.join(REPO_ROOT, scriptPath))).toBe(true); } + expect(fs.existsSync(E2E_CLOUD_EXPERIMENTAL_CHECKS_DIR)).toBe(true); + await runE2eCloudExperimentalChecks(scenario.id, instance.sandboxName, checkScripts, { + artifacts, + host, + secrets, + }); await artifacts.writeJson("scenario-result.json", { id: scenario.id, From 1f292227fc2346b7ca4f6d23d0c44007bde6ba60 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Thu, 25 Jun 2026 18:18:07 -0400 Subject: [PATCH 3/8] test(e2e): align deepagents state probes --- test/e2e-scenario/scenarios/expected-states.ts | 6 +++++- test/e2e-scenario/support-tests/e2e-expected-state.test.ts | 7 +++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/test/e2e-scenario/scenarios/expected-states.ts b/test/e2e-scenario/scenarios/expected-states.ts index 7da362975fd..6095890630f 100644 --- a/test/e2e-scenario/scenarios/expected-states.ts +++ b/test/e2e-scenario/scenarios/expected-states.ts @@ -31,10 +31,14 @@ const cloudHermesReady: ExpectedState = { credentials: { expected: "present" }, }; +// Deep Agents Code is a terminal-agent runtime, not an OpenClaw dashboard +// runtime. The P0-E parity target is sandbox policy/egress behavior, so the +// live typed scenario must not require a host dashboard forward on 18789 before +// running the in-sandbox cloud-experimental checks. const cloudDeepAgentsCodeReady: ExpectedState = { id: "cloud-deepagents-code-ready", cli: { installed: true }, - gateway: { expected: "present", health: "healthy" }, + gateway: { expected: "optional", health: "optional" }, sandbox: { expected: "present", status: "running", agent: "langchain-deepagents-code" }, inference: { expected: "available", provider: "nvidia" }, credentials: { expected: "present" }, diff --git a/test/e2e-scenario/support-tests/e2e-expected-state.test.ts b/test/e2e-scenario/support-tests/e2e-expected-state.test.ts index 8d255b484c6..76b84d5ec41 100644 --- a/test/e2e-scenario/support-tests/e2e-expected-state.test.ts +++ b/test/e2e-scenario/support-tests/e2e-expected-state.test.ts @@ -39,6 +39,13 @@ describe("probesForState maps typed expected-state into probe ids", () => { ]); }); + it("Deep Agents Code ready state omits host dashboard health for terminal-agent parity", () => { + expect(probesForState(requireExpectedState("cloud-deepagents-code-ready"))).toEqual([ + "cli-installed", + "sandbox-running", + ]); + }); + it("preflight-failure state emits cli-installed, gateway-absent, sandbox-absent", () => { expect(probesForState(requireExpectedState("preflight-failure-no-sandbox"))).toEqual([ "cli-installed", From f13533bc8b00b21ff2446397bfc754d3ef8dfb6a Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Thu, 25 Jun 2026 18:28:55 -0400 Subject: [PATCH 4/8] test(e2e): repair deepagents policy parity --- .../policy-additions.yaml | 18 ++++++++++++------ .../checks/06-deepagents-code-python-egress.sh | 6 +++--- test/langchain-deepagents-code-image.test.ts | 4 ++++ 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/agents/langchain-deepagents-code/policy-additions.yaml b/agents/langchain-deepagents-code/policy-additions.yaml index ae0a5a59851..6e5df7147f5 100644 --- a/agents/langchain-deepagents-code/policy-additions.yaml +++ b/agents/langchain-deepagents-code/policy-additions.yaml @@ -68,10 +68,13 @@ network_policies: - { path: /usr/bin/git } - { path: /usr/local/bin/dcode } # OpenShell observes Python module traffic from dcode as the Python - # interpreter, not only as the /usr/local/bin/dcode shell wrapper. Keep - # this broad Python boundary limited to approved GitHub hosts; optional - # Tavily, LangSmith, MCP, and arbitrary hosts are intentionally absent. + # interpreter, not only as the /usr/local/bin/dcode shell wrapper. Match + # both Debian and /usr/local Python entrypoints/module paths seen on CI. + # Keep this broad Python boundary limited to approved GitHub hosts; + # optional Tavily, LangSmith, MCP, and arbitrary hosts are intentionally absent. - { path: /usr/bin/python3* } + - { path: /usr/local/bin/python3* } + - { path: /usr/local/lib/python3.13/** } pypi: name: pypi @@ -90,8 +93,11 @@ network_policies: - allow: { method: GET, path: "/**" } binaries: - { path: /usr/local/bin/pip3 } - # pip and dcode package-install traffic execute through Python. This is - # intentionally process-wide only for the read-only PyPI hosts listed - # above; optional service egress must be added explicitly by policy. + # pip and dcode package-install traffic execute through Python. Match + # both Debian and /usr/local Python entrypoints/module paths seen on CI. + # This is intentionally process-wide only for the read-only PyPI hosts + # listed above; optional service egress must be added explicitly by policy. - { path: /usr/bin/python3* } + - { path: /usr/local/bin/python3* } + - { path: /usr/local/lib/python3.13/** } - { path: /usr/local/bin/dcode } diff --git a/test/e2e/e2e-cloud-experimental/checks/06-deepagents-code-python-egress.sh b/test/e2e/e2e-cloud-experimental/checks/06-deepagents-code-python-egress.sh index 1495004f890..5a7f0fd3afb 100755 --- a/test/e2e/e2e-cloud-experimental/checks/06-deepagents-code-python-egress.sh +++ b/test/e2e/e2e-cloud-experimental/checks/06-deepagents-code-python-egress.sh @@ -49,7 +49,7 @@ expect_reached() { local label="$1" local url="$2" local output - output="$(python_probe "$url")" + output="$(python_probe "$url" || true)" if echo "$output" | grep -q "REACHED:"; then pass "arbitrary Python can reach approved ${label} host" else @@ -61,8 +61,8 @@ expect_blocked() { local label="$1" local url="$2" local output - output="$(python_probe "$url")" - if echo "$output" | grep -q "BLOCKED:" && ! echo "$output" | grep -q "REACHED:"; then + output="$(python_probe "$url" || true)" + if ! echo "$output" | grep -q "REACHED:"; then pass "arbitrary Python cannot reach ${label} without explicit policy" else fail_test "arbitrary Python reached ${label} unexpectedly: $output" diff --git a/test/langchain-deepagents-code-image.test.ts b/test/langchain-deepagents-code-image.test.ts index d1d1184c6b8..f61b1a972f6 100644 --- a/test/langchain-deepagents-code-image.test.ts +++ b/test/langchain-deepagents-code-image.test.ts @@ -219,6 +219,9 @@ describe("LangChain Deep Agents Code image contracts", () => { expect(policy).toContain("fail closed when Landlock cannot be applied"); expect(policy).toContain("silently degrading"); expect(policy).toContain("observes Python module traffic from dcode as the Python"); + expect(policy).toContain("/usr/bin/python3*"); + expect(policy).toContain("/usr/local/bin/python3*"); + expect(policy).toContain("/usr/local/lib/python3.13/**"); expect(policy).toContain("process-wide only for the read-only PyPI hosts"); expect(policy).toContain( "Tavily, LangSmith, MCP, and arbitrary hosts are intentionally absent", @@ -257,6 +260,7 @@ describe("LangChain Deep Agents Code image contracts", () => { expect(landlockCheck).toContain("/usr is Landlock read-only for Deep Agents Code"); expect(landlockCheck).toContain("/etc is Landlock read-only for Deep Agents Code"); expect(pythonEgressCheck).toContain("python3 - ${url@Q} <<'PY'"); + expect(pythonEgressCheck).toContain('output="$(python_probe "$url" || true)"'); expect(pythonEgressCheck).toContain('expect_reached "GitHub" "https://api.github.com/"'); expect(pythonEgressCheck).toContain('expect_reached "PyPI" "https://pypi.org/"'); expect(pythonEgressCheck).toContain("https://api.tavily.com/"); From 77977671e8ce9b43451c87afb46cfa8425e1b135 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Thu, 25 Jun 2026 18:36:55 -0400 Subject: [PATCH 5/8] test(e2e): avoid multiline deepagents probes --- .../checks/06-deepagents-code-python-egress.sh | 12 +----------- test/langchain-deepagents-code-image.test.ts | 3 ++- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/test/e2e/e2e-cloud-experimental/checks/06-deepagents-code-python-egress.sh b/test/e2e/e2e-cloud-experimental/checks/06-deepagents-code-python-egress.sh index 5a7f0fd3afb..d94f1b888ed 100755 --- a/test/e2e/e2e-cloud-experimental/checks/06-deepagents-code-python-egress.sh +++ b/test/e2e/e2e-cloud-experimental/checks/06-deepagents-code-python-egress.sh @@ -32,17 +32,7 @@ sandbox_exec() { python_probe() { local url="$1" - sandbox_exec "python3 - ${url@Q} <<'PY' -import sys -import urllib.request -url = sys.argv[1] -try: - with urllib.request.urlopen(url, timeout=8) as response: - print(f'REACHED:{response.status}') -except Exception as exc: - print(f'BLOCKED:{type(exc).__name__}:{exc}') -PY -" + sandbox_exec "python3 -c \$'import sys\\nimport urllib.request\\nurl = sys.argv[1]\\ntry:\\n with urllib.request.urlopen(url, timeout=8) as response:\\n print(\"REACHED:%s\" % response.status)\\nexcept Exception as exc:\\n print(\"BLOCKED:%s:%s\" % (type(exc).__name__, exc))' ${url@Q}" } expect_reached() { diff --git a/test/langchain-deepagents-code-image.test.ts b/test/langchain-deepagents-code-image.test.ts index f61b1a972f6..cbe6895d058 100644 --- a/test/langchain-deepagents-code-image.test.ts +++ b/test/langchain-deepagents-code-image.test.ts @@ -259,7 +259,8 @@ describe("LangChain Deep Agents Code image contracts", () => { expect(landlockCheck).toContain("touch /tmp/deepagents-landlock-test"); expect(landlockCheck).toContain("/usr is Landlock read-only for Deep Agents Code"); expect(landlockCheck).toContain("/etc is Landlock read-only for Deep Agents Code"); - expect(pythonEgressCheck).toContain("python3 - ${url@Q} <<'PY'"); + expect(pythonEgressCheck).toContain("python3 -c"); + expect(pythonEgressCheck).not.toContain("<<'PY'"); expect(pythonEgressCheck).toContain('output="$(python_probe "$url" || true)"'); expect(pythonEgressCheck).toContain('expect_reached "GitHub" "https://api.github.com/"'); expect(pythonEgressCheck).toContain('expect_reached "PyPI" "https://pypi.org/"'); From 22d4fae19268237fb34d4b5fe2908db15ccb4518 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Thu, 25 Jun 2026 18:50:17 -0400 Subject: [PATCH 6/8] test(e2e): fail closed deepagents parity checks --- .../live/cloud-experimental-check-list.ts | 15 +++ .../live/cloud-experimental-checks.ts | 96 +++++++++++++++++++ .../live/registry-scenarios.test.ts | 61 ++---------- test/e2e-scenario/live/run-plan.ts | 11 ++- ...platform-parity-cloud-experimental.test.ts | 54 +++++++++++ .../06-deepagents-code-python-egress.sh | 6 +- test/langchain-deepagents-code-image.test.ts | 8 ++ 7 files changed, 192 insertions(+), 59 deletions(-) create mode 100644 test/e2e-scenario/live/cloud-experimental-check-list.ts create mode 100644 test/e2e-scenario/live/cloud-experimental-checks.ts create mode 100644 test/e2e-scenario/support-tests/platform-parity-cloud-experimental.test.ts diff --git a/test/e2e-scenario/live/cloud-experimental-check-list.ts b/test/e2e-scenario/live/cloud-experimental-check-list.ts new file mode 100644 index 00000000000..6846364f802 --- /dev/null +++ b/test/e2e-scenario/live/cloud-experimental-check-list.ts @@ -0,0 +1,15 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export const DEEPAGENTS_CLOUD_EXPERIMENTAL_CHECKS = [ + "test/e2e/e2e-cloud-experimental/checks/05-deepagents-code-landlock-readonly.sh", + "test/e2e/e2e-cloud-experimental/checks/06-deepagents-code-python-egress.sh", +] as const; + +export function cloudExperimentalChecksForOnboarding( + onboarding: string | undefined, +): readonly string[] { + return onboarding === "cloud-langchain-deepagents-code" + ? DEEPAGENTS_CLOUD_EXPERIMENTAL_CHECKS + : []; +} diff --git a/test/e2e-scenario/live/cloud-experimental-checks.ts b/test/e2e-scenario/live/cloud-experimental-checks.ts new file mode 100644 index 00000000000..096b2e3be72 --- /dev/null +++ b/test/e2e-scenario/live/cloud-experimental-checks.ts @@ -0,0 +1,96 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import path from "node:path"; + +import { expect } from "vitest"; +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { resultText } from "../fixtures/clients/command.ts"; +import type { E2EScenarioFixtures } from "../fixtures/e2e-test.ts"; +import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; + +const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); +const REQUIRED_CHECK_SKIP_PATTERN = /(^|\n).*\bSKIP\b/i; + +export function buildCloudExperimentalCommandEnv( + sandboxName: string, + apiKey: string, + base: NodeJS.ProcessEnv = process.env, +): NodeJS.ProcessEnv { + return { + ...buildAvailabilityProbeEnv(base), + CLOUD_EXPERIMENTAL_MODEL: base.NEMOCLAW_MODEL, + COMPATIBLE_API_KEY: apiKey, + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", + NEMOCLAW_E2E_CLOUD_API_KEY_ENV: "COMPATIBLE_API_KEY", + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_SANDBOX_NAME: sandboxName, + OPENSHELL_GATEWAY: "nemoclaw", + REPO: REPO_ROOT, + SANDBOX_NAME: sandboxName, + }; +} + +export function assertRequiredCloudExperimentalResult( + scriptPath: string, + result: ShellProbeResult, +): void { + const output = resultText(result); + expect(result.exitCode, `${scriptPath}: ${output}`).toBe(0); + expect( + output, + `${scriptPath}: required cloud-experimental check must not skip`, + ).not.toMatch(REQUIRED_CHECK_SKIP_PATTERN); +} + +async function assertDeepAgentsRuntimeObserved( + sandboxName: string, + context: Pick, +): Promise { + const result = await context.host.command( + "openshell", + [ + "sandbox", + "exec", + "--name", + sandboxName, + "--", + "bash", + "-c", + "test -d /sandbox/.deepagents && command -v dcode >/dev/null", + ], + { + artifactName: "cloud-experimental-deepagents-runtime", + env: buildCloudExperimentalCommandEnv(sandboxName, ""), + timeoutMs: 30_000, + }, + ); + expect(result.exitCode, `Deep Agents Code runtime marker missing: ${resultText(result)}`).toBe(0); +} + +export async function runE2eCloudExperimentalChecks( + scenarioId: string, + sandboxName: string, + checkScripts: readonly string[], + context: Pick, +): Promise { + const apiKey = context.secrets.optional("NVIDIA_INFERENCE_API_KEY") ?? ""; + await context.artifacts.writeJson("e2e-cloud-experimental-checks.json", { + scenarioId, + sandboxName, + checkScripts, + }); + await Promise.resolve( + checkScripts.length > 0 ? assertDeepAgentsRuntimeObserved(sandboxName, context) : undefined, + ); + for (const scriptPath of checkScripts) { + const result = await context.host.command("bash", [path.join(REPO_ROOT, scriptPath)], { + artifactName: `cloud-experimental-${path.basename(scriptPath, ".sh")}`, + cwd: REPO_ROOT, + env: buildCloudExperimentalCommandEnv(sandboxName, apiKey), + redactionValues: [apiKey], + timeoutMs: 180_000, + }); + assertRequiredCloudExperimentalResult(scriptPath, result); + } +} diff --git a/test/e2e-scenario/live/registry-scenarios.test.ts b/test/e2e-scenario/live/registry-scenarios.test.ts index 1e1df4edf9f..f3da5ad1408 100644 --- a/test/e2e-scenario/live/registry-scenarios.test.ts +++ b/test/e2e-scenario/live/registry-scenarios.test.ts @@ -2,13 +2,14 @@ // SPDX-License-Identifier: Apache-2.0 import fs from "node:fs"; -import os from "node:os"; import path from "node:path"; -import { expect, test, type E2EScenarioFixtures } from "../fixtures/e2e-test.ts"; +import { expect, test } from "../fixtures/e2e-test.ts"; import type { LifecycleProfile } from "../fixtures/phases/index.ts"; import { listScenarios } from "../scenarios/registry.ts"; import { liveScenarioSupport, liveScenarioTestName } from "../scenarios/runtime-support.ts"; +import { cloudExperimentalChecksForOnboarding } from "./cloud-experimental-check-list.ts"; +import { runE2eCloudExperimentalChecks } from "./cloud-experimental-checks.ts"; import { buildLiveScenarioRunPlan } from "./run-plan.ts"; const LIFECYCLE_PROFILES: ReadonlySet = new Set(["post-reboot-recovery"]); @@ -17,48 +18,6 @@ function isLifecycleProfile(value: string | undefined): value is LifecycleProfil return value !== undefined && LIFECYCLE_PROFILES.has(value as LifecycleProfile); } -function commandEnv(sandboxName: string, extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { - return { - ...process.env, - PATH: `${os.homedir()}/.local/bin:${os.homedir()}/.npm-global/bin:${process.env.PATH ?? ""}`, - NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", - NEMOCLAW_NON_INTERACTIVE: "1", - NEMOCLAW_SANDBOX_NAME: sandboxName, - OPENSHELL_GATEWAY: "nemoclaw", - ...extra, - }; -} - -async function runE2eCloudExperimentalChecks( - scenarioId: string, - sandboxName: string, - checkScripts: readonly string[], - context: Pick, -): Promise { - const apiKey = context.secrets.optional("NVIDIA_INFERENCE_API_KEY") ?? ""; - await context.artifacts.writeJson("e2e-cloud-experimental-checks.json", { - scenarioId, - sandboxName, - checkScripts, - }); - for (const scriptPath of checkScripts) { - const result = await context.host.command("bash", [path.join(REPO_ROOT, scriptPath)], { - artifactName: `cloud-experimental-${path.basename(scriptPath, ".sh")}`, - cwd: REPO_ROOT, - env: commandEnv(sandboxName, { - CLOUD_EXPERIMENTAL_MODEL: process.env.NEMOCLAW_MODEL, - COMPATIBLE_API_KEY: apiKey, - NEMOCLAW_E2E_CLOUD_API_KEY_ENV: "COMPATIBLE_API_KEY", - REPO: REPO_ROOT, - SANDBOX_NAME: sandboxName, - }), - redactionValues: [apiKey], - timeoutMs: 180_000, - }); - expect(result.exitCode, `${scriptPath}: ${result.stdout}\n${result.stderr}`).toBe(0); - } -} - const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); const CLI_DIST_ENTRYPOINT = path.join(REPO_ROOT, "dist", "nemoclaw.js"); const E2E_CLOUD_EXPERIMENTAL_CHECKS_DIR = path.join( @@ -108,7 +67,8 @@ for (const scenario of listScenarios()) { pendingRuntimeSuites: support.pendingRuntimeSuites, }); - await artifacts.writeJson("run-plan.json", buildLiveScenarioRunPlan(scenario)); + const runPlan = buildLiveScenarioRunPlan(scenario); + await artifacts.writeJson("run-plan.json", runPlan); const ready = await environment.assertReady(scenario.environment); const instance = await onboard.from(ready, { sandboxName: `e2e-${scenario.id}` }); @@ -139,13 +99,10 @@ for (const scenario of listScenarios()) { const validation = await stateValidation.from(scenario.expectedStateId, instance); - const checkScripts = - scenario.environment.onboarding === "cloud-langchain-deepagents-code" - ? [ - "test/e2e/e2e-cloud-experimental/checks/05-deepagents-code-landlock-readonly.sh", - "test/e2e/e2e-cloud-experimental/checks/06-deepagents-code-python-egress.sh", - ] - : []; + const checkScripts = runPlan.e2eCloudExperimentalChecks ?? []; + expect(checkScripts).toEqual( + cloudExperimentalChecksForOnboarding(scenario.environment.onboarding), + ); for (const scriptPath of checkScripts) { expect(fs.existsSync(path.join(REPO_ROOT, scriptPath))).toBe(true); } diff --git a/test/e2e-scenario/live/run-plan.ts b/test/e2e-scenario/live/run-plan.ts index 0ce195b93fd..df0d9481a4b 100644 --- a/test/e2e-scenario/live/run-plan.ts +++ b/test/e2e-scenario/live/run-plan.ts @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { cloudExperimentalChecksForOnboarding } from "./cloud-experimental-check-list.ts"; import type { ScenarioDefinition } from "../scenarios/types.ts"; export interface LiveScenarioRunPlan { @@ -25,11 +26,11 @@ export function buildLiveScenarioRunPlan(scenario: ScenarioDefinition): LiveScen "state-validation", ], }; - if (scenario.environment?.onboarding === "cloud-langchain-deepagents-code") { - plan.e2eCloudExperimentalChecks = [ - "test/e2e/e2e-cloud-experimental/checks/05-deepagents-code-landlock-readonly.sh", - "test/e2e/e2e-cloud-experimental/checks/06-deepagents-code-python-egress.sh", - ]; + const cloudExperimentalChecks = cloudExperimentalChecksForOnboarding( + scenario.environment?.onboarding, + ); + if (cloudExperimentalChecks.length > 0) { + plan.e2eCloudExperimentalChecks = [...cloudExperimentalChecks]; } return plan; } diff --git a/test/e2e-scenario/support-tests/platform-parity-cloud-experimental.test.ts b/test/e2e-scenario/support-tests/platform-parity-cloud-experimental.test.ts new file mode 100644 index 00000000000..fd7dd7d91d4 --- /dev/null +++ b/test/e2e-scenario/support-tests/platform-parity-cloud-experimental.test.ts @@ -0,0 +1,54 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + assertRequiredCloudExperimentalResult, + buildCloudExperimentalCommandEnv, +} from "../live/cloud-experimental-checks.ts"; +import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; + +function shellResult(exitCode: number, stdout: string, stderr = ""): ShellProbeResult { + return { + command: [], + exitCode, + signal: null, + timedOut: false, + stdout, + stderr, + }; +} + +describe("P0-E cloud-experimental parity guardrails", () => { + it("fails required Deep Agents cloud-experimental checks when scripts print SKIP", () => { + expect(() => + assertRequiredCloudExperimentalResult( + "test/e2e/e2e-cloud-experimental/checks/05-deepagents-code-landlock-readonly.sh", + shellResult(0, "05-deepagents-code-landlock-readonly: SKIP: not a Deep Agents sandbox\n"), + ), + ).toThrow(/must not skip/); + }); + + it("builds a minimal cloud-experimental child environment", () => { + const env = buildCloudExperimentalCommandEnv("deepagents-sandbox", "secret-key", { + HOME: "/home/runner", + PATH: "/usr/bin", + AWS_SECRET_ACCESS_KEY: "do-not-copy", + GITHUB_TOKEN: "do-not-copy", + NEMOCLAW_MODEL: "model-a", + RANDOM_RUNNER_SECRET: "do-not-copy", + }); + + expect(env).toMatchObject({ + COMPATIBLE_API_KEY: "secret-key", + CLOUD_EXPERIMENTAL_MODEL: "model-a", + NEMOCLAW_SANDBOX_NAME: "deepagents-sandbox", + SANDBOX_NAME: "deepagents-sandbox", + OPENSHELL_GATEWAY: "nemoclaw", + }); + expect(env.AWS_SECRET_ACCESS_KEY).toBeUndefined(); + expect(env.GITHUB_TOKEN).toBeUndefined(); + expect(env.RANDOM_RUNNER_SECRET).toBeUndefined(); + }); +}); diff --git a/test/e2e/e2e-cloud-experimental/checks/06-deepagents-code-python-egress.sh b/test/e2e/e2e-cloud-experimental/checks/06-deepagents-code-python-egress.sh index d94f1b888ed..0ea565dfd0b 100755 --- a/test/e2e/e2e-cloud-experimental/checks/06-deepagents-code-python-egress.sh +++ b/test/e2e/e2e-cloud-experimental/checks/06-deepagents-code-python-egress.sh @@ -52,10 +52,12 @@ expect_blocked() { local url="$2" local output output="$(python_probe "$url" || true)" - if ! echo "$output" | grep -q "REACHED:"; then + if echo "$output" | grep -q "BLOCKED:" && ! echo "$output" | grep -q "REACHED:"; then pass "arbitrary Python cannot reach ${label} without explicit policy" - else + elif echo "$output" | grep -q "REACHED:"; then fail_test "arbitrary Python reached ${label} unexpectedly: $output" + else + fail_test "arbitrary Python probe for ${label} lacked denial evidence: $output" fi } diff --git a/test/langchain-deepagents-code-image.test.ts b/test/langchain-deepagents-code-image.test.ts index cbe6895d058..006c181ab93 100644 --- a/test/langchain-deepagents-code-image.test.ts +++ b/test/langchain-deepagents-code-image.test.ts @@ -8,6 +8,8 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; +import { cloudExperimentalChecksForOnboarding } from "./e2e-scenario/live/cloud-experimental-check-list.ts"; + const agentDir = path.join(process.cwd(), "agents", "langchain-deepagents-code"); function readAgentFile(name: string): string { @@ -271,6 +273,12 @@ describe("LangChain Deep Agents Code image contracts", () => { expect(pythonEgressCheck).toContain( "arbitrary Python cannot reach ${label} without explicit policy", ); + expect(pythonEgressCheck).toContain("grep -q \"BLOCKED:\""); + expect(pythonEgressCheck).toContain("lacked denial evidence"); + expect(cloudExperimentalChecksForOnboarding("cloud-langchain-deepagents-code")).toEqual([ + "test/e2e/e2e-cloud-experimental/checks/05-deepagents-code-landlock-readonly.sh", + "test/e2e/e2e-cloud-experimental/checks/06-deepagents-code-python-egress.sh", + ]); }); it("hash-locks Deep Agents Code base image PyPI installs", () => { From 378dc5b5a28ad36f60f21a0caf6d8505a443a12f Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Thu, 25 Jun 2026 19:04:14 -0400 Subject: [PATCH 7/8] test(e2e): satisfy parity check typing --- test/e2e-scenario/live/cloud-experimental-checks.ts | 7 +++---- .../platform-parity-cloud-experimental.test.ts | 5 +++++ test/langchain-deepagents-code-image.test.ts | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/test/e2e-scenario/live/cloud-experimental-checks.ts b/test/e2e-scenario/live/cloud-experimental-checks.ts index 096b2e3be72..c5b1b20cf6a 100644 --- a/test/e2e-scenario/live/cloud-experimental-checks.ts +++ b/test/e2e-scenario/live/cloud-experimental-checks.ts @@ -37,10 +37,9 @@ export function assertRequiredCloudExperimentalResult( ): void { const output = resultText(result); expect(result.exitCode, `${scriptPath}: ${output}`).toBe(0); - expect( - output, - `${scriptPath}: required cloud-experimental check must not skip`, - ).not.toMatch(REQUIRED_CHECK_SKIP_PATTERN); + expect(output, `${scriptPath}: required cloud-experimental check must not skip`).not.toMatch( + REQUIRED_CHECK_SKIP_PATTERN, + ); } async function assertDeepAgentsRuntimeObserved( diff --git a/test/e2e-scenario/support-tests/platform-parity-cloud-experimental.test.ts b/test/e2e-scenario/support-tests/platform-parity-cloud-experimental.test.ts index fd7dd7d91d4..368f3c7032b 100644 --- a/test/e2e-scenario/support-tests/platform-parity-cloud-experimental.test.ts +++ b/test/e2e-scenario/support-tests/platform-parity-cloud-experimental.test.ts @@ -17,6 +17,11 @@ function shellResult(exitCode: number, stdout: string, stderr = ""): ShellProbeR timedOut: false, stdout, stderr, + artifacts: { + stdout: "stdout.txt", + stderr: "stderr.txt", + result: "result.json", + }, }; } diff --git a/test/langchain-deepagents-code-image.test.ts b/test/langchain-deepagents-code-image.test.ts index 006c181ab93..ee3ff5af24d 100644 --- a/test/langchain-deepagents-code-image.test.ts +++ b/test/langchain-deepagents-code-image.test.ts @@ -273,7 +273,7 @@ describe("LangChain Deep Agents Code image contracts", () => { expect(pythonEgressCheck).toContain( "arbitrary Python cannot reach ${label} without explicit policy", ); - expect(pythonEgressCheck).toContain("grep -q \"BLOCKED:\""); + expect(pythonEgressCheck).toContain('grep -q "BLOCKED:"'); expect(pythonEgressCheck).toContain("lacked denial evidence"); expect(cloudExperimentalChecksForOnboarding("cloud-langchain-deepagents-code")).toEqual([ "test/e2e/e2e-cloud-experimental/checks/05-deepagents-code-landlock-readonly.sh", From 334ca3658b37a964e1022759273d4ccf1ba23c0e Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Thu, 25 Jun 2026 18:05:08 -0700 Subject: [PATCH 8/8] test(e2e): stabilize post-merge checks --- test/dcode-wrapper-empty-prompt.test.ts | 11 ++++++++--- test/e2e-scenario/live/gpu-e2e.test.ts | 14 +++++++------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/test/dcode-wrapper-empty-prompt.test.ts b/test/dcode-wrapper-empty-prompt.test.ts index f5920982e34..2dc7c7c8969 100644 --- a/test/dcode-wrapper-empty-prompt.test.ts +++ b/test/dcode-wrapper-empty-prompt.test.ts @@ -48,12 +48,17 @@ type WrapperRun = { function runWrapper(args: string[]): WrapperRun { const dir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-dcode-wrapper-")); try { - fs.copyFileSync(WRAPPER, path.join(dir, "dcode")); - fs.chmodSync(path.join(dir, "dcode"), 0o755); - const marker = path.join(dir, "launched.txt"); const bin = path.join(dir, "bin"); fs.mkdirSync(bin); + const wrapperFixture = fs + .readFileSync(WRAPPER, "utf8") + .replace( + 'export PATH="/usr/local/bin:/opt/venv/bin:/usr/local/sbin:/usr/sbin:/usr/bin:/sbin:/bin"', + `export PATH="${bin}:/usr/local/bin:/opt/venv/bin:/usr/local/sbin:/usr/sbin:/usr/bin:/sbin:/bin"`, + ); + fs.writeFileSync(path.join(dir, "dcode"), wrapperFixture, { mode: 0o755 }); + fs.writeFileSync( path.join(bin, "python3"), `#!/usr/bin/env bash\nprintf '%s' "$*" > ${JSON.stringify(marker)}\nexit 0\n`, diff --git a/test/e2e-scenario/live/gpu-e2e.test.ts b/test/e2e-scenario/live/gpu-e2e.test.ts index fe250f4f19f..b479cc9cc87 100644 --- a/test/e2e-scenario/live/gpu-e2e.test.ts +++ b/test/e2e-scenario/live/gpu-e2e.test.ts @@ -58,13 +58,13 @@ function assertSmallContextCompactionPolicy(configText: string): void { }) .map(asRecord) .find((candidate) => { - if (!candidate || !primary || !primaryWithoutProvider) { - return false; - } - const identifiers = ["id", "name", "label"].flatMap((key) => { - const value = modelIdentifier(candidate, key); - return value ? [value] : []; - }); + const identifiers = + candidate && primary && primaryWithoutProvider + ? ["id", "name", "label"].flatMap((key) => { + const value = modelIdentifier(candidate, key); + return value ? [value] : []; + }) + : []; return identifiers.some( (identifier) => identifier === primary ||