From 20a3e3f58f79d31e42cfa550658beaf3cdaf5eb4 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sat, 27 Jun 2026 17:56:56 -0700 Subject: [PATCH 1/7] fix(e2e): correct full release gate lanes Signed-off-by: Carlos Villela --- .github/workflows/e2e-vitest-scenarios.yaml | 6 ++- .github/workflows/regression-e2e.yaml | 2 +- ...outer-provider-routed-inference-helpers.ts | 35 +++++++++++++ ...l-router-provider-routed-inference.test.ts | 26 +++------- ...l-router-provider-routed-inference.test.ts | 36 +++++++++++++ ...-model-router-provider-routed-inference.sh | 16 +++--- test/regression-e2e-workflow.test.ts | 9 ++++ tools/e2e-scenarios/workflow-boundary.mts | 52 ++++++++++++++----- 8 files changed, 142 insertions(+), 40 deletions(-) create mode 100644 test/e2e-scenario/live/model-router-provider-routed-inference-helpers.ts create mode 100644 test/e2e-scenario/support-tests/model-router-provider-routed-inference.test.ts diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index 74fac992066..0377aa958a3 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -3958,7 +3958,7 @@ jobs: # sandbox inference.local completion boundaries without adding registry # or migration-ledger wiring. env: - NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }} + NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} run: | set -euo pipefail npx vitest run --project e2e-scenarios-live \ @@ -4252,7 +4252,9 @@ jobs: # Docker/OpenShell mutation. run: | set -euo pipefail - npx vitest run --project cli test/gateway-drift-preflight.test.ts --silent=false --reporter=default + npx vitest run --project integration \ + test/gateway-drift-preflight.test.ts \ + --silent=false --reporter=default - name: Upload gateway drift preflight artifacts if: always() diff --git a/.github/workflows/regression-e2e.yaml b/.github/workflows/regression-e2e.yaml index 7180ef65820..2ed482a1465 100644 --- a/.github/workflows/regression-e2e.yaml +++ b/.github/workflows/regression-e2e.yaml @@ -248,7 +248,7 @@ jobs: - name: Run Model Router provider-routed inference E2E test env: - NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }} + NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" run: bash test/e2e/test-model-router-provider-routed-inference.sh diff --git a/test/e2e-scenario/live/model-router-provider-routed-inference-helpers.ts b/test/e2e-scenario/live/model-router-provider-routed-inference-helpers.ts new file mode 100644 index 00000000000..9d063f13d11 --- /dev/null +++ b/test/e2e-scenario/live/model-router-provider-routed-inference-helpers.ts @@ -0,0 +1,35 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; + +export const MODEL_ROUTER_PUBLIC_KEY_ENV = "NVIDIA_API_KEY"; + +export interface ModelRouterSecrets { + required(name: string): string; +} + +export function requireModelRouterPublicKey(secrets: ModelRouterSecrets): string { + const apiKey = secrets.required(MODEL_ROUTER_PUBLIC_KEY_ENV); + if (!apiKey.startsWith("nvapi-")) { + throw new Error("NVIDIA_API_KEY must be a public NVIDIA Endpoints nvapi-* key"); + } + return apiKey; +} + +export function buildProviderRoutedEnv( + apiKey: string, + sandboxName: string, + baseEnv: NodeJS.ProcessEnv = process.env, +): NodeJS.ProcessEnv { + return { + ...buildAvailabilityProbeEnv(baseEnv), + NVIDIA_INFERENCE_API_KEY: apiKey, + NEMOCLAW_PROVIDER_KEY: apiKey, + NEMOCLAW_SANDBOX_NAME: sandboxName, + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", + NEMOCLAW_POLICY_TIER: "open", + NEMOCLAW_PROVIDER: "routed", + }; +} diff --git a/test/e2e-scenario/live/model-router-provider-routed-inference.test.ts b/test/e2e-scenario/live/model-router-provider-routed-inference.test.ts index 79aec674626..da1c67804fe 100644 --- a/test/e2e-scenario/live/model-router-provider-routed-inference.test.ts +++ b/test/e2e-scenario/live/model-router-provider-routed-inference.test.ts @@ -7,6 +7,10 @@ import path from "node:path"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; import { shouldRunLiveE2EScenarios } from "../fixtures/live-project-gate.ts"; +import { + buildProviderRoutedEnv, + requireModelRouterPublicKey, +} from "./model-router-provider-routed-inference-helpers.ts"; // Focused Vitest live replacement for // test/e2e/test-model-router-provider-routed-inference.sh. Keep this as a @@ -72,19 +76,6 @@ function routedPongReason(raw: string): "ok" | string { return "ok"; } -function withProviderRoutedEnv(apiKey: string): NodeJS.ProcessEnv { - return { - ...buildAvailabilityProbeEnv(), - NVIDIA_INFERENCE_API_KEY: apiKey, - NEMOCLAW_PROVIDER_KEY: apiKey, - NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, - NEMOCLAW_NON_INTERACTIVE: "1", - NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", - NEMOCLAW_POLICY_TIER: "open", - NEMOCLAW_PROVIDER: "routed", - }; -} - test.skipIf(!shouldRunLiveE2EScenarios())( "model-router provider-routed onboard returns routed inference.local PONG", async ({ artifacts, cleanup, host, sandbox, secrets, skip }) => { @@ -107,10 +98,7 @@ test.skipIf(!shouldRunLiveE2EScenarios())( skip("Docker is required for provider-routed Model Router onboarding"); } - const apiKey = secrets.required("NVIDIA_INFERENCE_API_KEY"); - expect(apiKey.startsWith("nvapi-"), "NVIDIA_INFERENCE_API_KEY must start with nvapi-").toBe( - true, - ); + const apiKey = requireModelRouterPublicKey(secrets); await artifacts.writeJson("scenario.json", { id: "model-router-provider-routed-inference", @@ -119,7 +107,7 @@ test.skipIf(!shouldRunLiveE2EScenarios())( legacySource: "test/e2e/test-model-router-provider-routed-inference.sh", contract: [ "Docker is available before onboarding", - "NVIDIA_INFERENCE_API_KEY is present and nvapi-prefixed", + "NVIDIA_API_KEY is present and nvapi-prefixed, then staged for the router's NVIDIA_INFERENCE_API_KEY credential", "nemoclaw onboard --fresh completes with NEMOCLAW_PROVIDER=routed", "host model-router health reports at least one healthy endpoint", "sandbox inference.local returns model nvidia-routed with PONG content", @@ -152,7 +140,7 @@ test.skipIf(!shouldRunLiveE2EScenarios())( ], { artifactName: "onboard-model-router-provider-routed", - env: withProviderRoutedEnv(apiKey), + env: buildProviderRoutedEnv(apiKey, SANDBOX_NAME), redactionValues: [apiKey], timeoutMs: ONBOARD_TIMEOUT_MS, }, diff --git a/test/e2e-scenario/support-tests/model-router-provider-routed-inference.test.ts b/test/e2e-scenario/support-tests/model-router-provider-routed-inference.test.ts new file mode 100644 index 00000000000..2a3d199789e --- /dev/null +++ b/test/e2e-scenario/support-tests/model-router-provider-routed-inference.test.ts @@ -0,0 +1,36 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + buildProviderRoutedEnv, + requireModelRouterPublicKey, +} from "../live/model-router-provider-routed-inference-helpers.ts"; + +describe("Model Router provider-routed live support", () => { + it("requires the public NVIDIA secret", () => { + const requested: string[] = []; + const apiKey = requireModelRouterPublicKey({ + required(name) { + requested.push(name); + return "nvapi-public-test-key"; + }, + }); + + expect(requested).toEqual(["NVIDIA_API_KEY"]); + expect(apiKey).toBe("nvapi-public-test-key"); + expect(() => requireModelRouterPublicKey({ required: () => "hosted-compatible-key" })).toThrow( + "NVIDIA_API_KEY must be a public NVIDIA Endpoints nvapi-* key", + ); + }); + + it("stages the public key under the credential names consumed by the router", () => { + expect(buildProviderRoutedEnv("nvapi-public-test-key", "e2e-router", {})).toMatchObject({ + NVIDIA_INFERENCE_API_KEY: "nvapi-public-test-key", + NEMOCLAW_PROVIDER_KEY: "nvapi-public-test-key", + NEMOCLAW_PROVIDER: "routed", + NEMOCLAW_SANDBOX_NAME: "e2e-router", + }); + }); +}); diff --git a/test/e2e/test-model-router-provider-routed-inference.sh b/test/e2e/test-model-router-provider-routed-inference.sh index cac9687626c..32d4cfcfa83 100755 --- a/test/e2e/test-model-router-provider-routed-inference.sh +++ b/test/e2e/test-model-router-provider-routed-inference.sh @@ -68,7 +68,11 @@ redact_file() { python3 - "$file" <<'PY' import os, sys path = sys.argv[1] -secrets = [os.environ.get("NVIDIA_INFERENCE_API_KEY", ""), os.environ.get("NEMOCLAW_PROVIDER_KEY", "")] +secrets = [ + os.environ.get("NVIDIA_API_KEY", ""), + os.environ.get("NVIDIA_INFERENCE_API_KEY", ""), + os.environ.get("NEMOCLAW_PROVIDER_KEY", ""), +] text = open(path, "r", errors="replace").read() for secret in filter(None, secrets): text = text.replace(secret, "") @@ -97,10 +101,10 @@ else exit 1 fi -if [ -n "${NVIDIA_INFERENCE_API_KEY:-}" ] && [[ "${NVIDIA_INFERENCE_API_KEY}" == nvapi-* ]]; then - pass "NVIDIA_INFERENCE_API_KEY is set" +if [ -n "${NVIDIA_API_KEY:-}" ] && [[ "${NVIDIA_API_KEY}" == nvapi-* ]]; then + pass "NVIDIA_API_KEY is set" else - fail "NVIDIA_INFERENCE_API_KEY is required and must start with nvapi-" + fail "NVIDIA_API_KEY is required and must start with nvapi-" exit 1 fi @@ -124,13 +128,13 @@ rm -f "$HOME/.nemoclaw/onboard.lock" 2>/dev/null || true nemoclaw "$SANDBOX_NAME" destroy --yes >/dev/null 2>&1 || true env \ - NEMOCLAW_PROVIDER_KEY="$NVIDIA_INFERENCE_API_KEY" \ + NEMOCLAW_PROVIDER_KEY="$NVIDIA_API_KEY" \ NEMOCLAW_SANDBOX_NAME="$SANDBOX_NAME" \ NEMOCLAW_NON_INTERACTIVE=1 \ NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ NEMOCLAW_POLICY_TIER="open" \ NEMOCLAW_PROVIDER="routed" \ - NVIDIA_INFERENCE_API_KEY="$NVIDIA_INFERENCE_API_KEY" \ + NVIDIA_INFERENCE_API_KEY="$NVIDIA_API_KEY" \ "$TIMEOUT_CMD" 1500 nemoclaw onboard --fresh --non-interactive --yes-i-accept-third-party-software \ >"$ONBOARD_LOG" 2>&1 onboard_rc=$? diff --git a/test/regression-e2e-workflow.test.ts b/test/regression-e2e-workflow.test.ts index 87b8233402b..7d94fcee0e3 100644 --- a/test/regression-e2e-workflow.test.ts +++ b/test/regression-e2e-workflow.test.ts @@ -63,6 +63,15 @@ describe("Regression E2E workflow contract", () => { expect(runText).not.toContain("test/e2e/test-whatsapp-qr-compact-e2e.sh"); }); + it("stages the public NVIDIA key for the Model Router's NVIDIA credential", () => { + const job = workflow.jobs?.["model-router-provider-routed-inference-e2e"]; + const runStep = job?.steps?.find( + (step) => step.name === "Run Model Router provider-routed inference E2E test", + ); + expect(runStep?.env?.NVIDIA_API_KEY).toBe("${{ secrets.NVIDIA_API_KEY }}"); + expect(runStep?.env?.NVIDIA_INFERENCE_API_KEY).toBeUndefined(); + }); + it("runs OpenClaw plugin runtime-deps EXDEV through a secret-free Vitest lane", () => { const job = workflow.jobs?.["openclaw-plugin-runtime-exdev-e2e"]; const steps = job?.steps ?? []; diff --git a/tools/e2e-scenarios/workflow-boundary.mts b/tools/e2e-scenarios/workflow-boundary.mts index 519f29fc5f3..508e8e32a55 100644 --- a/tools/e2e-scenarios/workflow-boundary.mts +++ b/tools/e2e-scenarios/workflow-boundary.mts @@ -4997,6 +4997,7 @@ function validateModelRouterProviderRoutedInferenceVitestJob( ); } for (const secret of [ + "NVIDIA_API_KEY", "NVIDIA_INFERENCE_API_KEY", "DOCKERHUB_USERNAME", "DOCKERHUB_TOKEN", @@ -5020,9 +5021,15 @@ function validateModelRouterProviderRoutedInferenceVitestJob( errors, stepName, stepEnv, - "NVIDIA_INFERENCE_API_KEY", + "NVIDIA_API_KEY", ); } + requireEnvDoesNotExposeSecret( + errors, + stepName, + stepEnv, + "NVIDIA_INFERENCE_API_KEY", + ); if (step.name !== "Authenticate to Docker Hub") { requireEnvDoesNotExposeSecret( errors, @@ -5132,12 +5139,9 @@ function validateModelRouterProviderRoutedInferenceVitestJob( "Run Model Router provider-routed inference live test", ); const runVitestEnv = asRecord(runVitest?.env); - if ( - runVitestEnv.NVIDIA_INFERENCE_API_KEY !== - "${{ secrets.NVIDIA_INFERENCE_API_KEY }}" - ) { + if (runVitestEnv.NVIDIA_API_KEY !== "${{ secrets.NVIDIA_API_KEY }}") { errors.push( - "model-router-provider-routed-inference-vitest Vitest step must receive NVIDIA_INFERENCE_API_KEY from secrets", + "model-router-provider-routed-inference-vitest Vitest step must receive NVIDIA_API_KEY from secrets", ); } requireRunContains( @@ -5208,6 +5212,35 @@ function validateModelRouterProviderRoutedInferenceVitestJob( requireRunContains(errors, cleanup, 'rm -rf "${DOCKER_CONFIG}"'); } +function validateGatewayDriftPreflightVitestJob( + errors: string[], + jobs: WorkflowRecord, +): void { + const jobName = "gateway-drift-preflight-vitest"; + const job = asRecord(jobs[jobName]); + validateFreeStandingJobSelector( + errors, + jobs, + jobName, + "gateway-drift-preflight", + ); + if (Object.keys(job).length === 0) return; + + const runVitest = requireJobStep( + errors, + jobName, + asSteps(job.steps), + "Run gateway drift preflight Vitest test", + ); + requireRunContains(errors, runVitest, "npx vitest run --project integration"); + requireRunContains( + errors, + runVitest, + "test/gateway-drift-preflight.test.ts", + ); + requireRunDoesNotContain(errors, runVitest, "--project cli"); +} + function runContainsCloudflaredAptInstall(run: string): boolean { return /apt-get\s+install[\s\S]*cloudflared|apt\s+install[\s\S]*cloudflared|pkg\.cloudflare\.com\/cloudflared/.test( run, @@ -7805,12 +7838,7 @@ export function validateE2eVitestScenariosWorkflowBoundary( validateModelRouterProviderRoutedInferenceVitestJob(errors, jobs); validateSnapshotCommandsVitestJob(errors, jobs); validateSparkInstallVitestJob(errors, jobs); - validateFreeStandingJobSelector( - errors, - jobs, - "gateway-drift-preflight-vitest", - "gateway-drift-preflight", - ); + validateGatewayDriftPreflightVitestJob(errors, jobs); validateFreeStandingJobSelector( errors, From 3c52297a3f74bfb4aa1dca75868af5e09f40cec8 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sat, 27 Jun 2026 18:06:54 -0700 Subject: [PATCH 2/7] fix(e2e): stabilize late full-gate checks Signed-off-by: Carlos Villela --- ci/test-file-size-budget.json | 2 +- .../nemotron-3-ultra-managed-inference.json | 17 +++++++++++ .../live/hermes-inference-switch-helpers.ts | 18 +++++++----- test/e2e-scenario/live/network-policy.test.ts | 12 ++++++-- ...mes-inference-switch-command-shape.test.ts | 18 ++++++++++++ test/e2e/test-network-policy.sh | 2 +- test/generate-openclaw-config.test.ts | 28 +++++++++---------- 7 files changed, 71 insertions(+), 26 deletions(-) create mode 100644 nemoclaw-blueprint/model-specific-setup/openclaw/nemotron-3-ultra-managed-inference.json create mode 100644 test/e2e-scenario/support-tests/hermes-inference-switch-command-shape.test.ts diff --git a/ci/test-file-size-budget.json b/ci/test-file-size-budget.json index cb79f8cb7ef..ee67851ca93 100644 --- a/ci/test-file-size-budget.json +++ b/ci/test-file-size-budget.json @@ -6,7 +6,7 @@ "src/lib/inference/nim.test.ts": 2068, "src/lib/onboard/preflight.test.ts": 1904, "test/channels-add-preset.test.ts": 1871, - "test/generate-openclaw-config.test.ts": 1984, + "test/generate-openclaw-config.test.ts": 1982, "test/install-preflight.test.ts": 4006, "test/nemoclaw-start.test.ts": 5043, "test/onboard-messaging.test.ts": 2062, diff --git a/nemoclaw-blueprint/model-specific-setup/openclaw/nemotron-3-ultra-managed-inference.json b/nemoclaw-blueprint/model-specific-setup/openclaw/nemotron-3-ultra-managed-inference.json new file mode 100644 index 00000000000..1f1397c80f9 --- /dev/null +++ b/nemoclaw-blueprint/model-specific-setup/openclaw/nemotron-3-ultra-managed-inference.json @@ -0,0 +1,17 @@ +{ + "$schema": "../schema.json", + "id": "nemotron-3-ultra-managed-inference", + "agent": "openclaw", + "description": "Disables OpenClaw's native code-based tool search for hosted Nemotron 3 Ultra on the NemoClaw managed inference.local route. The model can emit invalid JavaScript for the tool_search_code surface and return '[tools] tool_search_code failed' instead of completing real tool calls; routing it back to the structured tool-calling surface preserves tool use.", + "match": { + "modelIds": ["nvidia/nvidia/nemotron-3-ultra"], + "providerKey": "inference", + "inferenceApi": "openai-completions", + "baseUrl": "https://inference.local/v1" + }, + "effects": { + "openclawTools": { + "toolSearch": false + } + } +} diff --git a/test/e2e-scenario/live/hermes-inference-switch-helpers.ts b/test/e2e-scenario/live/hermes-inference-switch-helpers.ts index 4b92ee6d7ce..2c4401f298e 100644 --- a/test/e2e-scenario/live/hermes-inference-switch-helpers.ts +++ b/test/e2e-scenario/live/hermes-inference-switch-helpers.ts @@ -347,14 +347,18 @@ export function expectedApiMode(): string | undefined { ]).get(SWITCH_API); } +export const API_KEY_SHAPE_PATTERN = `^[[:space:]]+api_key:[[:space:]]*["']?sk-[^"'[:space:]]+`; + +export function apiKeyShapeCommand(): string[] { + return ["grep", "-Eq", API_KEY_SHAPE_PATTERN, "/sandbox/.hermes/config.yaml"]; +} + export async function apiKeyShape(sandbox: SandboxClient): Promise { - return await sandbox.execShell( - SANDBOX_NAME, - trustedSandboxShellScript( - "python3 - <<'PY'\nimport re\ntext=open('/sandbox/.hermes/config.yaml', encoding='utf-8').read()\nmatch=re.search(r'^\\s+api_key:\\s*[\\\"\\']?(sk-[^\\\"\\'\\s]+)', text, re.M)\nraise SystemExit(0 if match else 1)\nPY", - ), - { artifactName: "hermes-config-api-key-shape", env: env(), timeoutMs: 30_000 }, - ); + return await sandbox.exec(SANDBOX_NAME, apiKeyShapeCommand(), { + artifactName: "hermes-config-api-key-shape", + env: env(), + timeoutMs: 30_000, + }); } export async function hashCheck( diff --git a/test/e2e-scenario/live/network-policy.test.ts b/test/e2e-scenario/live/network-policy.test.ts index fde7dfa5493..b19481708f4 100644 --- a/test/e2e-scenario/live/network-policy.test.ts +++ b/test/e2e-scenario/live/network-policy.test.ts @@ -560,11 +560,19 @@ hello curlStatus(sandbox, "https://pypi.org/simple/le/", "tc-net-02-pypi-post", "-X POST"), ).resolves.toBe("403"); - const slackBefore = await fetchStatus(sandbox, "https://slack.com/", "tc-net-03-slack-before"); + const slackBefore = await fetchStatus( + sandbox, + "https://slack.com/api/api.test", + "tc-net-03-slack-before", + ); expect(slackBefore).toMatch(/STATUS_403|ERROR_/); const slackApply = await applyPresetInteractively(host, "slack"); expect(slackApply.exitCode, text(slackApply)).toBe(0); - const slackAfter = await fetchStatus(sandbox, "https://slack.com/", "tc-net-03-slack-after"); + const slackAfter = await fetchStatus( + sandbox, + "https://slack.com/api/api.test", + "tc-net-03-slack-after", + ); expect(slackAfter).toMatch(/STATUS_200/); const atlassianBefore = await fetchStatus( diff --git a/test/e2e-scenario/support-tests/hermes-inference-switch-command-shape.test.ts b/test/e2e-scenario/support-tests/hermes-inference-switch-command-shape.test.ts new file mode 100644 index 00000000000..f88bc44085a --- /dev/null +++ b/test/e2e-scenario/support-tests/hermes-inference-switch-command-shape.test.ts @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + API_KEY_SHAPE_PATTERN, + apiKeyShapeCommand, +} from "../live/hermes-inference-switch-helpers.ts"; + +describe("Hermes inference switch command shape", () => { + it("uses direct single-line argv for the in-sandbox API-key probe", () => { + const command = apiKeyShapeCommand(); + + expect(command).toEqual(["grep", "-Eq", API_KEY_SHAPE_PATTERN, "/sandbox/.hermes/config.yaml"]); + expect(command.every((argument) => !/[\r\n]/u.test(argument))).toBe(true); + }); +}); diff --git a/test/e2e/test-network-policy.sh b/test/e2e/test-network-policy.sh index 41d5358e56d..2ee575885a1 100755 --- a/test/e2e/test-network-policy.sh +++ b/test/e2e/test-network-policy.sh @@ -467,7 +467,7 @@ bash /tmp/nemoclaw-brew-e2e.sh" "$PACKAGE_MANAGER_SANDBOX_TIMEOUT_SECONDS" 2>&1) test_net_03_live_policy_add() { log "=== TC-NET-03: Live Policy-Add Without Restart ===" - local target_url="https://slack.com/" + local target_url="https://slack.com/api/api.test" log " Step 1: Verify slack.com is blocked before policy-add..." local before diff --git a/test/generate-openclaw-config.test.ts b/test/generate-openclaw-config.test.ts index f518a6e2794..afe1ca8f8c8 100644 --- a/test/generate-openclaw-config.test.ts +++ b/test/generate-openclaw-config.test.ts @@ -1375,23 +1375,21 @@ describe("generate-openclaw-config.mts: config generation", () => { } }, 20_000); - // #4780: Nemotron generates invalid JS for OpenClaw's native code-based tool - // search (`tool_search_code`): CommonJS `require`, `openclaw.tools.search` - // called with an object instead of a string, `tool_describe`/`tool_call` - // invoked with bad ids. The run still succeeds via fallback, but the logs are - // flooded with `[tools] tool_search_code failed` errors. Disabling native - // tool search for this managed-inference route routes the model back to the - // structured tool-calling surface it handles correctly. + // #4780: Nemotron can generate invalid JS for OpenClaw's native + // `tool_search_code`. Disable it on affected managed-inference routes so + // these models use the structured tool-calling surface they handle correctly. it("disables native OpenClaw Tool Search for Nemotron managed inference (#4780)", () => { - const config = runConfigScript({ - NEMOCLAW_MODEL: "nvidia/nemotron-3-super-120b-a12b", - NEMOCLAW_PROVIDER_KEY: "inference", - NEMOCLAW_PRIMARY_MODEL_REF: "inference/nvidia/nemotron-3-super-120b-a12b", - NEMOCLAW_INFERENCE_BASE_URL: "https://inference.local/v1", - NEMOCLAW_INFERENCE_API: "openai-completions", - }); + for (const model of ["nvidia/nemotron-3-super-120b-a12b", "nvidia/nvidia/nemotron-3-ultra"]) { + const config = runConfigScript({ + NEMOCLAW_MODEL: model, + NEMOCLAW_PROVIDER_KEY: "inference", + NEMOCLAW_PRIMARY_MODEL_REF: `inference/${model}`, + NEMOCLAW_INFERENCE_BASE_URL: "https://inference.local/v1", + NEMOCLAW_INFERENCE_API: "openai-completions", + }); - expect(config.tools?.toolSearch).toBe(false); + expect(config.tools?.toolSearch, model).toBe(false); + } }); it("does not disable native Tool Search for Nemotron on non-matching routes (#4780)", () => { From 6236b5da0338dea13b8845bca0b67dd08db2c4b8 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sat, 27 Jun 2026 18:14:48 -0700 Subject: [PATCH 3/7] test(e2e): tighten Hermes key-shape probe Signed-off-by: Carlos Villela --- .../live/hermes-inference-switch-helpers.ts | 2 +- ...mes-inference-switch-command-shape.test.ts | 27 +++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/test/e2e-scenario/live/hermes-inference-switch-helpers.ts b/test/e2e-scenario/live/hermes-inference-switch-helpers.ts index 2c4401f298e..663e62bbb9b 100644 --- a/test/e2e-scenario/live/hermes-inference-switch-helpers.ts +++ b/test/e2e-scenario/live/hermes-inference-switch-helpers.ts @@ -347,7 +347,7 @@ export function expectedApiMode(): string | undefined { ]).get(SWITCH_API); } -export const API_KEY_SHAPE_PATTERN = `^[[:space:]]+api_key:[[:space:]]*["']?sk-[^"'[:space:]]+`; +export const API_KEY_SHAPE_PATTERN = `^[[:space:]]*api_key:[[:space:]]*("sk-[^"[:space:]]+"|'sk-[^'[:space:]]+'|sk-[^"'[:space:]]+)[[:space:]]*$`; export function apiKeyShapeCommand(): string[] { return ["grep", "-Eq", API_KEY_SHAPE_PATTERN, "/sandbox/.hermes/config.yaml"]; diff --git a/test/e2e-scenario/support-tests/hermes-inference-switch-command-shape.test.ts b/test/e2e-scenario/support-tests/hermes-inference-switch-command-shape.test.ts index f88bc44085a..8423b0a4eaa 100644 --- a/test/e2e-scenario/support-tests/hermes-inference-switch-command-shape.test.ts +++ b/test/e2e-scenario/support-tests/hermes-inference-switch-command-shape.test.ts @@ -1,6 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { spawnSync } from "node:child_process"; + import { describe, expect, it } from "vitest"; import { @@ -9,10 +11,35 @@ import { } from "../live/hermes-inference-switch-helpers.ts"; describe("Hermes inference switch command shape", () => { + function matchesApiKeyShape(line: string): boolean { + return ( + spawnSync("grep", ["-Eq", API_KEY_SHAPE_PATTERN], { + encoding: "utf8", + input: `${line}\n`, + }).status === 0 + ); + } + it("uses direct single-line argv for the in-sandbox API-key probe", () => { const command = apiKeyShapeCommand(); expect(command).toEqual(["grep", "-Eq", API_KEY_SHAPE_PATTERN, "/sandbox/.hermes/config.yaml"]); expect(command.every((argument) => !/[\r\n]/u.test(argument))).toBe(true); }); + + it("accepts only complete sk-prefixed YAML scalars", () => { + expect( + [" api_key: sk-value", ' api_key: "sk-value"', " api_key: 'sk-value'"].every( + matchesApiKeyShape, + ), + ).toBe(true); + expect( + [ + " api_key: not-sk-value", + " api_key: sk-value trailing", + ' api_key: "sk-value', + ' api_key: sk-value"', + ].some(matchesApiKeyShape), + ).toBe(false); + }); }); From 4121f2af9d2bc7ae0942a953547766af989941f6 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sat, 27 Jun 2026 18:41:42 -0700 Subject: [PATCH 4/7] fix(e2e): drive policy prompts interactively Signed-off-by: Carlos Villela --- .github/workflows/e2e-vitest-scenarios.yaml | 20 ++++++++ .../live/network-policy-interactive.ts | 50 +++++++++++++++++++ test/e2e-scenario/live/network-policy.test.ts | 46 +++++++++++------ .../network-policy-interactive.test.ts | 31 ++++++++++++ test/e2e-script-workflow.test.ts | 16 ++++++ tools/e2e-scenarios/workflow-boundary.mts | 22 ++++++++ 6 files changed, 170 insertions(+), 15 deletions(-) create mode 100644 test/e2e-scenario/live/network-policy-interactive.ts create mode 100644 test/e2e-scenario/support-tests/network-policy-interactive.test.ts diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index 0377aa958a3..3b02b778e58 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -1872,6 +1872,26 @@ jobs: with: persist-credentials: false + # Keep this privileged setup inline in trusted workflow YAML. This job + # executes a selected target ref, so it must not load a repo-local action + # from that ref with sudo privileges. + - name: Install network-policy host dependencies + shell: bash + run: | + set -euo pipefail + for attempt in 1 2 3; do + if sudo apt-get update; then + break + fi + if [ "$attempt" -eq 3 ]; then + echo "::error::apt-get update failed after 3 attempts." >&2 + exit 1 + fi + echo "::warning::apt-get update attempt ${attempt} failed; retrying." >&2 + sleep $((attempt * 5)) + done + sudo apt-get install -y --no-install-recommends expect + - name: Set up Node uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 with: diff --git a/test/e2e-scenario/live/network-policy-interactive.ts b/test/e2e-scenario/live/network-policy-interactive.ts new file mode 100644 index 00000000000..df289d89870 --- /dev/null +++ b/test/e2e-scenario/live/network-policy-interactive.ts @@ -0,0 +1,50 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export const POLICY_ADD_EXPECT_SCRIPT = String.raw` +set timeout 60 +spawn env NEMOCLAW_NON_INTERACTIVE= node $env(NEMOCLAW_E2E_CLI) $env(NEMOCLAW_E2E_SANDBOX) policy-add +expect { + -glob "*Choose preset*" { + send -- "$env(NEMOCLAW_E2E_PRESET_NUM)\r" + } + timeout { + puts stderr "timed out waiting for the policy preset prompt" + exit 2 + } + eof { + puts stderr "policy-add exited before the policy preset prompt" + exit 3 + } +} +expect { + -glob "*Y/n*" { + send -- "Y\r" + } + timeout { + puts stderr "timed out waiting for the policy confirmation prompt" + exit 4 + } + eof { + puts stderr "policy-add exited before the policy confirmation prompt" + exit 5 + } +} +expect { + eof {} + timeout { + puts stderr "policy-add did not exit after confirmation" + exit 6 + } +} +set wait_result [wait] +exit [lindex $wait_result 3] +`; + +export function findPolicyPresetNumber(output: string, preset: string): string | null { + const escapedPreset = preset.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + const match = new RegExp(`^\\s*(\\d+)\\)\\s+(?:[●○]\\s+)?${escapedPreset}(?:\\s|$)`, "m").exec( + output, + ); + return match?.[1] ?? null; +} diff --git a/test/e2e-scenario/live/network-policy.test.ts b/test/e2e-scenario/live/network-policy.test.ts index b19481708f4..5eb53572e68 100644 --- a/test/e2e-scenario/live/network-policy.test.ts +++ b/test/e2e-scenario/live/network-policy.test.ts @@ -6,9 +6,8 @@ * * This keeps the legacy contract real: onboarding a restricted OpenClaw * sandbox, mutating live OpenShell network policy from the NemoClaw CLI, and - * probing egress from inside the sandbox. Helpers stay local to this file so - * the security/policy anchor does not add a new framework or shared fixture - * before repeated migration needs prove one is warranted. + * probing egress from inside the sandbox. The prompt-driving helper is kept + * separate so support tests can pin its command shape without live infra. */ import fs from "node:fs"; @@ -23,6 +22,7 @@ import { type SandboxClient, trustedSandboxShellScript } from "../fixtures/clien import { expect, test } from "../fixtures/e2e-test.ts"; import { shouldRunLiveE2EScenarios } from "../fixtures/live-project-gate.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; +import { findPolicyPresetNumber, POLICY_ADD_EXPECT_SCRIPT } from "./network-policy-interactive.ts"; import { isTransientProviderValidationFailure } from "./network-policy-transient-provider.ts"; const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); @@ -110,22 +110,34 @@ async function applyPresetInteractively( host: HostCliClient, preset: string, ): Promise { - const script = String.raw` -set -euo pipefail -preset_list="$(env NEMOCLAW_NON_INTERACTIVE= node "$NEMOCLAW_E2E_CLI" "$NEMOCLAW_E2E_SANDBOX" policy-add &1 || true)" -preset_num="$(printf '%s\n' "$preset_list" | python3 -c 'import re,sys; preset=sys.argv[1]; text=sys.stdin.read(); m=re.search(r"(?m)^\s*(\d+)\).*" + re.escape(preset), text); print(m.group(1) if m else "")' "$NEMOCLAW_E2E_PRESET")" -if [ -z "$preset_num" ]; then - printf 'preset %s not found in list:\n%s\n' "$NEMOCLAW_E2E_PRESET" "$preset_list" >&2 - exit 1 -fi -printf '%s\nY\n' "$preset_num" | env NEMOCLAW_NON_INTERACTIVE= node "$NEMOCLAW_E2E_CLI" "$NEMOCLAW_E2E_SANDBOX" policy-add -`; - const result = await host.command("bash", ["-lc", script], { + const listResult = await host.command( + "bash", + [ + "-lc", + 'env NEMOCLAW_NON_INTERACTIVE= node "$NEMOCLAW_E2E_CLI" "$NEMOCLAW_E2E_SANDBOX" policy-add { + it("selects the exact requested preset from the interactive list", () => { + const output = ` + 14) ○ hermes-slack — unrelated prefix + 15) ○ slack — Slack API access + 16) ● pypi — Python Package Index +`; + + expect(findPolicyPresetNumber(output, "slack")).toBe("15"); + expect(findPolicyPresetNumber(output, "pypi")).toBe("16"); + expect(findPolicyPresetNumber(output, "missing")).toBeNull(); + }); + + it("waits for each prompt before sending the corresponding response", () => { + expect(POLICY_ADD_EXPECT_SCRIPT).toContain('-glob "*Choose preset*"'); + expect(POLICY_ADD_EXPECT_SCRIPT).toContain('send -- "$env(NEMOCLAW_E2E_PRESET_NUM)\\r"'); + expect(POLICY_ADD_EXPECT_SCRIPT).toContain('-glob "*Y/n*"'); + expect(POLICY_ADD_EXPECT_SCRIPT).toContain('send -- "Y\\r"'); + expect(POLICY_ADD_EXPECT_SCRIPT).not.toMatch(/printf.*Y/); + }); +}); diff --git a/test/e2e-script-workflow.test.ts b/test/e2e-script-workflow.test.ts index a2160c23dc8..874eabf91e2 100644 --- a/test/e2e-script-workflow.test.ts +++ b/test/e2e-script-workflow.test.ts @@ -1052,6 +1052,14 @@ describe("E2E reusable workflow contract", () => { (step) => step.name === "Install issue #4434 host dependencies", ); const issue4434VitestInstallRun = issue4434VitestInstallStep?.run ?? ""; + const networkPolicyVitestSteps = + vitestScenarioWorkflow.jobs["network-policy-vitest"].steps ?? []; + const networkPolicyVitestStepIndex = (name: string) => + networkPolicyVitestSteps.findIndex((step) => step.name === name); + const networkPolicyVitestInstallStep = networkPolicyVitestSteps.find( + (step) => step.name === "Install network-policy host dependencies", + ); + const networkPolicyVitestInstallRun = networkPolicyVitestInstallStep?.run ?? ""; const installActionRun = installActionStep?.run ?? ""; expect(issue4434VitestInstallStep?.uses).toBeUndefined(); @@ -1061,6 +1069,13 @@ describe("E2E reusable workflow contract", () => { expect(issue4434VitestStepIndex("Install issue #4434 host dependencies")).toBeLessThan( issue4434VitestStepIndex("Authenticate to Docker Hub"), ); + expect(networkPolicyVitestInstallStep?.uses).toBeUndefined(); + expect(networkPolicyVitestInstallRun).toContain( + "sudo apt-get install -y --no-install-recommends expect", + ); + expect(networkPolicyVitestStepIndex("Install network-policy host dependencies")).toBeLessThan( + networkPolicyVitestStepIndex("Run network-policy live test"), + ); expect(installActionStep?.env?.APT_PACKAGES).toBe("${{ inputs.packages }}"); expect(installActionStep?.run).toContain('read -r -a packages <<< "$APT_PACKAGES"'); @@ -1082,6 +1097,7 @@ describe("E2E reusable workflow contract", () => { "sudo apt-get install -y --no-install-recommends", ]) { expect(issue4434VitestInstallRun, fragment).toContain(fragment); + expect(networkPolicyVitestInstallRun, fragment).toContain(fragment); expect(installActionRun, fragment).toContain(fragment); } }); diff --git a/tools/e2e-scenarios/workflow-boundary.mts b/tools/e2e-scenarios/workflow-boundary.mts index 508e8e32a55..999c5e398c7 100644 --- a/tools/e2e-scenarios/workflow-boundary.mts +++ b/tools/e2e-scenarios/workflow-boundary.mts @@ -1034,6 +1034,28 @@ function validateNetworkPolicyVitestJob( ); } + const installHostDependencies = requireJobStep( + errors, + jobName, + steps, + "Install network-policy host dependencies", + ); + if (installHostDependencies?.uses) { + errors.push( + "network-policy-vitest host dependency setup must stay inline in trusted workflow YAML", + ); + } + for (const fragment of [ + "for attempt in 1 2 3", + "sudo apt-get update", + 'if [ "$attempt" -eq 3 ]; then', + "apt-get update failed after 3 attempts", + "sleep $((attempt * 5))", + "sudo apt-get install -y --no-install-recommends expect", + ]) { + requireRunContains(errors, installHostDependencies, fragment); + } + const setupNode = namedStep(steps, "Set up Node"); if (!setupNode) errors.push("network-policy-vitest job missing step: Set up Node"); From ec035e42df627ee25260d95628ea4ae7a4395de5 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sat, 27 Jun 2026 18:44:57 -0700 Subject: [PATCH 5/7] test(e2e): keep policy test flow linear Signed-off-by: Carlos Villela --- test/e2e-scenario/live/network-policy-interactive.ts | 8 ++++++++ test/e2e-scenario/live/network-policy.test.ts | 12 +++++------- .../support-tests/network-policy-interactive.test.ts | 2 ++ 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/test/e2e-scenario/live/network-policy-interactive.ts b/test/e2e-scenario/live/network-policy-interactive.ts index df289d89870..3b63fe884b2 100644 --- a/test/e2e-scenario/live/network-policy-interactive.ts +++ b/test/e2e-scenario/live/network-policy-interactive.ts @@ -48,3 +48,11 @@ export function findPolicyPresetNumber(output: string, preset: string): string | ); return match?.[1] ?? null; } + +export function requirePolicyPresetNumber(output: string, preset: string): string { + const presetNumber = findPolicyPresetNumber(output, preset); + if (!presetNumber) { + throw new Error(`preset ${preset} not found in interactive policy-add list: ${output}`); + } + return presetNumber; +} diff --git a/test/e2e-scenario/live/network-policy.test.ts b/test/e2e-scenario/live/network-policy.test.ts index 5eb53572e68..d731a5a3548 100644 --- a/test/e2e-scenario/live/network-policy.test.ts +++ b/test/e2e-scenario/live/network-policy.test.ts @@ -22,7 +22,10 @@ import { type SandboxClient, trustedSandboxShellScript } from "../fixtures/clien import { expect, test } from "../fixtures/e2e-test.ts"; import { shouldRunLiveE2EScenarios } from "../fixtures/live-project-gate.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; -import { findPolicyPresetNumber, POLICY_ADD_EXPECT_SCRIPT } from "./network-policy-interactive.ts"; +import { + POLICY_ADD_EXPECT_SCRIPT, + requirePolicyPresetNumber, +} from "./network-policy-interactive.ts"; import { isTransientProviderValidationFailure } from "./network-policy-transient-provider.ts"; const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); @@ -125,12 +128,7 @@ async function applyPresetInteractively( timeoutMs: SANDBOX_EXEC_TIMEOUT_MS, }, ); - const presetNumber = findPolicyPresetNumber(text(listResult), preset); - if (!presetNumber) { - throw new Error( - `preset ${preset} not found in interactive policy-add list: ${text(listResult)}`, - ); - } + const presetNumber = requirePolicyPresetNumber(text(listResult), preset); const result = await host.command("expect", ["-c", POLICY_ADD_EXPECT_SCRIPT], { artifactName: `policy-add-${preset}-interactive`, diff --git a/test/e2e-scenario/support-tests/network-policy-interactive.test.ts b/test/e2e-scenario/support-tests/network-policy-interactive.test.ts index b8e03b1ce0f..661bafaed9d 100644 --- a/test/e2e-scenario/support-tests/network-policy-interactive.test.ts +++ b/test/e2e-scenario/support-tests/network-policy-interactive.test.ts @@ -6,6 +6,7 @@ import { describe, expect, it } from "vitest"; import { findPolicyPresetNumber, POLICY_ADD_EXPECT_SCRIPT, + requirePolicyPresetNumber, } from "../live/network-policy-interactive.ts"; describe("network-policy interactive preset harness", () => { @@ -19,6 +20,7 @@ describe("network-policy interactive preset harness", () => { expect(findPolicyPresetNumber(output, "slack")).toBe("15"); expect(findPolicyPresetNumber(output, "pypi")).toBe("16"); expect(findPolicyPresetNumber(output, "missing")).toBeNull(); + expect(() => requirePolicyPresetNumber(output, "missing")).toThrow(/preset missing not found/); }); it("waits for each prompt before sending the corresponding response", () => { From d4a96b998230c8d7842aaa99557a4fe7332132e7 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sat, 27 Jun 2026 18:52:49 -0700 Subject: [PATCH 6/7] docs(e2e): clarify inference credential aliases Signed-off-by: Carlos Villela --- .../live/model-router-provider-routed-inference-helpers.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/e2e-scenario/live/model-router-provider-routed-inference-helpers.ts b/test/e2e-scenario/live/model-router-provider-routed-inference-helpers.ts index 9d063f13d11..e07245f24df 100644 --- a/test/e2e-scenario/live/model-router-provider-routed-inference-helpers.ts +++ b/test/e2e-scenario/live/model-router-provider-routed-inference-helpers.ts @@ -24,6 +24,11 @@ export function buildProviderRoutedEnv( ): NodeJS.ProcessEnv { return { ...buildAvailabilityProbeEnv(baseEnv), + // CI's NVIDIA_API_KEY is the public nvapi-* credential for + // integrate.api.nvidia.com. The routed blueprint still declares the + // historical NVIDIA_INFERENCE_API_KEY runtime credential name, so alias + // the public value only in this child environment. Hosted lanes instead + // source their sk-* NVIDIA_INFERENCE_API_KEY for inference-api.nvidia.com. NVIDIA_INFERENCE_API_KEY: apiKey, NEMOCLAW_PROVIDER_KEY: apiKey, NEMOCLAW_SANDBOX_NAME: sandboxName, From 27766b926f1b0019e9478beb740dd81b8c6a76b4 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sat, 27 Jun 2026 19:05:11 -0700 Subject: [PATCH 7/7] docs(e2e): document live test trust boundaries Signed-off-by: Carlos Villela --- .github/workflows/e2e-vitest-scenarios.yaml | 7 ++++--- test/e2e-scenario/live/hermes-inference-switch-helpers.ts | 2 ++ test/e2e-scenario/live/network-policy-interactive.ts | 5 +++++ test/e2e-scenario/live/network-policy.test.ts | 2 ++ test/e2e/test-network-policy.sh | 2 ++ test/generate-openclaw-config.test.ts | 4 ++-- 6 files changed, 17 insertions(+), 5 deletions(-) diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index 3b02b778e58..bd32b225b4a 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -1872,9 +1872,10 @@ jobs: with: persist-credentials: false - # Keep this privileged setup inline in trusted workflow YAML. This job - # executes a selected target ref, so it must not load a repo-local action - # from that ref with sudo privileges. + # Expect is a reviewed host-tool consumer for the interactive policy-add + # test. Keep this privileged setup inline in trusted workflow YAML. This + # job executes a selected target ref, so it must not load a repo-local + # action from that ref with sudo privileges. - name: Install network-policy host dependencies shell: bash run: | diff --git a/test/e2e-scenario/live/hermes-inference-switch-helpers.ts b/test/e2e-scenario/live/hermes-inference-switch-helpers.ts index 663e62bbb9b..3294bb5f9d5 100644 --- a/test/e2e-scenario/live/hermes-inference-switch-helpers.ts +++ b/test/e2e-scenario/live/hermes-inference-switch-helpers.ts @@ -347,6 +347,8 @@ export function expectedApiMode(): string | undefined { ]).get(SWITCH_API); } +// This live lane runs on ubuntu-latest and intentionally uses GNU grep's +// POSIX ERE character classes; support tests pin the accepted scalar shapes. export const API_KEY_SHAPE_PATTERN = `^[[:space:]]*api_key:[[:space:]]*("sk-[^"[:space:]]+"|'sk-[^'[:space:]]+'|sk-[^"'[:space:]]+)[[:space:]]*$`; export function apiKeyShapeCommand(): string[] { diff --git a/test/e2e-scenario/live/network-policy-interactive.ts b/test/e2e-scenario/live/network-policy-interactive.ts index 3b63fe884b2..f057e12c95f 100644 --- a/test/e2e-scenario/live/network-policy-interactive.ts +++ b/test/e2e-scenario/live/network-policy-interactive.ts @@ -1,6 +1,11 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +// Trust boundary: the Expect program receives only a numeric preset index +// parsed from NemoClaw's own numbered menu plus the literal confirmation Y. +// No dispatch input, secret, or other user-controlled text enters the script. +// Exit codes: 2=preset timeout, 3=preset EOF, 4=confirmation timeout, +// 5=confirmation EOF, and 6=post-confirmation timeout. export const POLICY_ADD_EXPECT_SCRIPT = String.raw` set timeout 60 spawn env NEMOCLAW_NON_INTERACTIVE= node $env(NEMOCLAW_E2E_CLI) $env(NEMOCLAW_E2E_SANDBOX) policy-add diff --git a/test/e2e-scenario/live/network-policy.test.ts b/test/e2e-scenario/live/network-policy.test.ts index d731a5a3548..fde18970411 100644 --- a/test/e2e-scenario/live/network-policy.test.ts +++ b/test/e2e-scenario/live/network-policy.test.ts @@ -570,6 +570,8 @@ hello curlStatus(sandbox, "https://pypi.org/simple/le/", "tc-net-02-pypi-post", "-X POST"), ).resolves.toBe("403"); + // Use Slack's non-redirecting API probe on the preset's actual API host; + // the marketing root can leave the slack.com allowlist during redirects. const slackBefore = await fetchStatus( sandbox, "https://slack.com/api/api.test", diff --git a/test/e2e/test-network-policy.sh b/test/e2e/test-network-policy.sh index 2ee575885a1..3f8f32391b7 100755 --- a/test/e2e/test-network-policy.sh +++ b/test/e2e/test-network-policy.sh @@ -467,6 +467,8 @@ bash /tmp/nemoclaw-brew-e2e.sh" "$PACKAGE_MANAGER_SANDBOX_TIMEOUT_SECONDS" 2>&1) test_net_03_live_policy_add() { log "=== TC-NET-03: Live Policy-Add Without Restart ===" + # Probe Slack's non-redirecting API path. The marketing root can redirect + # outside the slack.com allowlist and does not isolate preset behavior. local target_url="https://slack.com/api/api.test" log " Step 1: Verify slack.com is blocked before policy-add..." diff --git a/test/generate-openclaw-config.test.ts b/test/generate-openclaw-config.test.ts index afe1ca8f8c8..28b47b7e3ca 100644 --- a/test/generate-openclaw-config.test.ts +++ b/test/generate-openclaw-config.test.ts @@ -1376,8 +1376,8 @@ describe("generate-openclaw-config.mts: config generation", () => { }, 20_000); // #4780: Nemotron can generate invalid JS for OpenClaw's native - // `tool_search_code`. Disable it on affected managed-inference routes so - // these models use the structured tool-calling surface they handle correctly. + // `tool_search_code`. The Super and Ultra managed-inference manifests disable + // it so both models use the structured tool-calling surface they handle. it("disables native OpenClaw Tool Search for Nemotron managed inference (#4780)", () => { for (const model of ["nvidia/nemotron-3-super-120b-a12b", "nvidia/nvidia/nemotron-3-ultra"]) { const config = runConfigScript({