Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 25 additions & 2 deletions .github/workflows/e2e-vitest-scenarios.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -1872,6 +1872,27 @@ jobs:
with:
persist-credentials: false

# Expect is a reviewed host-tool consumer for the interactive policy-add
# test. Keep this privileged setup inline in trusted workflow YAML. This
# job executes a selected target ref, so it must not load a repo-local
# action from that ref with sudo privileges.
- name: Install network-policy host dependencies
shell: bash
run: |
set -euo pipefail
for attempt in 1 2 3; do
if sudo apt-get update; then
break
fi
if [ "$attempt" -eq 3 ]; then
echo "::error::apt-get update failed after 3 attempts." >&2
exit 1
fi
echo "::warning::apt-get update attempt ${attempt} failed; retrying." >&2
sleep $((attempt * 5))
done
sudo apt-get install -y --no-install-recommends expect

- name: Set up Node
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0
with:
Expand Down Expand Up @@ -3958,7 +3979,7 @@ jobs:
# sandbox inference.local completion boundaries without adding registry
# or migration-ledger wiring.
env:
NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }}
NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
run: |
set -euo pipefail
npx vitest run --project e2e-scenarios-live \
Expand Down Expand Up @@ -4252,7 +4273,9 @@ jobs:
# Docker/OpenShell mutation.
run: |
set -euo pipefail
npx vitest run --project cli test/gateway-drift-preflight.test.ts --silent=false --reporter=default
npx vitest run --project integration \
test/gateway-drift-preflight.test.ts \
--silent=false --reporter=default

- name: Upload gateway drift preflight artifacts
if: always()
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/regression-e2e.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -248,7 +248,7 @@ jobs:

- name: Run Model Router provider-routed inference E2E test
env:
NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }}
NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
NEMOCLAW_NON_INTERACTIVE: "1"
NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
run: bash test/e2e/test-model-router-provider-routed-inference.sh
Expand Down
2 changes: 1 addition & 1 deletion ci/test-file-size-budget.json
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
"src/lib/inference/nim.test.ts": 2068,
"src/lib/onboard/preflight.test.ts": 1904,
"test/channels-add-preset.test.ts": 1871,
"test/generate-openclaw-config.test.ts": 1984,
"test/generate-openclaw-config.test.ts": 1982,
"test/install-preflight.test.ts": 4006,
"test/nemoclaw-start.test.ts": 5043,
"test/onboard-messaging.test.ts": 2062,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
{
"$schema": "../schema.json",
"id": "nemotron-3-ultra-managed-inference",
"agent": "openclaw",
"description": "Disables OpenClaw's native code-based tool search for hosted Nemotron 3 Ultra on the NemoClaw managed inference.local route. The model can emit invalid JavaScript for the tool_search_code surface and return '[tools] tool_search_code failed' instead of completing real tool calls; routing it back to the structured tool-calling surface preserves tool use.",
"match": {
"modelIds": ["nvidia/nvidia/nemotron-3-ultra"],
"providerKey": "inference",
"inferenceApi": "openai-completions",
"baseUrl": "https://inference.local/v1"
},
"effects": {
"openclawTools": {
"toolSearch": false
}
}
}
20 changes: 13 additions & 7 deletions test/e2e-scenario/live/hermes-inference-switch-helpers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -347,14 +347,20 @@ export function expectedApiMode(): string | undefined {
]).get(SWITCH_API);
}

// This live lane runs on ubuntu-latest and intentionally uses GNU grep's
// POSIX ERE character classes; support tests pin the accepted scalar shapes.
export const API_KEY_SHAPE_PATTERN = `^[[:space:]]*api_key:[[:space:]]*("sk-[^"[:space:]]+"|'sk-[^'[:space:]]+'|sk-[^"'[:space:]]+)[[:space:]]*$`;

export function apiKeyShapeCommand(): string[] {
return ["grep", "-Eq", API_KEY_SHAPE_PATTERN, "/sandbox/.hermes/config.yaml"];
}

export async function apiKeyShape(sandbox: SandboxClient): Promise<ShellProbeResult> {
return await sandbox.execShell(
SANDBOX_NAME,
trustedSandboxShellScript(
"python3 - <<'PY'\nimport re\ntext=open('/sandbox/.hermes/config.yaml', encoding='utf-8').read()\nmatch=re.search(r'^\\s+api_key:\\s*[\\\"\\']?(sk-[^\\\"\\'\\s]+)', text, re.M)\nraise SystemExit(0 if match else 1)\nPY",
),
{ artifactName: "hermes-config-api-key-shape", env: env(), timeoutMs: 30_000 },
);
return await sandbox.exec(SANDBOX_NAME, apiKeyShapeCommand(), {
artifactName: "hermes-config-api-key-shape",
env: env(),
timeoutMs: 30_000,
});
}

export async function hashCheck(
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts";

export const MODEL_ROUTER_PUBLIC_KEY_ENV = "NVIDIA_API_KEY";

export interface ModelRouterSecrets {
required(name: string): string;
}

export function requireModelRouterPublicKey(secrets: ModelRouterSecrets): string {
const apiKey = secrets.required(MODEL_ROUTER_PUBLIC_KEY_ENV);
if (!apiKey.startsWith("nvapi-")) {
throw new Error("NVIDIA_API_KEY must be a public NVIDIA Endpoints nvapi-* key");
}
return apiKey;
}

export function buildProviderRoutedEnv(
apiKey: string,
sandboxName: string,
baseEnv: NodeJS.ProcessEnv = process.env,
): NodeJS.ProcessEnv {
return {
...buildAvailabilityProbeEnv(baseEnv),
// CI's NVIDIA_API_KEY is the public nvapi-* credential for
// integrate.api.nvidia.com. The routed blueprint still declares the
// historical NVIDIA_INFERENCE_API_KEY runtime credential name, so alias
// the public value only in this child environment. Hosted lanes instead
// source their sk-* NVIDIA_INFERENCE_API_KEY for inference-api.nvidia.com.
NVIDIA_INFERENCE_API_KEY: apiKey,
NEMOCLAW_PROVIDER_KEY: apiKey,
NEMOCLAW_SANDBOX_NAME: sandboxName,
NEMOCLAW_NON_INTERACTIVE: "1",
NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1",
NEMOCLAW_POLICY_TIER: "open",
NEMOCLAW_PROVIDER: "routed",
};
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,10 @@ import path from "node:path";
import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts";
import { expect, test } from "../fixtures/e2e-test.ts";
import { shouldRunLiveE2EScenarios } from "../fixtures/live-project-gate.ts";
import {
buildProviderRoutedEnv,
requireModelRouterPublicKey,
} from "./model-router-provider-routed-inference-helpers.ts";

// Focused Vitest live replacement for
// test/e2e/test-model-router-provider-routed-inference.sh. Keep this as a
Expand Down Expand Up @@ -72,19 +76,6 @@ function routedPongReason(raw: string): "ok" | string {
return "ok";
}

function withProviderRoutedEnv(apiKey: string): NodeJS.ProcessEnv {
return {
...buildAvailabilityProbeEnv(),
NVIDIA_INFERENCE_API_KEY: apiKey,
NEMOCLAW_PROVIDER_KEY: apiKey,
NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME,
NEMOCLAW_NON_INTERACTIVE: "1",
NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1",
NEMOCLAW_POLICY_TIER: "open",
NEMOCLAW_PROVIDER: "routed",
};
}

test.skipIf(!shouldRunLiveE2EScenarios())(
"model-router provider-routed onboard returns routed inference.local PONG",
async ({ artifacts, cleanup, host, sandbox, secrets, skip }) => {
Expand All @@ -107,10 +98,7 @@ test.skipIf(!shouldRunLiveE2EScenarios())(
skip("Docker is required for provider-routed Model Router onboarding");
}

const apiKey = secrets.required("NVIDIA_INFERENCE_API_KEY");
expect(apiKey.startsWith("nvapi-"), "NVIDIA_INFERENCE_API_KEY must start with nvapi-").toBe(
true,
);
const apiKey = requireModelRouterPublicKey(secrets);

await artifacts.writeJson("scenario.json", {
id: "model-router-provider-routed-inference",
Expand All @@ -119,7 +107,7 @@ test.skipIf(!shouldRunLiveE2EScenarios())(
legacySource: "test/e2e/test-model-router-provider-routed-inference.sh",
contract: [
"Docker is available before onboarding",
"NVIDIA_INFERENCE_API_KEY is present and nvapi-prefixed",
"NVIDIA_API_KEY is present and nvapi-prefixed, then staged for the router's NVIDIA_INFERENCE_API_KEY credential",
"nemoclaw onboard --fresh completes with NEMOCLAW_PROVIDER=routed",
"host model-router health reports at least one healthy endpoint",
"sandbox inference.local returns model nvidia-routed with PONG content",
Expand Down Expand Up @@ -152,7 +140,7 @@ test.skipIf(!shouldRunLiveE2EScenarios())(
],
{
artifactName: "onboard-model-router-provider-routed",
env: withProviderRoutedEnv(apiKey),
env: buildProviderRoutedEnv(apiKey, SANDBOX_NAME),
redactionValues: [apiKey],
timeoutMs: ONBOARD_TIMEOUT_MS,
},
Expand Down
63 changes: 63 additions & 0 deletions test/e2e-scenario/live/network-policy-interactive.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

// Trust boundary: the Expect program receives only a numeric preset index
// parsed from NemoClaw's own numbered menu plus the literal confirmation Y.
// No dispatch input, secret, or other user-controlled text enters the script.
// Exit codes: 2=preset timeout, 3=preset EOF, 4=confirmation timeout,
// 5=confirmation EOF, and 6=post-confirmation timeout.
export const POLICY_ADD_EXPECT_SCRIPT = String.raw`
set timeout 60
spawn env NEMOCLAW_NON_INTERACTIVE= node $env(NEMOCLAW_E2E_CLI) $env(NEMOCLAW_E2E_SANDBOX) policy-add
expect {
-glob "*Choose preset*" {
send -- "$env(NEMOCLAW_E2E_PRESET_NUM)\r"
}
timeout {
puts stderr "timed out waiting for the policy preset prompt"
exit 2
}
eof {
puts stderr "policy-add exited before the policy preset prompt"
exit 3
}
}
expect {
-glob "*Y/n*" {
send -- "Y\r"
}
timeout {
puts stderr "timed out waiting for the policy confirmation prompt"
exit 4
}
eof {
puts stderr "policy-add exited before the policy confirmation prompt"
exit 5
}
}
expect {
eof {}
timeout {
puts stderr "policy-add did not exit after confirmation"
exit 6
}
}
set wait_result [wait]
exit [lindex $wait_result 3]
`;

export function findPolicyPresetNumber(output: string, preset: string): string | null {
const escapedPreset = preset.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
const match = new RegExp(`^\\s*(\\d+)\\)\\s+(?:[●○]\\s+)?${escapedPreset}(?:\\s|$)`, "m").exec(
output,
);
return match?.[1] ?? null;
}

export function requirePolicyPresetNumber(output: string, preset: string): string {
const presetNumber = findPolicyPresetNumber(output, preset);
if (!presetNumber) {
throw new Error(`preset ${preset} not found in interactive policy-add list: ${output}`);
}
return presetNumber;
}
58 changes: 41 additions & 17 deletions test/e2e-scenario/live/network-policy.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,8 @@
*
* This keeps the legacy contract real: onboarding a restricted OpenClaw
* sandbox, mutating live OpenShell network policy from the NemoClaw CLI, and
* probing egress from inside the sandbox. Helpers stay local to this file so
* the security/policy anchor does not add a new framework or shared fixture
* before repeated migration needs prove one is warranted.
* probing egress from inside the sandbox. The prompt-driving helper is kept
* separate so support tests can pin its command shape without live infra.
*/

import fs from "node:fs";
Expand All @@ -23,6 +22,10 @@ import { type SandboxClient, trustedSandboxShellScript } from "../fixtures/clien
import { expect, test } from "../fixtures/e2e-test.ts";
import { shouldRunLiveE2EScenarios } from "../fixtures/live-project-gate.ts";
import type { ShellProbeResult } from "../fixtures/shell-probe.ts";
import {
POLICY_ADD_EXPECT_SCRIPT,
requirePolicyPresetNumber,
} from "./network-policy-interactive.ts";
import { isTransientProviderValidationFailure } from "./network-policy-transient-provider.ts";

const REPO_ROOT = path.resolve(import.meta.dirname, "../../..");
Expand Down Expand Up @@ -110,22 +113,29 @@ async function applyPresetInteractively(
host: HostCliClient,
preset: string,
): Promise<ShellProbeResult> {
const script = String.raw`
set -euo pipefail
preset_list="$(env NEMOCLAW_NON_INTERACTIVE= node "$NEMOCLAW_E2E_CLI" "$NEMOCLAW_E2E_SANDBOX" policy-add </dev/null 2>&1 || true)"
preset_num="$(printf '%s\n' "$preset_list" | python3 -c 'import re,sys; preset=sys.argv[1]; text=sys.stdin.read(); m=re.search(r"(?m)^\s*(\d+)\).*" + re.escape(preset), text); print(m.group(1) if m else "")' "$NEMOCLAW_E2E_PRESET")"
if [ -z "$preset_num" ]; then
printf 'preset %s not found in list:\n%s\n' "$NEMOCLAW_E2E_PRESET" "$preset_list" >&2
exit 1
fi
printf '%s\nY\n' "$preset_num" | env NEMOCLAW_NON_INTERACTIVE= node "$NEMOCLAW_E2E_CLI" "$NEMOCLAW_E2E_SANDBOX" policy-add
`;
const result = await host.command("bash", ["-lc", script], {
const listResult = await host.command(
"bash",
[
"-lc",
'env NEMOCLAW_NON_INTERACTIVE= node "$NEMOCLAW_E2E_CLI" "$NEMOCLAW_E2E_SANDBOX" policy-add </dev/null',
],
{
artifactName: `policy-add-${preset}-interactive-list`,
env: baseEnv({
NEMOCLAW_E2E_CLI: CLI_ENTRYPOINT,
NEMOCLAW_E2E_SANDBOX: SANDBOX_NAME,
}),
timeoutMs: SANDBOX_EXEC_TIMEOUT_MS,
},
);
const presetNumber = requirePolicyPresetNumber(text(listResult), preset);

const result = await host.command("expect", ["-c", POLICY_ADD_EXPECT_SCRIPT], {
artifactName: `policy-add-${preset}-interactive`,
env: baseEnv({
NEMOCLAW_E2E_CLI: CLI_ENTRYPOINT,
NEMOCLAW_E2E_SANDBOX: SANDBOX_NAME,
NEMOCLAW_E2E_PRESET: preset,
NEMOCLAW_E2E_PRESET_NUM: presetNumber,
}),
timeoutMs: SANDBOX_EXEC_TIMEOUT_MS,
});
Expand Down Expand Up @@ -560,11 +570,25 @@ hello
curlStatus(sandbox, "https://pypi.org/simple/le/", "tc-net-02-pypi-post", "-X POST"),
).resolves.toBe("403");

const slackBefore = await fetchStatus(sandbox, "https://slack.com/", "tc-net-03-slack-before");
// Use Slack's non-redirecting API probe on the preset's actual API host;
// the marketing root can leave the slack.com allowlist during redirects.
const slackBefore = await fetchStatus(
sandbox,
"https://slack.com/api/api.test",
"tc-net-03-slack-before",
);
expect(slackBefore).toMatch(/STATUS_403|ERROR_/);
const slackApply = await applyPresetInteractively(host, "slack");
expect(slackApply.exitCode, text(slackApply)).toBe(0);
const slackAfter = await fetchStatus(sandbox, "https://slack.com/", "tc-net-03-slack-after");
const slackPolicyList = await runNemoclaw(host, [SANDBOX_NAME, "policy-list"], {
artifactName: "tc-net-03-policy-list-slack",
});
expect(text(slackPolicyList)).toMatch(/● slack/);
const slackAfter = await fetchStatus(
sandbox,
"https://slack.com/api/api.test",
"tc-net-03-slack-after",
);
expect(slackAfter).toMatch(/STATUS_200/);

const atlassianBefore = await fetchStatus(
Expand Down
Loading
Loading