diff --git a/test/e2e/e2e-cloud-experimental/checks/04-deepagents-code-fresh-reonboard.sh b/test/e2e/e2e-cloud-experimental/checks/04-deepagents-code-fresh-reonboard.sh index ae31bc17cab..93292cbc4c7 100755 --- a/test/e2e/e2e-cloud-experimental/checks/04-deepagents-code-fresh-reonboard.sh +++ b/test/e2e/e2e-cloud-experimental/checks/04-deepagents-code-fresh-reonboard.sh @@ -17,9 +17,19 @@ CLI="${NEMOCLAW_CLI_BIN:-${REPO}/bin/nemoclaw.js}" PREFIX="04-deepagents-code-fresh-reonboard" PRIMARY_TARGET_MODEL="openai/openai/gpt-5.5" FALLBACK_TARGET_MODEL="nvidia/nvidia/nemotron-3-ultra" -HOSTED_ENDPOINT="${NEMOCLAW_ENDPOINT_URL:-https://inference-api.nvidia.com/v1}" CREDENTIAL_CANARY="nemoclaw-dcode-config-get-canary" +# shellcheck source=test/e2e/lib/hermetic-compatible-inference.sh +. "$REPO/test/e2e/lib/hermetic-compatible-inference.sh" + +cleanup() { + local status=$? + nemoclaw_e2e_stop_hermetic_compatible_inference + return "$status" +} + +trap cleanup EXIT + fail() { printf '%s: FAIL: %s\n' "$PREFIX" "$1" >&2 exit 1 @@ -150,7 +160,6 @@ if ! sandbox_exec "test -d /sandbox/.deepagents && command -v dcode >/dev/null 2 exit 0 fi -[ -n "${COMPATIBLE_API_KEY:-}" ] || fail "COMPATIBLE_API_KEY is required" [ -x "$CLI" ] || fail "NemoClaw CLI is not executable at $CLI" if ! identity_before="$(dcode_identity)"; then @@ -173,6 +182,12 @@ fi [ "$model_a" != "$model_b" ] || fail "model A and model B must differ" pass "initial live identity reports model A" +export NEMOCLAW_E2E_COMPATIBLE_MODEL="$model_b" +if ! nemoclaw_e2e_start_hermetic_compatible_inference; then + fail "could not start hermetic compatible inference" +fi +pass "started hermetic compatible inference for re-onboard" + seed_source="$(seed_config_source | encode_source)" seed_command="printf '%s' ${seed_source@Q} | base64 -d | /opt/venv/bin/python3 -I - ${model_a@Q} ${CREDENTIAL_CANARY@Q}" seed_output="$(sandbox_exec "$seed_command")" || fail "could not seed stale DCode config" @@ -224,8 +239,7 @@ if ! reonboard_output="$( NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ NEMOCLAW_AGENT=langchain-deepagents-code \ NEMOCLAW_COMPAT_MODEL="$model_b" \ - NEMOCLAW_E2E_USE_HOSTED_INFERENCE=1 \ - NEMOCLAW_ENDPOINT_URL="$HOSTED_ENDPOINT" \ + NEMOCLAW_ENDPOINT_URL="$NEMOCLAW_ENDPOINT_URL" \ NEMOCLAW_MODEL="$model_b" \ NEMOCLAW_NON_INTERACTIVE=1 \ NEMOCLAW_PREFERRED_API=openai-completions \ @@ -239,6 +253,7 @@ if ! reonboard_output="$( fi printf '%s\n' "$reonboard_output" | grep -Fq "Backing up workspace state before recreating sandbox..." || fail "re-onboard did not take the pre-recreate backup path" printf '%s\n' "$reonboard_output" | grep -Fq "Restoring workspace state from pre-recreate backup..." || fail "re-onboard did not take the restore path" +nemoclaw_e2e_assert_hermetic_compatible_endpoint_consulted || fail "re-onboard did not consult the hermetic compatible endpoint" pass "same-name --fresh re-onboard crossed backup and restore boundaries" sandbox_list="$(openshell sandbox list 2>&1)" || fail "could not list sandbox after re-onboard" @@ -284,4 +299,4 @@ verify_output="$(sandbox_exec "$verify_command")" || fail "live DCode config doe printf '%s\n' "$verify_output" | grep -Fq "NEMOCLAW_DCODE_FRESH_CONFIG_VERIFIED" || fail "fresh config verification marker is missing" pass "config keeps model B and only the allowlisted preferences" -printf '%s: 11 passed, 0 failed\n' "$PREFIX" +printf '%s: 12 passed, 0 failed\n' "$PREFIX" diff --git a/test/e2e/lib/hermetic-compatible-inference.sh b/test/e2e/lib/hermetic-compatible-inference.sh index b63482b2b3e..93adad5648a 100755 --- a/test/e2e/lib/hermetic-compatible-inference.sh +++ b/test/e2e/lib/hermetic-compatible-inference.sh @@ -24,6 +24,7 @@ nemoclaw_e2e_start_hermetic_compatible_inference() { export FAKE_OPENAI_API_KEY="$fake_key" export FAKE_OPENAI_REQUIRE_AUTH=1 export NEMOCLAW_FAKE_OPENAI_REQUESTS_FILE="$FAKE_OPENAI_REQUESTS_FILE" + export NEMOCLAW_FAKE_OPENAI_REQUIRE_AUTH_MODELS=1 if ! start_fake_openai_compatible_api; then return 1 @@ -42,6 +43,29 @@ nemoclaw_e2e_stop_hermetic_compatible_inference() { stop_fake_openai_compatible_api } +nemoclaw_e2e_assert_hermetic_compatible_endpoint_consulted() { + node - "${FAKE_OPENAI_REQUESTS_FILE:-}" <<'NODE' +const fs = require("fs"); +const requestsFile = process.argv[2]; +if (!requestsFile || !fs.existsSync(requestsFile)) { + throw new Error(`request log missing: ${requestsFile || ""}`); +} +const entries = fs.readFileSync(requestsFile, "utf8") + .split(/\n+/) + .filter(Boolean) + .map((line) => JSON.parse(line)); +const consultations = entries.filter((entry) => + entry.method === "GET" && + ["/v1/models", "/models"].includes(entry.path) && + entry.authorizationSent === true && + entry.auth === "ok" +); +if (consultations.length === 0) { + throw new Error(`expected an authenticated fake endpoint models consultation, got ${JSON.stringify(entries)}`); +} +NODE +} + nemoclaw_e2e_assert_hermetic_compatible_inference_used() { node - "${FAKE_OPENAI_REQUESTS_FILE:-}" <<'NODE' const fs = require("fs"); diff --git a/test/e2e/lib/openai-compatible-api-proof.sh b/test/e2e/lib/openai-compatible-api-proof.sh index d1c813fbce7..1f18d09c3da 100755 --- a/test/e2e/lib/openai-compatible-api-proof.sh +++ b/test/e2e/lib/openai-compatible-api-proof.sh @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 start_fake_openai_compatible_api() { - local script_dir server_script port_file ready_host public_host + local script_dir server_script port_file ready_host public_host readiness_status script_dir="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" server_script="${script_dir}/fake-openai-compatible-api.mts" port_file="${FAKE_OPENAI_PORT_FILE:-$(mktemp)}" @@ -32,7 +32,10 @@ start_fake_openai_compatible_api() { for _ in $(seq 1 "${FAKE_OPENAI_READY_ATTEMPTS:-30}"); do if [ -s "$port_file" ]; then FAKE_OPENAI_PORT="$(cat "$port_file")" - if curl -sf "http://${ready_host}:${FAKE_OPENAI_PORT}/v1/models" >/dev/null 2>&1; then + readiness_status="$(curl -sS -o /dev/null -w '%{http_code}' "http://${ready_host}:${FAKE_OPENAI_PORT}/v1/models" 2>/dev/null || true)" + # A 401 still proves the server is listening when the fixture deliberately + # requires credentials on /v1/models. + if [ "$readiness_status" = "200" ] || [ "$readiness_status" = "401" ]; then public_host="${FAKE_OPENAI_PUBLIC_HOST:-$FAKE_OPENAI_HOST}" if [ "$public_host" = "0.0.0.0" ]; then public_host="127.0.0.1" diff --git a/test/e2e/live/cloud-experimental-checks.ts b/test/e2e/live/cloud-experimental-checks.ts index 688dbf0bcbc..52cba65c80c 100644 --- a/test/e2e/live/cloud-experimental-checks.ts +++ b/test/e2e/live/cloud-experimental-checks.ts @@ -77,6 +77,10 @@ export function buildCloudExperimentalCommandEnv( }; } +export function cloudExperimentalApiKeyForCheck(scriptPath: string, apiKey: string): string { + return scriptPath === DEEPAGENTS_FRESH_REONBOARD_CHECK ? "" : apiKey; +} + export function assertRequiredCloudExperimentalResult( scriptPath: string, result: ShellProbeResult, @@ -137,11 +141,12 @@ export async function runE2eCloudExperimentalChecks( checkScripts.length > 0 ? assertDeepAgentsRuntimeObserved(sandboxName, context) : undefined, ); for (const scriptPath of checkScripts) { + const scriptApiKey = cloudExperimentalApiKeyForCheck(scriptPath, apiKey); const result = await context.host.command("bash", [path.join(REPO_ROOT, scriptPath)], { artifactName: `cloud-experimental-${path.basename(scriptPath, ".sh")}`, cwd: REPO_ROOT, - env: buildCloudExperimentalCommandEnv(sandboxName, apiKey), - redactionValues: [apiKey], + env: buildCloudExperimentalCommandEnv(sandboxName, scriptApiKey), + redactionValues: scriptApiKey ? [scriptApiKey] : [], timeoutMs: cloudExperimentalCheckTimeoutMs(scriptPath), }); assertRequiredCloudExperimentalResult(scriptPath, result); diff --git a/test/e2e/support/platform-parity-cloud-experimental.test.ts b/test/e2e/support/platform-parity-cloud-experimental.test.ts index a6d3eca3fb4..7b6aec9ba56 100644 --- a/test/e2e/support/platform-parity-cloud-experimental.test.ts +++ b/test/e2e/support/platform-parity-cloud-experimental.test.ts @@ -15,11 +15,15 @@ vi.mock("../../../src/lib/actions/sandbox/exec", () => ({ import SandboxExecCommand from "../../../src/commands/sandbox/exec.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; -import { DEEPAGENTS_CLOUD_EXPERIMENTAL_CHECKS } from "../live/cloud-experimental-check-list.ts"; +import { + DEEPAGENTS_CLOUD_EXPERIMENTAL_CHECKS, + DEEPAGENTS_FRESH_REONBOARD_CHECK, +} from "../live/cloud-experimental-check-list.ts"; import { assertRequiredCloudExperimentalResult, buildCloudExperimentalChecksEvidence, buildCloudExperimentalCommandEnv, + cloudExperimentalApiKeyForCheck, cloudExperimentalCheckTimeoutMs, } from "../live/cloud-experimental-checks.ts"; @@ -27,10 +31,12 @@ const dcodeTavilyCheck = path.join( process.cwd(), "test/e2e/e2e-cloud-experimental/checks/09-deepagents-code-tavily-opt-in.sh", ); -const dcodeFreshReonboardCheck = path.join( - process.cwd(), - "test/e2e/e2e-cloud-experimental/checks/04-deepagents-code-fresh-reonboard.sh", -); +const dcodeFreshReonboardCheck = path.join(process.cwd(), DEEPAGENTS_FRESH_REONBOARD_CHECK); +const freshReonboardTimeoutMs = 30_000; + +function writeExecutable(filePath: string, lines: string[]): void { + fs.writeFileSync(filePath, `${lines.join("\n")}\n`, { mode: 0o755 }); +} function shellResult(exitCode: number, stdout: string, stderr = ""): ShellProbeResult { return { @@ -82,6 +88,186 @@ describe("P0-E cloud-experimental parity guardrails", () => { expect(script).toContain("re-onboard with the new selection"); }); + it( + "runs fresh re-onboard against a fake endpoint without hosted inference secrets (#5747)", + () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-fresh-reonboard-")); + try { + const binDir = path.join(tmp, "bin"); + const homeDir = path.join(tmp, "home"); + fs.mkdirSync(binDir); + fs.mkdirSync(homeDir); + const marker = path.join(tmp, "reonboard-done"); + const requestsFile = path.join(tmp, "requests.jsonl"); + const openshell = path.join(binDir, "openshell"); + const cli = path.join(binDir, "nemoclaw"); + + writeExecutable(openshell, [ + "#!/usr/bin/env bash", + "set -euo pipefail", + 'if [ "$1" = "sandbox" ] && [ "$2" = "exec" ]; then', + " shift 2", + ' [ "$1" = "--name" ] || { echo "missing --name" >&2; exit 2; }', + " shift 2", + ' [ "$1" = "--" ] || { echo "missing command boundary" >&2; exit 2; }', + " shift", + ' if [ "$1" = "bash" ] && [ "$2" = "-c" ]; then', + ' case "$3" in', + ' *"test -d /sandbox/.deepagents"*) exit 0 ;;', + ' *"sha256sum /sandbox/.deepagents/config.toml"*)', + " printf '%s\\n' 0000000000000000000000000000000000000000000000000000000000000000", + " exit 0", + " ;;", + " *)", + ' if [ -f "$FAKE_REONBOARD_DONE" ]; then', + ' printf "%s\\n" "NEMOCLAW_DCODE_FRESH_CONFIG_VERIFIED"', + " else", + ' printf "%s\\n" "NEMOCLAW_DCODE_STALE_CONFIG_SEEDED"', + " fi", + " exit 0", + " ;;", + " esac", + " fi", + ' if [ "$1" = "/usr/local/bin/dcode" ] && [ "$2" = "identity" ]; then', + ' if [ -f "$FAKE_REONBOARD_DONE" ]; then', + ' model="openai/openai/gpt-5.5"', + " else", + ' model="nvidia/nvidia/nemotron-3-ultra"', + " fi", + ' printf "Route: inference\\nProvider: compatible-endpoint\\nModel: openai:%s\\nEndpoint: https://inference.local/v1\\n" "$model"', + " exit 0", + " fi", + "fi", + 'if [ "$1" = "sandbox" ] && [ "$2" = "list" ]; then', + ' printf "%s\\n" "deepagents-sandbox Ready"', + " exit 0", + "fi", + 'printf "unexpected openshell args: %s\\n" "$*" >&2', + "exit 2", + ]); + + writeExecutable(cli, [ + "#!/usr/bin/env bash", + "set -euo pipefail", + "fail() { printf 'fake nemoclaw: %s\\n' \"$1\" >&2; exit 2; }", + 'if [ "$1" = "onboard" ]; then', + ' if printenv NVIDIA_INFERENCE_API_KEY >/dev/null; then fail "NVIDIA_INFERENCE_API_KEY leaked"; fi', + ' if [ "$(printenv NEMOCLAW_E2E_USE_HOSTED_INFERENCE 2>/dev/null || true)" = "1" ]; then fail "hosted inference flag leaked"; fi', + ' [ "$(printenv COMPATIBLE_API_KEY)" = "e2e-compatible-key" ] || fail "missing fake compatible key"', + ' [ "$(printenv NEMOCLAW_PROVIDER)" = "custom" ] || fail "missing custom provider"', + ' [ "$(printenv NEMOCLAW_MODEL)" = "openai/openai/gpt-5.5" ] || fail "wrong target model"', + ' case "$(printenv NEMOCLAW_ENDPOINT_URL)" in', + " http://127.0.0.1:*/v1) ;;", + ' *) fail "unexpected compatible endpoint" ;;', + " esac", + " node --input-type=module <<'NODE'", + 'const response = await fetch(process.env.NEMOCLAW_ENDPOINT_URL + "/models", {', + " headers: {", + ' authorization: "Bearer " + process.env.COMPATIBLE_API_KEY', + " },", + "});", + "if (!response.ok) {", + " console.error(await response.text());", + " process.exit(1);", + "}", + "NODE", + ' mkdir -p "$HOME/.nemoclaw"', + ' printf \'%s\\n\' \'{"sandboxes":{"deepagents-sandbox":{"agent":"langchain-deepagents-code","model":"openai/openai/gpt-5.5","provider":"compatible-endpoint","credentialEnv":"COMPATIBLE_API_KEY"}}}\' > "$HOME/.nemoclaw/sandboxes.json"', + ' touch "$FAKE_REONBOARD_DONE"', + ' printf "%s\\n" "Backing up workspace state before recreating sandbox..."', + ' printf "%s\\n" "Restoring workspace state from pre-recreate backup..."', + " exit 0", + "fi", + 'if [ "$1" = "deepagents-sandbox" ] && [ "$2" = "config" ] && [ "$3" = "get" ]; then', + ' if [ -f "$FAKE_REONBOARD_DONE" ]; then', + ' model="openai/openai/gpt-5.5"', + " else", + ' model="nvidia/nvidia/nemotron-3-ultra"', + " fi", + ' if [ "$#" -eq 3 ]; then', + ' MODEL="$model" node -e \'console.log(JSON.stringify({models:{default:"openai:" + process.env.MODEL},headers:{authorization:"[STRIPPED_BY_MIGRATION]"}}))\'', + ' elif [ "$4" = "--key" ] && [ "$5" = "models.default" ]; then', + ' printf \'"openai:%s"\\n\' "$model"', + ' elif [ "$4" = "--format" ] && [ "$5" = "yaml" ]; then', + ' printf \'models:\\n default: openai:%s\\nheaders:\\n authorization: "[STRIPPED_BY_MIGRATION]"\\n\' "$model"', + " else", + ' fail "unexpected config get args"', + " fi", + " exit 0", + "fi", + 'if [ "$1" = "deepagents-sandbox" ] && [ "$2" = "config" ] && [ "$3" = "set" ]; then', + ' printf "%s\\n" "config is baked into the sandbox image at build time" >&2', + ' printf "%s\\n" "re-onboard with the new selection using --fresh" >&2', + " exit 1", + "fi", + 'if [ "$1" = "deepagents-sandbox" ] && [ "$2" = "status" ] && [ "$3" = "--json" ]; then', + ' printf \'%s\\n\' \'{"name":"deepagents-sandbox","model":"openai/openai/gpt-5.5","provider":"compatible-endpoint"}\'', + " exit 0", + "fi", + 'printf "unexpected nemoclaw args: %s\\n" "$*" >&2', + "exit 2", + ]); + + const result = spawnSync("bash", [dcodeFreshReonboardCheck], { + encoding: "utf8", + timeout: freshReonboardTimeoutMs, + env: { + COMPATIBLE_API_KEY: "hosted-compatible-secret-should-not-be-used", + FAKE_OPENAI_PUBLIC_HOST: "127.0.0.1", + FAKE_OPENAI_REQUESTS_FILE: requestsFile, + FAKE_REONBOARD_DONE: marker, + HOME: homeDir, + NEMOCLAW_CLI_BIN: cli, + NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1", + NEMOCLAW_ENDPOINT_URL: "https://inference-api.nvidia.com/v1", + NVIDIA_INFERENCE_API_KEY: "hosted-nvidia-secret-should-not-be-used", + PATH: binDir + ":" + (process.env.PATH ?? "/usr/bin:/bin"), + REPO: process.cwd(), + SANDBOX_NAME: "deepagents-sandbox", + }, + }); + + expect(result.status, result.stdout + "\n" + result.stderr).toBe(0); + expect(result.stdout).toContain( + "04-deepagents-code-fresh-reonboard: OK (started hermetic compatible inference for re-onboard)", + ); + expect(result.stdout).toContain("04-deepagents-code-fresh-reonboard: 12 passed, 0 failed"); + expect(result.stdout + result.stderr).not.toContain( + "hosted-nvidia-secret-should-not-be-used", + ); + expect(result.stdout + result.stderr).not.toContain( + "hosted-compatible-secret-should-not-be-used", + ); + const endpointRequests = fs + .readFileSync(requestsFile, "utf8") + .trim() + .split("\n") + .map( + (line) => + JSON.parse(line) as { + auth?: string; + authorizationSent?: boolean; + method: string; + path: string; + }, + ); + expect( + endpointRequests.some( + (request) => + request.method === "GET" && + request.path === "/v1/models" && + request.authorizationSent === true && + request.auth === "ok", + ), + ).toBe(true); + expect(endpointRequests.some((request) => request.method === "POST")).toBe(false); + } finally { + fs.rmSync(tmp, { force: true, recursive: true }); + } + }, + freshReonboardTimeoutMs, + ); + it("preserves the repeated env-unset pairs from the failed observability invocation", async () => { await SandboxExecCommand.run( [ @@ -479,6 +665,18 @@ describe("P0-E cloud-experimental parity guardrails", () => { ).toBe(35 * 60_000); }); + it("withholds the hosted inference key from the hermetic re-onboard check (#5747)", () => { + expect(cloudExperimentalApiKeyForCheck(DEEPAGENTS_FRESH_REONBOARD_CHECK, "secret-key")).toBe( + "", + ); + expect( + cloudExperimentalApiKeyForCheck( + "test/e2e/e2e-cloud-experimental/checks/07-deepagents-code-headless-inference.sh", + "secret-key", + ), + ).toBe("secret-key"); + }); + it("documents Deep Agents check scripts in generated launch/QA evidence", () => { const evidence = buildCloudExperimentalChecksEvidence( "cloud-langchain-deepagents-code",