From bdf2d2c4619639c334d1731b25d9e05c88278560 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Fri, 19 Jun 2026 12:10:49 -0700 Subject: [PATCH 1/8] test(e2e): migrate Brave search to vitest Signed-off-by: Carlos Villela --- .github/workflows/e2e-vitest-scenarios.yaml | 66 +++++ test/e2e-scenario/live/brave-search.test.ts | 253 ++++++++++++++++++++ 2 files changed, 319 insertions(+) create mode 100644 test/e2e-scenario/live/brave-search.test.ts diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index d270a3b670b..a48bc6a4575 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -742,6 +742,71 @@ jobs: if-no-files-found: ignore retention-days: 14 + brave-search-vitest: + needs: generate-matrix + if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',brave-search-vitest,') || contains(format(',{0},', inputs.scenarios), ',brave-search,') }} + runs-on: ubuntu-latest + timeout-minutes: 45 + env: + FREE_STANDING_VITEST_JOB: "1" + FREE_STANDING_SCENARIO_ID: "brave-search" + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/brave-search + NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js + NEMOCLAW_RUN_E2E_SCENARIOS: "1" + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + NEMOCLAW_SANDBOX_NAME: "e2e-brave-search" + OPENSHELL_GATEWAY: "nemoclaw" + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Set up Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 + with: + node-version: 22 + cache: npm + + - name: Install root dependencies + run: npm ci --ignore-scripts + + - name: Build CLI + run: npm run build:cli + + - name: Install OpenShell CLI + run: bash scripts/install-openshell.sh + + - name: Run Brave search live Vitest test + env: + BRAVE_API_KEY: ${{ secrets.BRAVE_API_KEY }} + NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }} + run: | + set -euo pipefail + export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH" + if command -v openshell >/dev/null 2>&1; then + OPENSHELL_BIN="$(command -v openshell)" + elif [ -x "$HOME/.local/bin/openshell" ]; then + OPENSHELL_BIN="$HOME/.local/bin/openshell" + else + echo "::error::OpenShell CLI not found after install" + ls -la /usr/local/bin/openshell "$HOME/.local/bin/openshell" 2>&1 || true + exit 1 + fi + export OPENSHELL_BIN + "$OPENSHELL_BIN" --version + npx vitest run --project e2e-scenarios-live test/e2e-scenario/live/brave-search.test.ts --silent=false --reporter=default + + - name: Upload Brave search artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: e2e-vitest-scenarios-brave-search + path: e2e-artifacts/vitest/brave-search/ + include-hidden-files: false + if-no-files-found: ignore + retention-days: 14 + issue-4434-tui-unreachable-inference-vitest: needs: generate-matrix if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',issue-4434-tui-unreachable-inference-vitest,') || contains(format(',{0},', inputs.scenarios), ',issue-4434-tui-unreachable-inference,') }} @@ -3586,6 +3651,7 @@ jobs: openclaw-skill-cli-vitest, inference-routing-vitest, cloud-inference-vitest, + brave-search-vitest, credential-sanitization-vitest, credential-migration-vitest, sessions-agents-cli-vitest, diff --git a/test/e2e-scenario/live/brave-search.test.ts b/test/e2e-scenario/live/brave-search.test.ts new file mode 100644 index 00000000000..9c8873df16b --- /dev/null +++ b/test/e2e-scenario/live/brave-search.test.ts @@ -0,0 +1,253 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Live Vitest replacement for test/e2e/test-brave-search-e2e.sh. + * + * Preserves the legacy #2687 acceptance boundary: non-interactive onboard with + * a real BRAVE_API_KEY, brave policy/config wiring, secret non-leak checks, + * a real agent web-search turn, and a direct in-sandbox Brave API curl using + * the OpenShell credential placeholder. + */ + +import path from "node:path"; + +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { resultText } from "../fixtures/clients/index.ts"; +import { + type SandboxClient, + trustedSandboxShellScript, + validateSandboxName, +} from "../fixtures/clients/sandbox.ts"; +import { expect, test } from "../fixtures/e2e-test.ts"; +import { shouldRunLiveE2EScenarios } from "../fixtures/live-project-gate.ts"; +import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; +import { isTransientProviderValidationFailure } from "./network-policy-transient-provider.ts"; + +const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); +const CLI_ENTRYPOINT = path.join(REPO_ROOT, "bin", "nemoclaw.js"); +const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-brave-search"; +validateSandboxName(SANDBOX_NAME); +const INSTALL_ATTEMPTS = process.env.CI === "true" || process.env.GITHUB_ACTIONS === "true" ? 3 : 1; +const LIVE_TIMEOUT_MS = 35 * 60_000; +const PLACEHOLDER_PATTERN = /openshell:resolve:env:([A-Za-z0-9_]+_)?BRAVE_API_KEY/; + +function commandEnv(extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { + return { + ...buildAvailabilityProbeEnv(), + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_RECREATE_SANDBOX: "1", + NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, + OPENSHELL_GATEWAY: process.env.OPENSHELL_GATEWAY ?? "nemoclaw", + ...extra, + }; +} + +async function bestEffort(run: () => Promise): Promise { + try { + await run(); + } catch { + // Cleanup should not mask primary failures. + } +} + +function singleLineShell(script: string): string { + const encoded = Buffer.from(script, "utf8").toString("base64"); + return `tmp=$(mktemp); trap 'rm -f "$tmp"' EXIT; printf %s '${encoded}' | base64 -d > "$tmp"; sh "$tmp"`; +} + +async function sandboxShell( + sandbox: SandboxClient, + script: string, + options: { artifactName: string; timeoutMs?: number; redactionValues?: string[] }, +): Promise { + return await sandbox.execShell(SANDBOX_NAME, trustedSandboxShellScript(singleLineShell(script)), { + artifactName: options.artifactName, + env: commandEnv(), + timeoutMs: options.timeoutMs ?? 60_000, + redactionValues: options.redactionValues, + }); +} + +async function cleanupBraveSandbox(sandbox: SandboxClient): Promise { + await bestEffort(() => + sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { + artifactName: "cleanup-openshell-delete-brave-search", + env: commandEnv(), + timeoutMs: 60_000, + }), + ); +} + +function parsePlaceholder(configText: string): string | undefined { + const parsed = JSON.parse(configText) as { + tools?: { web?: { search?: { apiKey?: unknown } } }; + }; + const value = parsed.tools?.web?.search?.apiKey; + return typeof value === "string" && value ? value : undefined; +} + +test.skipIf(!shouldRunLiveE2EScenarios())( + "Brave search preset wires policy/config, hides the real key, and performs real searches (#2687)", + { timeout: LIVE_TIMEOUT_MS }, + async ({ artifacts, cleanup, host, sandbox, secrets, skip }) => { + const braveKey = secrets.required("BRAVE_API_KEY"); + const inferenceKey = secrets.required("NVIDIA_INFERENCE_API_KEY"); + const redactionValues = [braveKey, inferenceKey]; + + await artifacts.writeJson("scenario.json", { + id: "brave-search", + runner: "vitest", + legacySource: "test/e2e/test-brave-search-e2e.sh", + boundary: + "source CLI onboard + OpenShell policy/config + in-sandbox OpenClaw/Brave API calls", + sandboxName: SANDBOX_NAME, + contracts: [ + "onboard succeeds with BRAVE_API_KEY present", + "the brave network policy preset includes api.search.brave.com", + "OpenClaw web search config is enabled and selects provider=brave", + "the real BRAVE_API_KEY is absent from openclaw.json and sandbox shell env", + "OpenClaw agent can perform a Brave-backed web search", + "curl from inside the sandbox can query Brave using the placeholder token header", + ], + }); + + const dockerInfo = await host.command("docker", ["info"], { + artifactName: "phase-0-docker-info", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }); + if (dockerInfo.exitCode !== 0) { + if (process.env.GITHUB_ACTIONS === "true") { + throw new Error(`Docker is required for Brave search E2E: ${resultText(dockerInfo)}`); + } + skip(`Docker is required for Brave search E2E: ${resultText(dockerInfo)}`); + } + + cleanup.add(`destroy brave search sandbox ${SANDBOX_NAME}`, async () => { + await bestEffort(() => + host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { + artifactName: "cleanup-nemoclaw-destroy-brave-search", + env: commandEnv(), + timeoutMs: 120_000, + }), + ); + await cleanupBraveSandbox(sandbox); + }); + + await bestEffort(() => + host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { + artifactName: "pre-cleanup-nemoclaw-destroy-brave-search", + env: commandEnv(), + timeoutMs: 120_000, + }), + ); + await cleanupBraveSandbox(sandbox); + + let onboard: ShellProbeResult | undefined; + for (let attempt = 1; attempt <= INSTALL_ATTEMPTS; attempt += 1) { + onboard = await host.command( + "node", + [ + CLI_ENTRYPOINT, + "onboard", + "--fresh", + "--non-interactive", + "--yes-i-accept-third-party-software", + ], + { + artifactName: + attempt === 1 + ? "phase-1-onboard-brave-search" + : `phase-1-onboard-brave-search-attempt-${attempt}`, + cwd: REPO_ROOT, + env: commandEnv({ + BRAVE_API_KEY: braveKey, + NVIDIA_INFERENCE_API_KEY: inferenceKey, + NVIDIA_API_KEY: inferenceKey, + }), + redactionValues, + timeoutMs: 20 * 60_000, + }, + ); + if (onboard.exitCode === 0) break; + if (isTransientProviderValidationFailure(onboard) && attempt < INSTALL_ATTEMPTS) { + await new Promise((resolve) => setTimeout(resolve, 10_000 * attempt)); + continue; + } + break; + } + expect(onboard, "onboard command must run").toBeDefined(); + expect(onboard?.exitCode, resultText(onboard as ShellProbeResult)).toBe(0); + + const policy = await sandbox.openshell(["policy", "get", "--full", SANDBOX_NAME], { + artifactName: "phase-2-brave-policy", + env: commandEnv(), + timeoutMs: 60_000, + }); + expect(policy.exitCode, resultText(policy)).toBe(0); + expect(resultText(policy)).toContain("api.search.brave.com"); + + const config = await sandbox.exec(SANDBOX_NAME, ["cat", "/sandbox/.openclaw/openclaw.json"], { + artifactName: "phase-2-openclaw-config", + env: commandEnv(), + redactionValues, + timeoutMs: 60_000, + }); + expect(config.exitCode, resultText(config)).toBe(0); + expect(config.stdout).not.toContain(braveKey); + const parsedConfig = JSON.parse(config.stdout) as { + tools?: { web?: { search?: { enabled?: unknown; provider?: unknown; apiKey?: unknown } } }; + }; + const searchConfig = parsedConfig.tools?.web?.search; + expect(searchConfig?.enabled, config.stdout).toBe(true); + expect(searchConfig?.provider, config.stdout).toBe("brave"); + const placeholder = parsePlaceholder(config.stdout); + expect(placeholder, config.stdout).toMatch(PLACEHOLDER_PATTERN); + + const envCheck = await sandbox.exec( + SANDBOX_NAME, + ["sh", "-lc", "printenv BRAVE_API_KEY || true"], + { + artifactName: "phase-3-sandbox-brave-env", + env: commandEnv(), + redactionValues, + timeoutMs: 30_000, + }, + ); + expect(envCheck.exitCode, resultText(envCheck)).toBe(0); + expect(envCheck.stdout).not.toContain(braveKey); + if (envCheck.stdout.trim()) expect(envCheck.stdout.trim()).toMatch(PLACEHOLDER_PATTERN); + + const agent = await sandboxShell( + sandbox, + `openclaw agent --agent main --json --session-id e2e-brave-agent-$$ -m 'Use the web search tool to find one result for the query: NVIDIA. Reply with only the title of the top result.'`, + { + artifactName: "phase-4a-agent-web-search", + timeoutMs: 150_000, + redactionValues, + }, + ); + expect(resultText(agent)).not.toMatch( + /SsrFBlockedError|Blocked hostname|ECONNREFUSED|EAI_AGAIN|gateway unavailable|network connection error/i, + ); + expect(agent.exitCode, resultText(agent)).toBe(0); + expect(resultText(agent)).toMatch(/nvidia|geforce|cuda|gpu/i); + + const curl = await sandboxShell( + sandbox, + `curl -sS --max-time 20 -G 'https://api.search.brave.com/res/v1/web/search' --data-urlencode 'q=NVIDIA' --data-urlencode 'count=1' -H 'X-Subscription-Token: ${placeholder}' -w '\nHTTP_STATUS:%{http_code}\n'`, + { + artifactName: "phase-4b-direct-brave-curl", + timeoutMs: 60_000, + redactionValues, + }, + ); + const status = resultText(curl).match(/HTTP_STATUS:(\d{3})/)?.[1]; + expect(status, resultText(curl)).toBe("200"); + const body = resultText(curl).replace(/\n?HTTP_STATUS:\d{3}\s*$/u, ""); + const braveResponse = JSON.parse(body) as { web?: { results?: unknown[] } }; + expect(braveResponse.web?.results?.length ?? 0, body.slice(0, 500)).toBeGreaterThan(0); + }, +); From 441a57d161cfcc28bb161f3fddf0e841b04c4b6c Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Fri, 19 Jun 2026 12:53:00 -0700 Subject: [PATCH 2/8] test(e2e): harden Brave search assertions Signed-off-by: Carlos Villela --- test/e2e-scenario/live/brave-search.test.ts | 50 +++++++++++++++++++-- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/test/e2e-scenario/live/brave-search.test.ts b/test/e2e-scenario/live/brave-search.test.ts index 9c8873df16b..b2dd5684ed3 100644 --- a/test/e2e-scenario/live/brave-search.test.ts +++ b/test/e2e-scenario/live/brave-search.test.ts @@ -30,7 +30,7 @@ const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-brave-search"; validateSandboxName(SANDBOX_NAME); const INSTALL_ATTEMPTS = process.env.CI === "true" || process.env.GITHUB_ACTIONS === "true" ? 3 : 1; const LIVE_TIMEOUT_MS = 35 * 60_000; -const PLACEHOLDER_PATTERN = /openshell:resolve:env:([A-Za-z0-9_]+_)?BRAVE_API_KEY/; +const PLACEHOLDER_PATTERN = /^openshell:resolve:env:([A-Za-z0-9_]+_)?BRAVE_API_KEY$/; function commandEnv(extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { return { @@ -88,6 +88,29 @@ function parsePlaceholder(configText: string): string | undefined { return typeof value === "string" && value ? value : undefined; } +function extractOpenClawAgentText(output: string): string { + for (const index of [...output] + .map((char, idx) => (char === "{" ? idx : -1)) + .filter((idx) => idx >= 0)) { + try { + const parsed = JSON.parse(output.slice(index)) as { + payloads?: Array<{ text?: unknown }>; + meta?: { finalAssistantVisibleText?: unknown }; + }; + const payloadText = parsed.payloads + ?.map((payload) => payload.text) + .find((value): value is string => typeof value === "string" && value.trim().length > 0); + if (payloadText) return payloadText; + if (typeof parsed.meta?.finalAssistantVisibleText === "string") { + return parsed.meta.finalAssistantVisibleText; + } + } catch { + // Keep scanning; OpenClaw can emit non-JSON progress before the result. + } + } + return ""; +} + test.skipIf(!shouldRunLiveE2EScenarios())( "Brave search preset wires policy/config, hides the real key, and performs real searches (#2687)", { timeout: LIVE_TIMEOUT_MS }, @@ -196,7 +219,27 @@ test.skipIf(!shouldRunLiveE2EScenarios())( timeoutMs: 60_000, }); expect(config.exitCode, resultText(config)).toBe(0); - expect(config.stdout).not.toContain(braveKey); + const rawLeakCheck = await sandbox.execShell( + SANDBOX_NAME, + trustedSandboxShellScript( + `python3 - <<'PY' +from pathlib import Path +needle = ${JSON.stringify(braveKey)} +body = Path('/sandbox/.openclaw/openclaw.json').read_text(encoding='utf-8') +raise SystemExit(1 if needle in body else 0) +PY`, + ), + { + artifactName: "phase-3-openclaw-config-raw-secret-leak-check", + env: commandEnv(), + redactionValues, + timeoutMs: 30_000, + }, + ); + expect( + rawLeakCheck.exitCode, + "raw BRAVE_API_KEY must not appear anywhere in openclaw.json", + ).toBe(0); const parsedConfig = JSON.parse(config.stdout) as { tools?: { web?: { search?: { enabled?: unknown; provider?: unknown; apiKey?: unknown } } }; }; @@ -233,7 +276,8 @@ test.skipIf(!shouldRunLiveE2EScenarios())( /SsrFBlockedError|Blocked hostname|ECONNREFUSED|EAI_AGAIN|gateway unavailable|network connection error/i, ); expect(agent.exitCode, resultText(agent)).toBe(0); - expect(resultText(agent)).toMatch(/nvidia|geforce|cuda|gpu/i); + const assistantText = extractOpenClawAgentText(agent.stdout); + expect(assistantText, resultText(agent)).toMatch(/nvidia|geforce|cuda|gpu/i); const curl = await sandboxShell( sandbox, From ba3c6f59d94d978b275644fb8cb2f4de09afbf1d Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Fri, 19 Jun 2026 14:07:35 -0700 Subject: [PATCH 3/8] test(e2e): avoid Brave key argv leakage Signed-off-by: Carlos Villela --- test/e2e-scenario/live/brave-search.test.ts | 31 +++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/test/e2e-scenario/live/brave-search.test.ts b/test/e2e-scenario/live/brave-search.test.ts index b2dd5684ed3..43ee960c94b 100644 --- a/test/e2e-scenario/live/brave-search.test.ts +++ b/test/e2e-scenario/live/brave-search.test.ts @@ -10,6 +10,8 @@ * the OpenShell credential placeholder. */ +import fs from "node:fs"; +import os from "node:os"; import path from "node:path"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; @@ -88,6 +90,11 @@ function parsePlaceholder(configText: string): string | undefined { return typeof value === "string" && value ? value : undefined; } +/** + * Source boundary: `openclaw agent --json` may emit launcher progress before + * the final JSON envelope. This mirrors the legacy shell parser until OpenClaw + * exposes a stable JSON-only stdout contract for live E2E consumers. + */ function extractOpenClawAgentText(output: string): string { for (const index of [...output] .map((char, idx) => (char === "{" ? idx : -1)) @@ -219,12 +226,33 @@ test.skipIf(!shouldRunLiveE2EScenarios())( timeoutMs: 60_000, }); expect(config.exitCode, resultText(config)).toBe(0); + const secretDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-brave-secret-")); + const secretFile = path.join(secretDir, "brave-key"); + fs.writeFileSync(secretFile, braveKey, { mode: 0o600 }); + const remoteSecretFile = "/tmp/nemoclaw-brave-key-leak-check"; + cleanup.add("remove temporary Brave leak-check secret", async () => { + fs.rmSync(secretDir, { recursive: true, force: true }); + await bestEffort(() => + sandbox.execShell(SANDBOX_NAME, trustedSandboxShellScript(`rm -f ${remoteSecretFile}`), { + artifactName: "cleanup-brave-leak-secret", + env: commandEnv(), + timeoutMs: 30_000, + }), + ); + }); + const uploadSecret = await sandbox.upload(SANDBOX_NAME, secretFile, remoteSecretFile, { + artifactName: "phase-3-upload-brave-leak-secret", + env: commandEnv(), + redactionValues, + timeoutMs: 30_000, + }); + expect(uploadSecret.exitCode, resultText(uploadSecret)).toBe(0); const rawLeakCheck = await sandbox.execShell( SANDBOX_NAME, trustedSandboxShellScript( `python3 - <<'PY' from pathlib import Path -needle = ${JSON.stringify(braveKey)} +needle = Path('${remoteSecretFile}').read_text(encoding='utf-8') body = Path('/sandbox/.openclaw/openclaw.json').read_text(encoding='utf-8') raise SystemExit(1 if needle in body else 0) PY`, @@ -232,7 +260,6 @@ PY`, { artifactName: "phase-3-openclaw-config-raw-secret-leak-check", env: commandEnv(), - redactionValues, timeoutMs: 30_000, }, ); From 5b8c4fe58d1967ce3d52e0b0f573efbe432e1c83 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Fri, 19 Jun 2026 14:16:05 -0700 Subject: [PATCH 4/8] test(e2e): mirror Brave agent JSON parsing Signed-off-by: Carlos Villela --- test/e2e-scenario/live/brave-search.test.ts | 78 +++++++++++++++------ 1 file changed, 56 insertions(+), 22 deletions(-) diff --git a/test/e2e-scenario/live/brave-search.test.ts b/test/e2e-scenario/live/brave-search.test.ts index 43ee960c94b..cfe433d4a41 100644 --- a/test/e2e-scenario/live/brave-search.test.ts +++ b/test/e2e-scenario/live/brave-search.test.ts @@ -90,32 +90,66 @@ function parsePlaceholder(configText: string): string | undefined { return typeof value === "string" && value ? value : undefined; } +function firstJsonObject(output: string): unknown { + for (let start = output.indexOf("{"); start >= 0; start = output.indexOf("{", start + 1)) { + let depth = 0; + let inString = false; + let escaped = false; + for (let index = start; index < output.length; index += 1) { + const char = output[index]; + if (inString) { + if (escaped) escaped = false; + else if (char === "\\") escaped = true; + else if (char === '"') inString = false; + continue; + } + if (char === '"') inString = true; + else if (char === "{") depth += 1; + else if (char === "}") { + depth -= 1; + if (depth === 0) { + try { + return JSON.parse(output.slice(start, index + 1)); + } catch { + break; + } + } + } + } + } + return undefined; +} + +function collectAssistantText(value: unknown): string[] { + if (typeof value === "string" && value.trim()) return [value.trim()]; + if (!value || typeof value !== "object") return []; + if (Array.isArray(value)) return value.flatMap(collectAssistantText); + const record = value as Record; + const texts: string[] = []; + for (const key of [ + "result", + "payloads", + "messages", + "choices", + "message", + "delta", + "content", + "text", + ]) { + if (key in record) texts.push(...collectAssistantText(record[key])); + } + return texts; +} + /** * Source boundary: `openclaw agent --json` may emit launcher progress before - * the final JSON envelope. This mirrors the legacy shell parser until OpenClaw - * exposes a stable JSON-only stdout contract for live E2E consumers. + * and after the final JSON envelope. This mirrors the retained legacy shell + * parser's tolerant envelope handling until OpenClaw exposes a stable + * JSON-only stdout contract for live E2E consumers. */ function extractOpenClawAgentText(output: string): string { - for (const index of [...output] - .map((char, idx) => (char === "{" ? idx : -1)) - .filter((idx) => idx >= 0)) { - try { - const parsed = JSON.parse(output.slice(index)) as { - payloads?: Array<{ text?: unknown }>; - meta?: { finalAssistantVisibleText?: unknown }; - }; - const payloadText = parsed.payloads - ?.map((payload) => payload.text) - .find((value): value is string => typeof value === "string" && value.trim().length > 0); - if (payloadText) return payloadText; - if (typeof parsed.meta?.finalAssistantVisibleText === "string") { - return parsed.meta.finalAssistantVisibleText; - } - } catch { - // Keep scanning; OpenClaw can emit non-JSON progress before the result. - } - } - return ""; + const parsed = firstJsonObject(output); + return collectAssistantText(parsed)[0] ?? ""; } test.skipIf(!shouldRunLiveE2EScenarios())( From afd8cdaebfaf4abb2f8e5fd864b377ff19914cab Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Fri, 19 Jun 2026 14:49:49 -0700 Subject: [PATCH 5/8] test(e2e): relax workflow inventory timeout Signed-off-by: Carlos Villela --- .../support-tests/e2e-scenarios-workflow.test.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts index 2e3e22e0a7c..c812fb20caa 100644 --- a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts +++ b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts @@ -8,14 +8,13 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; import YAML from "yaml"; - -import { testTimeoutOptions } from "../../helpers/timeouts"; import { evaluateE2eVitestWorkflowDispatchSelectors, readFreeStandingJobsInventory, validateE2eVitestScenariosWorkflowBoundary, validateFreeStandingWorkflowInventory, } from "../../../tools/e2e-scenarios/workflow-boundary.mts"; +import { testTimeoutOptions } from "../../helpers/timeouts"; function readWorkflow(): Record { return YAML.parse( @@ -565,7 +564,7 @@ describe("e2e-vitest-scenarios workflow boundary", () => { }, ); - it("derives the free-standing inventory from workflow job metadata", () => { + it("derives the free-standing inventory from workflow job metadata", { timeout: 60_000 }, () => { const inventory = readFreeStandingJobsInventory(); expect(validateFreeStandingWorkflowInventory()).toEqual([]); expect(inventory.allowedJobs).toContain("openshell-version-pin-vitest"); From 803b42e7288aa262ac52ce7f53b2de4111076e63 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sat, 20 Jun 2026 08:53:40 -0700 Subject: [PATCH 6/8] test(e2e): move scenario logic out of test wrapper Signed-off-by: Carlos Villela --- .../live/brave-search.scenario.ts | 358 ++++++++++++++++++ test/e2e-scenario/live/brave-search.test.ts | 356 +---------------- 2 files changed, 359 insertions(+), 355 deletions(-) create mode 100644 test/e2e-scenario/live/brave-search.scenario.ts diff --git a/test/e2e-scenario/live/brave-search.scenario.ts b/test/e2e-scenario/live/brave-search.scenario.ts new file mode 100644 index 00000000000..cfe433d4a41 --- /dev/null +++ b/test/e2e-scenario/live/brave-search.scenario.ts @@ -0,0 +1,358 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Live Vitest replacement for test/e2e/test-brave-search-e2e.sh. + * + * Preserves the legacy #2687 acceptance boundary: non-interactive onboard with + * a real BRAVE_API_KEY, brave policy/config wiring, secret non-leak checks, + * a real agent web-search turn, and a direct in-sandbox Brave API curl using + * the OpenShell credential placeholder. + */ + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { resultText } from "../fixtures/clients/index.ts"; +import { + type SandboxClient, + trustedSandboxShellScript, + validateSandboxName, +} from "../fixtures/clients/sandbox.ts"; +import { expect, test } from "../fixtures/e2e-test.ts"; +import { shouldRunLiveE2EScenarios } from "../fixtures/live-project-gate.ts"; +import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; +import { isTransientProviderValidationFailure } from "./network-policy-transient-provider.ts"; + +const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); +const CLI_ENTRYPOINT = path.join(REPO_ROOT, "bin", "nemoclaw.js"); +const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-brave-search"; +validateSandboxName(SANDBOX_NAME); +const INSTALL_ATTEMPTS = process.env.CI === "true" || process.env.GITHUB_ACTIONS === "true" ? 3 : 1; +const LIVE_TIMEOUT_MS = 35 * 60_000; +const PLACEHOLDER_PATTERN = /^openshell:resolve:env:([A-Za-z0-9_]+_)?BRAVE_API_KEY$/; + +function commandEnv(extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { + return { + ...buildAvailabilityProbeEnv(), + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_RECREATE_SANDBOX: "1", + NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, + OPENSHELL_GATEWAY: process.env.OPENSHELL_GATEWAY ?? "nemoclaw", + ...extra, + }; +} + +async function bestEffort(run: () => Promise): Promise { + try { + await run(); + } catch { + // Cleanup should not mask primary failures. + } +} + +function singleLineShell(script: string): string { + const encoded = Buffer.from(script, "utf8").toString("base64"); + return `tmp=$(mktemp); trap 'rm -f "$tmp"' EXIT; printf %s '${encoded}' | base64 -d > "$tmp"; sh "$tmp"`; +} + +async function sandboxShell( + sandbox: SandboxClient, + script: string, + options: { artifactName: string; timeoutMs?: number; redactionValues?: string[] }, +): Promise { + return await sandbox.execShell(SANDBOX_NAME, trustedSandboxShellScript(singleLineShell(script)), { + artifactName: options.artifactName, + env: commandEnv(), + timeoutMs: options.timeoutMs ?? 60_000, + redactionValues: options.redactionValues, + }); +} + +async function cleanupBraveSandbox(sandbox: SandboxClient): Promise { + await bestEffort(() => + sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { + artifactName: "cleanup-openshell-delete-brave-search", + env: commandEnv(), + timeoutMs: 60_000, + }), + ); +} + +function parsePlaceholder(configText: string): string | undefined { + const parsed = JSON.parse(configText) as { + tools?: { web?: { search?: { apiKey?: unknown } } }; + }; + const value = parsed.tools?.web?.search?.apiKey; + return typeof value === "string" && value ? value : undefined; +} + +function firstJsonObject(output: string): unknown { + for (let start = output.indexOf("{"); start >= 0; start = output.indexOf("{", start + 1)) { + let depth = 0; + let inString = false; + let escaped = false; + for (let index = start; index < output.length; index += 1) { + const char = output[index]; + if (inString) { + if (escaped) escaped = false; + else if (char === "\\") escaped = true; + else if (char === '"') inString = false; + continue; + } + if (char === '"') inString = true; + else if (char === "{") depth += 1; + else if (char === "}") { + depth -= 1; + if (depth === 0) { + try { + return JSON.parse(output.slice(start, index + 1)); + } catch { + break; + } + } + } + } + } + return undefined; +} + +function collectAssistantText(value: unknown): string[] { + if (typeof value === "string" && value.trim()) return [value.trim()]; + if (!value || typeof value !== "object") return []; + if (Array.isArray(value)) return value.flatMap(collectAssistantText); + const record = value as Record; + const texts: string[] = []; + for (const key of [ + "result", + "payloads", + "messages", + "choices", + "message", + "delta", + "content", + "text", + ]) { + if (key in record) texts.push(...collectAssistantText(record[key])); + } + return texts; +} + +/** + * Source boundary: `openclaw agent --json` may emit launcher progress before + * and after the final JSON envelope. This mirrors the retained legacy shell + * parser's tolerant envelope handling until OpenClaw exposes a stable + * JSON-only stdout contract for live E2E consumers. + */ +function extractOpenClawAgentText(output: string): string { + const parsed = firstJsonObject(output); + return collectAssistantText(parsed)[0] ?? ""; +} + +test.skipIf(!shouldRunLiveE2EScenarios())( + "Brave search preset wires policy/config, hides the real key, and performs real searches (#2687)", + { timeout: LIVE_TIMEOUT_MS }, + async ({ artifacts, cleanup, host, sandbox, secrets, skip }) => { + const braveKey = secrets.required("BRAVE_API_KEY"); + const inferenceKey = secrets.required("NVIDIA_INFERENCE_API_KEY"); + const redactionValues = [braveKey, inferenceKey]; + + await artifacts.writeJson("scenario.json", { + id: "brave-search", + runner: "vitest", + legacySource: "test/e2e/test-brave-search-e2e.sh", + boundary: + "source CLI onboard + OpenShell policy/config + in-sandbox OpenClaw/Brave API calls", + sandboxName: SANDBOX_NAME, + contracts: [ + "onboard succeeds with BRAVE_API_KEY present", + "the brave network policy preset includes api.search.brave.com", + "OpenClaw web search config is enabled and selects provider=brave", + "the real BRAVE_API_KEY is absent from openclaw.json and sandbox shell env", + "OpenClaw agent can perform a Brave-backed web search", + "curl from inside the sandbox can query Brave using the placeholder token header", + ], + }); + + const dockerInfo = await host.command("docker", ["info"], { + artifactName: "phase-0-docker-info", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }); + if (dockerInfo.exitCode !== 0) { + if (process.env.GITHUB_ACTIONS === "true") { + throw new Error(`Docker is required for Brave search E2E: ${resultText(dockerInfo)}`); + } + skip(`Docker is required for Brave search E2E: ${resultText(dockerInfo)}`); + } + + cleanup.add(`destroy brave search sandbox ${SANDBOX_NAME}`, async () => { + await bestEffort(() => + host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { + artifactName: "cleanup-nemoclaw-destroy-brave-search", + env: commandEnv(), + timeoutMs: 120_000, + }), + ); + await cleanupBraveSandbox(sandbox); + }); + + await bestEffort(() => + host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { + artifactName: "pre-cleanup-nemoclaw-destroy-brave-search", + env: commandEnv(), + timeoutMs: 120_000, + }), + ); + await cleanupBraveSandbox(sandbox); + + let onboard: ShellProbeResult | undefined; + for (let attempt = 1; attempt <= INSTALL_ATTEMPTS; attempt += 1) { + onboard = await host.command( + "node", + [ + CLI_ENTRYPOINT, + "onboard", + "--fresh", + "--non-interactive", + "--yes-i-accept-third-party-software", + ], + { + artifactName: + attempt === 1 + ? "phase-1-onboard-brave-search" + : `phase-1-onboard-brave-search-attempt-${attempt}`, + cwd: REPO_ROOT, + env: commandEnv({ + BRAVE_API_KEY: braveKey, + NVIDIA_INFERENCE_API_KEY: inferenceKey, + NVIDIA_API_KEY: inferenceKey, + }), + redactionValues, + timeoutMs: 20 * 60_000, + }, + ); + if (onboard.exitCode === 0) break; + if (isTransientProviderValidationFailure(onboard) && attempt < INSTALL_ATTEMPTS) { + await new Promise((resolve) => setTimeout(resolve, 10_000 * attempt)); + continue; + } + break; + } + expect(onboard, "onboard command must run").toBeDefined(); + expect(onboard?.exitCode, resultText(onboard as ShellProbeResult)).toBe(0); + + const policy = await sandbox.openshell(["policy", "get", "--full", SANDBOX_NAME], { + artifactName: "phase-2-brave-policy", + env: commandEnv(), + timeoutMs: 60_000, + }); + expect(policy.exitCode, resultText(policy)).toBe(0); + expect(resultText(policy)).toContain("api.search.brave.com"); + + const config = await sandbox.exec(SANDBOX_NAME, ["cat", "/sandbox/.openclaw/openclaw.json"], { + artifactName: "phase-2-openclaw-config", + env: commandEnv(), + redactionValues, + timeoutMs: 60_000, + }); + expect(config.exitCode, resultText(config)).toBe(0); + const secretDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-brave-secret-")); + const secretFile = path.join(secretDir, "brave-key"); + fs.writeFileSync(secretFile, braveKey, { mode: 0o600 }); + const remoteSecretFile = "/tmp/nemoclaw-brave-key-leak-check"; + cleanup.add("remove temporary Brave leak-check secret", async () => { + fs.rmSync(secretDir, { recursive: true, force: true }); + await bestEffort(() => + sandbox.execShell(SANDBOX_NAME, trustedSandboxShellScript(`rm -f ${remoteSecretFile}`), { + artifactName: "cleanup-brave-leak-secret", + env: commandEnv(), + timeoutMs: 30_000, + }), + ); + }); + const uploadSecret = await sandbox.upload(SANDBOX_NAME, secretFile, remoteSecretFile, { + artifactName: "phase-3-upload-brave-leak-secret", + env: commandEnv(), + redactionValues, + timeoutMs: 30_000, + }); + expect(uploadSecret.exitCode, resultText(uploadSecret)).toBe(0); + const rawLeakCheck = await sandbox.execShell( + SANDBOX_NAME, + trustedSandboxShellScript( + `python3 - <<'PY' +from pathlib import Path +needle = Path('${remoteSecretFile}').read_text(encoding='utf-8') +body = Path('/sandbox/.openclaw/openclaw.json').read_text(encoding='utf-8') +raise SystemExit(1 if needle in body else 0) +PY`, + ), + { + artifactName: "phase-3-openclaw-config-raw-secret-leak-check", + env: commandEnv(), + timeoutMs: 30_000, + }, + ); + expect( + rawLeakCheck.exitCode, + "raw BRAVE_API_KEY must not appear anywhere in openclaw.json", + ).toBe(0); + const parsedConfig = JSON.parse(config.stdout) as { + tools?: { web?: { search?: { enabled?: unknown; provider?: unknown; apiKey?: unknown } } }; + }; + const searchConfig = parsedConfig.tools?.web?.search; + expect(searchConfig?.enabled, config.stdout).toBe(true); + expect(searchConfig?.provider, config.stdout).toBe("brave"); + const placeholder = parsePlaceholder(config.stdout); + expect(placeholder, config.stdout).toMatch(PLACEHOLDER_PATTERN); + + const envCheck = await sandbox.exec( + SANDBOX_NAME, + ["sh", "-lc", "printenv BRAVE_API_KEY || true"], + { + artifactName: "phase-3-sandbox-brave-env", + env: commandEnv(), + redactionValues, + timeoutMs: 30_000, + }, + ); + expect(envCheck.exitCode, resultText(envCheck)).toBe(0); + expect(envCheck.stdout).not.toContain(braveKey); + if (envCheck.stdout.trim()) expect(envCheck.stdout.trim()).toMatch(PLACEHOLDER_PATTERN); + + const agent = await sandboxShell( + sandbox, + `openclaw agent --agent main --json --session-id e2e-brave-agent-$$ -m 'Use the web search tool to find one result for the query: NVIDIA. Reply with only the title of the top result.'`, + { + artifactName: "phase-4a-agent-web-search", + timeoutMs: 150_000, + redactionValues, + }, + ); + expect(resultText(agent)).not.toMatch( + /SsrFBlockedError|Blocked hostname|ECONNREFUSED|EAI_AGAIN|gateway unavailable|network connection error/i, + ); + expect(agent.exitCode, resultText(agent)).toBe(0); + const assistantText = extractOpenClawAgentText(agent.stdout); + expect(assistantText, resultText(agent)).toMatch(/nvidia|geforce|cuda|gpu/i); + + const curl = await sandboxShell( + sandbox, + `curl -sS --max-time 20 -G 'https://api.search.brave.com/res/v1/web/search' --data-urlencode 'q=NVIDIA' --data-urlencode 'count=1' -H 'X-Subscription-Token: ${placeholder}' -w '\nHTTP_STATUS:%{http_code}\n'`, + { + artifactName: "phase-4b-direct-brave-curl", + timeoutMs: 60_000, + redactionValues, + }, + ); + const status = resultText(curl).match(/HTTP_STATUS:(\d{3})/)?.[1]; + expect(status, resultText(curl)).toBe("200"); + const body = resultText(curl).replace(/\n?HTTP_STATUS:\d{3}\s*$/u, ""); + const braveResponse = JSON.parse(body) as { web?: { results?: unknown[] } }; + expect(braveResponse.web?.results?.length ?? 0, body.slice(0, 500)).toBeGreaterThan(0); + }, +); diff --git a/test/e2e-scenario/live/brave-search.test.ts b/test/e2e-scenario/live/brave-search.test.ts index cfe433d4a41..683a3c1468c 100644 --- a/test/e2e-scenario/live/brave-search.test.ts +++ b/test/e2e-scenario/live/brave-search.test.ts @@ -1,358 +1,4 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -/** - * Live Vitest replacement for test/e2e/test-brave-search-e2e.sh. - * - * Preserves the legacy #2687 acceptance boundary: non-interactive onboard with - * a real BRAVE_API_KEY, brave policy/config wiring, secret non-leak checks, - * a real agent web-search turn, and a direct in-sandbox Brave API curl using - * the OpenShell credential placeholder. - */ - -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; - -import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; -import { resultText } from "../fixtures/clients/index.ts"; -import { - type SandboxClient, - trustedSandboxShellScript, - validateSandboxName, -} from "../fixtures/clients/sandbox.ts"; -import { expect, test } from "../fixtures/e2e-test.ts"; -import { shouldRunLiveE2EScenarios } from "../fixtures/live-project-gate.ts"; -import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; -import { isTransientProviderValidationFailure } from "./network-policy-transient-provider.ts"; - -const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); -const CLI_ENTRYPOINT = path.join(REPO_ROOT, "bin", "nemoclaw.js"); -const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-brave-search"; -validateSandboxName(SANDBOX_NAME); -const INSTALL_ATTEMPTS = process.env.CI === "true" || process.env.GITHUB_ACTIONS === "true" ? 3 : 1; -const LIVE_TIMEOUT_MS = 35 * 60_000; -const PLACEHOLDER_PATTERN = /^openshell:resolve:env:([A-Za-z0-9_]+_)?BRAVE_API_KEY$/; - -function commandEnv(extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { - return { - ...buildAvailabilityProbeEnv(), - NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", - NEMOCLAW_NON_INTERACTIVE: "1", - NEMOCLAW_RECREATE_SANDBOX: "1", - NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, - OPENSHELL_GATEWAY: process.env.OPENSHELL_GATEWAY ?? "nemoclaw", - ...extra, - }; -} - -async function bestEffort(run: () => Promise): Promise { - try { - await run(); - } catch { - // Cleanup should not mask primary failures. - } -} - -function singleLineShell(script: string): string { - const encoded = Buffer.from(script, "utf8").toString("base64"); - return `tmp=$(mktemp); trap 'rm -f "$tmp"' EXIT; printf %s '${encoded}' | base64 -d > "$tmp"; sh "$tmp"`; -} - -async function sandboxShell( - sandbox: SandboxClient, - script: string, - options: { artifactName: string; timeoutMs?: number; redactionValues?: string[] }, -): Promise { - return await sandbox.execShell(SANDBOX_NAME, trustedSandboxShellScript(singleLineShell(script)), { - artifactName: options.artifactName, - env: commandEnv(), - timeoutMs: options.timeoutMs ?? 60_000, - redactionValues: options.redactionValues, - }); -} - -async function cleanupBraveSandbox(sandbox: SandboxClient): Promise { - await bestEffort(() => - sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { - artifactName: "cleanup-openshell-delete-brave-search", - env: commandEnv(), - timeoutMs: 60_000, - }), - ); -} - -function parsePlaceholder(configText: string): string | undefined { - const parsed = JSON.parse(configText) as { - tools?: { web?: { search?: { apiKey?: unknown } } }; - }; - const value = parsed.tools?.web?.search?.apiKey; - return typeof value === "string" && value ? value : undefined; -} - -function firstJsonObject(output: string): unknown { - for (let start = output.indexOf("{"); start >= 0; start = output.indexOf("{", start + 1)) { - let depth = 0; - let inString = false; - let escaped = false; - for (let index = start; index < output.length; index += 1) { - const char = output[index]; - if (inString) { - if (escaped) escaped = false; - else if (char === "\\") escaped = true; - else if (char === '"') inString = false; - continue; - } - if (char === '"') inString = true; - else if (char === "{") depth += 1; - else if (char === "}") { - depth -= 1; - if (depth === 0) { - try { - return JSON.parse(output.slice(start, index + 1)); - } catch { - break; - } - } - } - } - } - return undefined; -} - -function collectAssistantText(value: unknown): string[] { - if (typeof value === "string" && value.trim()) return [value.trim()]; - if (!value || typeof value !== "object") return []; - if (Array.isArray(value)) return value.flatMap(collectAssistantText); - const record = value as Record; - const texts: string[] = []; - for (const key of [ - "result", - "payloads", - "messages", - "choices", - "message", - "delta", - "content", - "text", - ]) { - if (key in record) texts.push(...collectAssistantText(record[key])); - } - return texts; -} - -/** - * Source boundary: `openclaw agent --json` may emit launcher progress before - * and after the final JSON envelope. This mirrors the retained legacy shell - * parser's tolerant envelope handling until OpenClaw exposes a stable - * JSON-only stdout contract for live E2E consumers. - */ -function extractOpenClawAgentText(output: string): string { - const parsed = firstJsonObject(output); - return collectAssistantText(parsed)[0] ?? ""; -} - -test.skipIf(!shouldRunLiveE2EScenarios())( - "Brave search preset wires policy/config, hides the real key, and performs real searches (#2687)", - { timeout: LIVE_TIMEOUT_MS }, - async ({ artifacts, cleanup, host, sandbox, secrets, skip }) => { - const braveKey = secrets.required("BRAVE_API_KEY"); - const inferenceKey = secrets.required("NVIDIA_INFERENCE_API_KEY"); - const redactionValues = [braveKey, inferenceKey]; - - await artifacts.writeJson("scenario.json", { - id: "brave-search", - runner: "vitest", - legacySource: "test/e2e/test-brave-search-e2e.sh", - boundary: - "source CLI onboard + OpenShell policy/config + in-sandbox OpenClaw/Brave API calls", - sandboxName: SANDBOX_NAME, - contracts: [ - "onboard succeeds with BRAVE_API_KEY present", - "the brave network policy preset includes api.search.brave.com", - "OpenClaw web search config is enabled and selects provider=brave", - "the real BRAVE_API_KEY is absent from openclaw.json and sandbox shell env", - "OpenClaw agent can perform a Brave-backed web search", - "curl from inside the sandbox can query Brave using the placeholder token header", - ], - }); - - const dockerInfo = await host.command("docker", ["info"], { - artifactName: "phase-0-docker-info", - env: buildAvailabilityProbeEnv(), - timeoutMs: 30_000, - }); - if (dockerInfo.exitCode !== 0) { - if (process.env.GITHUB_ACTIONS === "true") { - throw new Error(`Docker is required for Brave search E2E: ${resultText(dockerInfo)}`); - } - skip(`Docker is required for Brave search E2E: ${resultText(dockerInfo)}`); - } - - cleanup.add(`destroy brave search sandbox ${SANDBOX_NAME}`, async () => { - await bestEffort(() => - host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { - artifactName: "cleanup-nemoclaw-destroy-brave-search", - env: commandEnv(), - timeoutMs: 120_000, - }), - ); - await cleanupBraveSandbox(sandbox); - }); - - await bestEffort(() => - host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { - artifactName: "pre-cleanup-nemoclaw-destroy-brave-search", - env: commandEnv(), - timeoutMs: 120_000, - }), - ); - await cleanupBraveSandbox(sandbox); - - let onboard: ShellProbeResult | undefined; - for (let attempt = 1; attempt <= INSTALL_ATTEMPTS; attempt += 1) { - onboard = await host.command( - "node", - [ - CLI_ENTRYPOINT, - "onboard", - "--fresh", - "--non-interactive", - "--yes-i-accept-third-party-software", - ], - { - artifactName: - attempt === 1 - ? "phase-1-onboard-brave-search" - : `phase-1-onboard-brave-search-attempt-${attempt}`, - cwd: REPO_ROOT, - env: commandEnv({ - BRAVE_API_KEY: braveKey, - NVIDIA_INFERENCE_API_KEY: inferenceKey, - NVIDIA_API_KEY: inferenceKey, - }), - redactionValues, - timeoutMs: 20 * 60_000, - }, - ); - if (onboard.exitCode === 0) break; - if (isTransientProviderValidationFailure(onboard) && attempt < INSTALL_ATTEMPTS) { - await new Promise((resolve) => setTimeout(resolve, 10_000 * attempt)); - continue; - } - break; - } - expect(onboard, "onboard command must run").toBeDefined(); - expect(onboard?.exitCode, resultText(onboard as ShellProbeResult)).toBe(0); - - const policy = await sandbox.openshell(["policy", "get", "--full", SANDBOX_NAME], { - artifactName: "phase-2-brave-policy", - env: commandEnv(), - timeoutMs: 60_000, - }); - expect(policy.exitCode, resultText(policy)).toBe(0); - expect(resultText(policy)).toContain("api.search.brave.com"); - - const config = await sandbox.exec(SANDBOX_NAME, ["cat", "/sandbox/.openclaw/openclaw.json"], { - artifactName: "phase-2-openclaw-config", - env: commandEnv(), - redactionValues, - timeoutMs: 60_000, - }); - expect(config.exitCode, resultText(config)).toBe(0); - const secretDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-brave-secret-")); - const secretFile = path.join(secretDir, "brave-key"); - fs.writeFileSync(secretFile, braveKey, { mode: 0o600 }); - const remoteSecretFile = "/tmp/nemoclaw-brave-key-leak-check"; - cleanup.add("remove temporary Brave leak-check secret", async () => { - fs.rmSync(secretDir, { recursive: true, force: true }); - await bestEffort(() => - sandbox.execShell(SANDBOX_NAME, trustedSandboxShellScript(`rm -f ${remoteSecretFile}`), { - artifactName: "cleanup-brave-leak-secret", - env: commandEnv(), - timeoutMs: 30_000, - }), - ); - }); - const uploadSecret = await sandbox.upload(SANDBOX_NAME, secretFile, remoteSecretFile, { - artifactName: "phase-3-upload-brave-leak-secret", - env: commandEnv(), - redactionValues, - timeoutMs: 30_000, - }); - expect(uploadSecret.exitCode, resultText(uploadSecret)).toBe(0); - const rawLeakCheck = await sandbox.execShell( - SANDBOX_NAME, - trustedSandboxShellScript( - `python3 - <<'PY' -from pathlib import Path -needle = Path('${remoteSecretFile}').read_text(encoding='utf-8') -body = Path('/sandbox/.openclaw/openclaw.json').read_text(encoding='utf-8') -raise SystemExit(1 if needle in body else 0) -PY`, - ), - { - artifactName: "phase-3-openclaw-config-raw-secret-leak-check", - env: commandEnv(), - timeoutMs: 30_000, - }, - ); - expect( - rawLeakCheck.exitCode, - "raw BRAVE_API_KEY must not appear anywhere in openclaw.json", - ).toBe(0); - const parsedConfig = JSON.parse(config.stdout) as { - tools?: { web?: { search?: { enabled?: unknown; provider?: unknown; apiKey?: unknown } } }; - }; - const searchConfig = parsedConfig.tools?.web?.search; - expect(searchConfig?.enabled, config.stdout).toBe(true); - expect(searchConfig?.provider, config.stdout).toBe("brave"); - const placeholder = parsePlaceholder(config.stdout); - expect(placeholder, config.stdout).toMatch(PLACEHOLDER_PATTERN); - - const envCheck = await sandbox.exec( - SANDBOX_NAME, - ["sh", "-lc", "printenv BRAVE_API_KEY || true"], - { - artifactName: "phase-3-sandbox-brave-env", - env: commandEnv(), - redactionValues, - timeoutMs: 30_000, - }, - ); - expect(envCheck.exitCode, resultText(envCheck)).toBe(0); - expect(envCheck.stdout).not.toContain(braveKey); - if (envCheck.stdout.trim()) expect(envCheck.stdout.trim()).toMatch(PLACEHOLDER_PATTERN); - - const agent = await sandboxShell( - sandbox, - `openclaw agent --agent main --json --session-id e2e-brave-agent-$$ -m 'Use the web search tool to find one result for the query: NVIDIA. Reply with only the title of the top result.'`, - { - artifactName: "phase-4a-agent-web-search", - timeoutMs: 150_000, - redactionValues, - }, - ); - expect(resultText(agent)).not.toMatch( - /SsrFBlockedError|Blocked hostname|ECONNREFUSED|EAI_AGAIN|gateway unavailable|network connection error/i, - ); - expect(agent.exitCode, resultText(agent)).toBe(0); - const assistantText = extractOpenClawAgentText(agent.stdout); - expect(assistantText, resultText(agent)).toMatch(/nvidia|geforce|cuda|gpu/i); - - const curl = await sandboxShell( - sandbox, - `curl -sS --max-time 20 -G 'https://api.search.brave.com/res/v1/web/search' --data-urlencode 'q=NVIDIA' --data-urlencode 'count=1' -H 'X-Subscription-Token: ${placeholder}' -w '\nHTTP_STATUS:%{http_code}\n'`, - { - artifactName: "phase-4b-direct-brave-curl", - timeoutMs: 60_000, - redactionValues, - }, - ); - const status = resultText(curl).match(/HTTP_STATUS:(\d{3})/)?.[1]; - expect(status, resultText(curl)).toBe("200"); - const body = resultText(curl).replace(/\n?HTTP_STATUS:\d{3}\s*$/u, ""); - const braveResponse = JSON.parse(body) as { web?: { results?: unknown[] } }; - expect(braveResponse.web?.results?.length ?? 0, body.slice(0, 500)).toBeGreaterThan(0); - }, -); +import "./brave-search.scenario.ts"; From 38092b1ffbcbb3ffda5519a3e8006c6c90e78f9a Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sat, 20 Jun 2026 09:28:04 -0700 Subject: [PATCH 7/8] test(e2e): restore scenario logic to test file Signed-off-by: Carlos Villela --- .../live/brave-search.scenario.ts | 358 ------------------ test/e2e-scenario/live/brave-search.test.ts | 356 ++++++++++++++++- 2 files changed, 355 insertions(+), 359 deletions(-) delete mode 100644 test/e2e-scenario/live/brave-search.scenario.ts diff --git a/test/e2e-scenario/live/brave-search.scenario.ts b/test/e2e-scenario/live/brave-search.scenario.ts deleted file mode 100644 index cfe433d4a41..00000000000 --- a/test/e2e-scenario/live/brave-search.scenario.ts +++ /dev/null @@ -1,358 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -/** - * Live Vitest replacement for test/e2e/test-brave-search-e2e.sh. - * - * Preserves the legacy #2687 acceptance boundary: non-interactive onboard with - * a real BRAVE_API_KEY, brave policy/config wiring, secret non-leak checks, - * a real agent web-search turn, and a direct in-sandbox Brave API curl using - * the OpenShell credential placeholder. - */ - -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; - -import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; -import { resultText } from "../fixtures/clients/index.ts"; -import { - type SandboxClient, - trustedSandboxShellScript, - validateSandboxName, -} from "../fixtures/clients/sandbox.ts"; -import { expect, test } from "../fixtures/e2e-test.ts"; -import { shouldRunLiveE2EScenarios } from "../fixtures/live-project-gate.ts"; -import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; -import { isTransientProviderValidationFailure } from "./network-policy-transient-provider.ts"; - -const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); -const CLI_ENTRYPOINT = path.join(REPO_ROOT, "bin", "nemoclaw.js"); -const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-brave-search"; -validateSandboxName(SANDBOX_NAME); -const INSTALL_ATTEMPTS = process.env.CI === "true" || process.env.GITHUB_ACTIONS === "true" ? 3 : 1; -const LIVE_TIMEOUT_MS = 35 * 60_000; -const PLACEHOLDER_PATTERN = /^openshell:resolve:env:([A-Za-z0-9_]+_)?BRAVE_API_KEY$/; - -function commandEnv(extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { - return { - ...buildAvailabilityProbeEnv(), - NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", - NEMOCLAW_NON_INTERACTIVE: "1", - NEMOCLAW_RECREATE_SANDBOX: "1", - NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, - OPENSHELL_GATEWAY: process.env.OPENSHELL_GATEWAY ?? "nemoclaw", - ...extra, - }; -} - -async function bestEffort(run: () => Promise): Promise { - try { - await run(); - } catch { - // Cleanup should not mask primary failures. - } -} - -function singleLineShell(script: string): string { - const encoded = Buffer.from(script, "utf8").toString("base64"); - return `tmp=$(mktemp); trap 'rm -f "$tmp"' EXIT; printf %s '${encoded}' | base64 -d > "$tmp"; sh "$tmp"`; -} - -async function sandboxShell( - sandbox: SandboxClient, - script: string, - options: { artifactName: string; timeoutMs?: number; redactionValues?: string[] }, -): Promise { - return await sandbox.execShell(SANDBOX_NAME, trustedSandboxShellScript(singleLineShell(script)), { - artifactName: options.artifactName, - env: commandEnv(), - timeoutMs: options.timeoutMs ?? 60_000, - redactionValues: options.redactionValues, - }); -} - -async function cleanupBraveSandbox(sandbox: SandboxClient): Promise { - await bestEffort(() => - sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { - artifactName: "cleanup-openshell-delete-brave-search", - env: commandEnv(), - timeoutMs: 60_000, - }), - ); -} - -function parsePlaceholder(configText: string): string | undefined { - const parsed = JSON.parse(configText) as { - tools?: { web?: { search?: { apiKey?: unknown } } }; - }; - const value = parsed.tools?.web?.search?.apiKey; - return typeof value === "string" && value ? value : undefined; -} - -function firstJsonObject(output: string): unknown { - for (let start = output.indexOf("{"); start >= 0; start = output.indexOf("{", start + 1)) { - let depth = 0; - let inString = false; - let escaped = false; - for (let index = start; index < output.length; index += 1) { - const char = output[index]; - if (inString) { - if (escaped) escaped = false; - else if (char === "\\") escaped = true; - else if (char === '"') inString = false; - continue; - } - if (char === '"') inString = true; - else if (char === "{") depth += 1; - else if (char === "}") { - depth -= 1; - if (depth === 0) { - try { - return JSON.parse(output.slice(start, index + 1)); - } catch { - break; - } - } - } - } - } - return undefined; -} - -function collectAssistantText(value: unknown): string[] { - if (typeof value === "string" && value.trim()) return [value.trim()]; - if (!value || typeof value !== "object") return []; - if (Array.isArray(value)) return value.flatMap(collectAssistantText); - const record = value as Record; - const texts: string[] = []; - for (const key of [ - "result", - "payloads", - "messages", - "choices", - "message", - "delta", - "content", - "text", - ]) { - if (key in record) texts.push(...collectAssistantText(record[key])); - } - return texts; -} - -/** - * Source boundary: `openclaw agent --json` may emit launcher progress before - * and after the final JSON envelope. This mirrors the retained legacy shell - * parser's tolerant envelope handling until OpenClaw exposes a stable - * JSON-only stdout contract for live E2E consumers. - */ -function extractOpenClawAgentText(output: string): string { - const parsed = firstJsonObject(output); - return collectAssistantText(parsed)[0] ?? ""; -} - -test.skipIf(!shouldRunLiveE2EScenarios())( - "Brave search preset wires policy/config, hides the real key, and performs real searches (#2687)", - { timeout: LIVE_TIMEOUT_MS }, - async ({ artifacts, cleanup, host, sandbox, secrets, skip }) => { - const braveKey = secrets.required("BRAVE_API_KEY"); - const inferenceKey = secrets.required("NVIDIA_INFERENCE_API_KEY"); - const redactionValues = [braveKey, inferenceKey]; - - await artifacts.writeJson("scenario.json", { - id: "brave-search", - runner: "vitest", - legacySource: "test/e2e/test-brave-search-e2e.sh", - boundary: - "source CLI onboard + OpenShell policy/config + in-sandbox OpenClaw/Brave API calls", - sandboxName: SANDBOX_NAME, - contracts: [ - "onboard succeeds with BRAVE_API_KEY present", - "the brave network policy preset includes api.search.brave.com", - "OpenClaw web search config is enabled and selects provider=brave", - "the real BRAVE_API_KEY is absent from openclaw.json and sandbox shell env", - "OpenClaw agent can perform a Brave-backed web search", - "curl from inside the sandbox can query Brave using the placeholder token header", - ], - }); - - const dockerInfo = await host.command("docker", ["info"], { - artifactName: "phase-0-docker-info", - env: buildAvailabilityProbeEnv(), - timeoutMs: 30_000, - }); - if (dockerInfo.exitCode !== 0) { - if (process.env.GITHUB_ACTIONS === "true") { - throw new Error(`Docker is required for Brave search E2E: ${resultText(dockerInfo)}`); - } - skip(`Docker is required for Brave search E2E: ${resultText(dockerInfo)}`); - } - - cleanup.add(`destroy brave search sandbox ${SANDBOX_NAME}`, async () => { - await bestEffort(() => - host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { - artifactName: "cleanup-nemoclaw-destroy-brave-search", - env: commandEnv(), - timeoutMs: 120_000, - }), - ); - await cleanupBraveSandbox(sandbox); - }); - - await bestEffort(() => - host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { - artifactName: "pre-cleanup-nemoclaw-destroy-brave-search", - env: commandEnv(), - timeoutMs: 120_000, - }), - ); - await cleanupBraveSandbox(sandbox); - - let onboard: ShellProbeResult | undefined; - for (let attempt = 1; attempt <= INSTALL_ATTEMPTS; attempt += 1) { - onboard = await host.command( - "node", - [ - CLI_ENTRYPOINT, - "onboard", - "--fresh", - "--non-interactive", - "--yes-i-accept-third-party-software", - ], - { - artifactName: - attempt === 1 - ? "phase-1-onboard-brave-search" - : `phase-1-onboard-brave-search-attempt-${attempt}`, - cwd: REPO_ROOT, - env: commandEnv({ - BRAVE_API_KEY: braveKey, - NVIDIA_INFERENCE_API_KEY: inferenceKey, - NVIDIA_API_KEY: inferenceKey, - }), - redactionValues, - timeoutMs: 20 * 60_000, - }, - ); - if (onboard.exitCode === 0) break; - if (isTransientProviderValidationFailure(onboard) && attempt < INSTALL_ATTEMPTS) { - await new Promise((resolve) => setTimeout(resolve, 10_000 * attempt)); - continue; - } - break; - } - expect(onboard, "onboard command must run").toBeDefined(); - expect(onboard?.exitCode, resultText(onboard as ShellProbeResult)).toBe(0); - - const policy = await sandbox.openshell(["policy", "get", "--full", SANDBOX_NAME], { - artifactName: "phase-2-brave-policy", - env: commandEnv(), - timeoutMs: 60_000, - }); - expect(policy.exitCode, resultText(policy)).toBe(0); - expect(resultText(policy)).toContain("api.search.brave.com"); - - const config = await sandbox.exec(SANDBOX_NAME, ["cat", "/sandbox/.openclaw/openclaw.json"], { - artifactName: "phase-2-openclaw-config", - env: commandEnv(), - redactionValues, - timeoutMs: 60_000, - }); - expect(config.exitCode, resultText(config)).toBe(0); - const secretDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-brave-secret-")); - const secretFile = path.join(secretDir, "brave-key"); - fs.writeFileSync(secretFile, braveKey, { mode: 0o600 }); - const remoteSecretFile = "/tmp/nemoclaw-brave-key-leak-check"; - cleanup.add("remove temporary Brave leak-check secret", async () => { - fs.rmSync(secretDir, { recursive: true, force: true }); - await bestEffort(() => - sandbox.execShell(SANDBOX_NAME, trustedSandboxShellScript(`rm -f ${remoteSecretFile}`), { - artifactName: "cleanup-brave-leak-secret", - env: commandEnv(), - timeoutMs: 30_000, - }), - ); - }); - const uploadSecret = await sandbox.upload(SANDBOX_NAME, secretFile, remoteSecretFile, { - artifactName: "phase-3-upload-brave-leak-secret", - env: commandEnv(), - redactionValues, - timeoutMs: 30_000, - }); - expect(uploadSecret.exitCode, resultText(uploadSecret)).toBe(0); - const rawLeakCheck = await sandbox.execShell( - SANDBOX_NAME, - trustedSandboxShellScript( - `python3 - <<'PY' -from pathlib import Path -needle = Path('${remoteSecretFile}').read_text(encoding='utf-8') -body = Path('/sandbox/.openclaw/openclaw.json').read_text(encoding='utf-8') -raise SystemExit(1 if needle in body else 0) -PY`, - ), - { - artifactName: "phase-3-openclaw-config-raw-secret-leak-check", - env: commandEnv(), - timeoutMs: 30_000, - }, - ); - expect( - rawLeakCheck.exitCode, - "raw BRAVE_API_KEY must not appear anywhere in openclaw.json", - ).toBe(0); - const parsedConfig = JSON.parse(config.stdout) as { - tools?: { web?: { search?: { enabled?: unknown; provider?: unknown; apiKey?: unknown } } }; - }; - const searchConfig = parsedConfig.tools?.web?.search; - expect(searchConfig?.enabled, config.stdout).toBe(true); - expect(searchConfig?.provider, config.stdout).toBe("brave"); - const placeholder = parsePlaceholder(config.stdout); - expect(placeholder, config.stdout).toMatch(PLACEHOLDER_PATTERN); - - const envCheck = await sandbox.exec( - SANDBOX_NAME, - ["sh", "-lc", "printenv BRAVE_API_KEY || true"], - { - artifactName: "phase-3-sandbox-brave-env", - env: commandEnv(), - redactionValues, - timeoutMs: 30_000, - }, - ); - expect(envCheck.exitCode, resultText(envCheck)).toBe(0); - expect(envCheck.stdout).not.toContain(braveKey); - if (envCheck.stdout.trim()) expect(envCheck.stdout.trim()).toMatch(PLACEHOLDER_PATTERN); - - const agent = await sandboxShell( - sandbox, - `openclaw agent --agent main --json --session-id e2e-brave-agent-$$ -m 'Use the web search tool to find one result for the query: NVIDIA. Reply with only the title of the top result.'`, - { - artifactName: "phase-4a-agent-web-search", - timeoutMs: 150_000, - redactionValues, - }, - ); - expect(resultText(agent)).not.toMatch( - /SsrFBlockedError|Blocked hostname|ECONNREFUSED|EAI_AGAIN|gateway unavailable|network connection error/i, - ); - expect(agent.exitCode, resultText(agent)).toBe(0); - const assistantText = extractOpenClawAgentText(agent.stdout); - expect(assistantText, resultText(agent)).toMatch(/nvidia|geforce|cuda|gpu/i); - - const curl = await sandboxShell( - sandbox, - `curl -sS --max-time 20 -G 'https://api.search.brave.com/res/v1/web/search' --data-urlencode 'q=NVIDIA' --data-urlencode 'count=1' -H 'X-Subscription-Token: ${placeholder}' -w '\nHTTP_STATUS:%{http_code}\n'`, - { - artifactName: "phase-4b-direct-brave-curl", - timeoutMs: 60_000, - redactionValues, - }, - ); - const status = resultText(curl).match(/HTTP_STATUS:(\d{3})/)?.[1]; - expect(status, resultText(curl)).toBe("200"); - const body = resultText(curl).replace(/\n?HTTP_STATUS:\d{3}\s*$/u, ""); - const braveResponse = JSON.parse(body) as { web?: { results?: unknown[] } }; - expect(braveResponse.web?.results?.length ?? 0, body.slice(0, 500)).toBeGreaterThan(0); - }, -); diff --git a/test/e2e-scenario/live/brave-search.test.ts b/test/e2e-scenario/live/brave-search.test.ts index 683a3c1468c..cfe433d4a41 100644 --- a/test/e2e-scenario/live/brave-search.test.ts +++ b/test/e2e-scenario/live/brave-search.test.ts @@ -1,4 +1,358 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import "./brave-search.scenario.ts"; +/** + * Live Vitest replacement for test/e2e/test-brave-search-e2e.sh. + * + * Preserves the legacy #2687 acceptance boundary: non-interactive onboard with + * a real BRAVE_API_KEY, brave policy/config wiring, secret non-leak checks, + * a real agent web-search turn, and a direct in-sandbox Brave API curl using + * the OpenShell credential placeholder. + */ + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { resultText } from "../fixtures/clients/index.ts"; +import { + type SandboxClient, + trustedSandboxShellScript, + validateSandboxName, +} from "../fixtures/clients/sandbox.ts"; +import { expect, test } from "../fixtures/e2e-test.ts"; +import { shouldRunLiveE2EScenarios } from "../fixtures/live-project-gate.ts"; +import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; +import { isTransientProviderValidationFailure } from "./network-policy-transient-provider.ts"; + +const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); +const CLI_ENTRYPOINT = path.join(REPO_ROOT, "bin", "nemoclaw.js"); +const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-brave-search"; +validateSandboxName(SANDBOX_NAME); +const INSTALL_ATTEMPTS = process.env.CI === "true" || process.env.GITHUB_ACTIONS === "true" ? 3 : 1; +const LIVE_TIMEOUT_MS = 35 * 60_000; +const PLACEHOLDER_PATTERN = /^openshell:resolve:env:([A-Za-z0-9_]+_)?BRAVE_API_KEY$/; + +function commandEnv(extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { + return { + ...buildAvailabilityProbeEnv(), + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_RECREATE_SANDBOX: "1", + NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, + OPENSHELL_GATEWAY: process.env.OPENSHELL_GATEWAY ?? "nemoclaw", + ...extra, + }; +} + +async function bestEffort(run: () => Promise): Promise { + try { + await run(); + } catch { + // Cleanup should not mask primary failures. + } +} + +function singleLineShell(script: string): string { + const encoded = Buffer.from(script, "utf8").toString("base64"); + return `tmp=$(mktemp); trap 'rm -f "$tmp"' EXIT; printf %s '${encoded}' | base64 -d > "$tmp"; sh "$tmp"`; +} + +async function sandboxShell( + sandbox: SandboxClient, + script: string, + options: { artifactName: string; timeoutMs?: number; redactionValues?: string[] }, +): Promise { + return await sandbox.execShell(SANDBOX_NAME, trustedSandboxShellScript(singleLineShell(script)), { + artifactName: options.artifactName, + env: commandEnv(), + timeoutMs: options.timeoutMs ?? 60_000, + redactionValues: options.redactionValues, + }); +} + +async function cleanupBraveSandbox(sandbox: SandboxClient): Promise { + await bestEffort(() => + sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { + artifactName: "cleanup-openshell-delete-brave-search", + env: commandEnv(), + timeoutMs: 60_000, + }), + ); +} + +function parsePlaceholder(configText: string): string | undefined { + const parsed = JSON.parse(configText) as { + tools?: { web?: { search?: { apiKey?: unknown } } }; + }; + const value = parsed.tools?.web?.search?.apiKey; + return typeof value === "string" && value ? value : undefined; +} + +function firstJsonObject(output: string): unknown { + for (let start = output.indexOf("{"); start >= 0; start = output.indexOf("{", start + 1)) { + let depth = 0; + let inString = false; + let escaped = false; + for (let index = start; index < output.length; index += 1) { + const char = output[index]; + if (inString) { + if (escaped) escaped = false; + else if (char === "\\") escaped = true; + else if (char === '"') inString = false; + continue; + } + if (char === '"') inString = true; + else if (char === "{") depth += 1; + else if (char === "}") { + depth -= 1; + if (depth === 0) { + try { + return JSON.parse(output.slice(start, index + 1)); + } catch { + break; + } + } + } + } + } + return undefined; +} + +function collectAssistantText(value: unknown): string[] { + if (typeof value === "string" && value.trim()) return [value.trim()]; + if (!value || typeof value !== "object") return []; + if (Array.isArray(value)) return value.flatMap(collectAssistantText); + const record = value as Record; + const texts: string[] = []; + for (const key of [ + "result", + "payloads", + "messages", + "choices", + "message", + "delta", + "content", + "text", + ]) { + if (key in record) texts.push(...collectAssistantText(record[key])); + } + return texts; +} + +/** + * Source boundary: `openclaw agent --json` may emit launcher progress before + * and after the final JSON envelope. This mirrors the retained legacy shell + * parser's tolerant envelope handling until OpenClaw exposes a stable + * JSON-only stdout contract for live E2E consumers. + */ +function extractOpenClawAgentText(output: string): string { + const parsed = firstJsonObject(output); + return collectAssistantText(parsed)[0] ?? ""; +} + +test.skipIf(!shouldRunLiveE2EScenarios())( + "Brave search preset wires policy/config, hides the real key, and performs real searches (#2687)", + { timeout: LIVE_TIMEOUT_MS }, + async ({ artifacts, cleanup, host, sandbox, secrets, skip }) => { + const braveKey = secrets.required("BRAVE_API_KEY"); + const inferenceKey = secrets.required("NVIDIA_INFERENCE_API_KEY"); + const redactionValues = [braveKey, inferenceKey]; + + await artifacts.writeJson("scenario.json", { + id: "brave-search", + runner: "vitest", + legacySource: "test/e2e/test-brave-search-e2e.sh", + boundary: + "source CLI onboard + OpenShell policy/config + in-sandbox OpenClaw/Brave API calls", + sandboxName: SANDBOX_NAME, + contracts: [ + "onboard succeeds with BRAVE_API_KEY present", + "the brave network policy preset includes api.search.brave.com", + "OpenClaw web search config is enabled and selects provider=brave", + "the real BRAVE_API_KEY is absent from openclaw.json and sandbox shell env", + "OpenClaw agent can perform a Brave-backed web search", + "curl from inside the sandbox can query Brave using the placeholder token header", + ], + }); + + const dockerInfo = await host.command("docker", ["info"], { + artifactName: "phase-0-docker-info", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }); + if (dockerInfo.exitCode !== 0) { + if (process.env.GITHUB_ACTIONS === "true") { + throw new Error(`Docker is required for Brave search E2E: ${resultText(dockerInfo)}`); + } + skip(`Docker is required for Brave search E2E: ${resultText(dockerInfo)}`); + } + + cleanup.add(`destroy brave search sandbox ${SANDBOX_NAME}`, async () => { + await bestEffort(() => + host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { + artifactName: "cleanup-nemoclaw-destroy-brave-search", + env: commandEnv(), + timeoutMs: 120_000, + }), + ); + await cleanupBraveSandbox(sandbox); + }); + + await bestEffort(() => + host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { + artifactName: "pre-cleanup-nemoclaw-destroy-brave-search", + env: commandEnv(), + timeoutMs: 120_000, + }), + ); + await cleanupBraveSandbox(sandbox); + + let onboard: ShellProbeResult | undefined; + for (let attempt = 1; attempt <= INSTALL_ATTEMPTS; attempt += 1) { + onboard = await host.command( + "node", + [ + CLI_ENTRYPOINT, + "onboard", + "--fresh", + "--non-interactive", + "--yes-i-accept-third-party-software", + ], + { + artifactName: + attempt === 1 + ? "phase-1-onboard-brave-search" + : `phase-1-onboard-brave-search-attempt-${attempt}`, + cwd: REPO_ROOT, + env: commandEnv({ + BRAVE_API_KEY: braveKey, + NVIDIA_INFERENCE_API_KEY: inferenceKey, + NVIDIA_API_KEY: inferenceKey, + }), + redactionValues, + timeoutMs: 20 * 60_000, + }, + ); + if (onboard.exitCode === 0) break; + if (isTransientProviderValidationFailure(onboard) && attempt < INSTALL_ATTEMPTS) { + await new Promise((resolve) => setTimeout(resolve, 10_000 * attempt)); + continue; + } + break; + } + expect(onboard, "onboard command must run").toBeDefined(); + expect(onboard?.exitCode, resultText(onboard as ShellProbeResult)).toBe(0); + + const policy = await sandbox.openshell(["policy", "get", "--full", SANDBOX_NAME], { + artifactName: "phase-2-brave-policy", + env: commandEnv(), + timeoutMs: 60_000, + }); + expect(policy.exitCode, resultText(policy)).toBe(0); + expect(resultText(policy)).toContain("api.search.brave.com"); + + const config = await sandbox.exec(SANDBOX_NAME, ["cat", "/sandbox/.openclaw/openclaw.json"], { + artifactName: "phase-2-openclaw-config", + env: commandEnv(), + redactionValues, + timeoutMs: 60_000, + }); + expect(config.exitCode, resultText(config)).toBe(0); + const secretDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-brave-secret-")); + const secretFile = path.join(secretDir, "brave-key"); + fs.writeFileSync(secretFile, braveKey, { mode: 0o600 }); + const remoteSecretFile = "/tmp/nemoclaw-brave-key-leak-check"; + cleanup.add("remove temporary Brave leak-check secret", async () => { + fs.rmSync(secretDir, { recursive: true, force: true }); + await bestEffort(() => + sandbox.execShell(SANDBOX_NAME, trustedSandboxShellScript(`rm -f ${remoteSecretFile}`), { + artifactName: "cleanup-brave-leak-secret", + env: commandEnv(), + timeoutMs: 30_000, + }), + ); + }); + const uploadSecret = await sandbox.upload(SANDBOX_NAME, secretFile, remoteSecretFile, { + artifactName: "phase-3-upload-brave-leak-secret", + env: commandEnv(), + redactionValues, + timeoutMs: 30_000, + }); + expect(uploadSecret.exitCode, resultText(uploadSecret)).toBe(0); + const rawLeakCheck = await sandbox.execShell( + SANDBOX_NAME, + trustedSandboxShellScript( + `python3 - <<'PY' +from pathlib import Path +needle = Path('${remoteSecretFile}').read_text(encoding='utf-8') +body = Path('/sandbox/.openclaw/openclaw.json').read_text(encoding='utf-8') +raise SystemExit(1 if needle in body else 0) +PY`, + ), + { + artifactName: "phase-3-openclaw-config-raw-secret-leak-check", + env: commandEnv(), + timeoutMs: 30_000, + }, + ); + expect( + rawLeakCheck.exitCode, + "raw BRAVE_API_KEY must not appear anywhere in openclaw.json", + ).toBe(0); + const parsedConfig = JSON.parse(config.stdout) as { + tools?: { web?: { search?: { enabled?: unknown; provider?: unknown; apiKey?: unknown } } }; + }; + const searchConfig = parsedConfig.tools?.web?.search; + expect(searchConfig?.enabled, config.stdout).toBe(true); + expect(searchConfig?.provider, config.stdout).toBe("brave"); + const placeholder = parsePlaceholder(config.stdout); + expect(placeholder, config.stdout).toMatch(PLACEHOLDER_PATTERN); + + const envCheck = await sandbox.exec( + SANDBOX_NAME, + ["sh", "-lc", "printenv BRAVE_API_KEY || true"], + { + artifactName: "phase-3-sandbox-brave-env", + env: commandEnv(), + redactionValues, + timeoutMs: 30_000, + }, + ); + expect(envCheck.exitCode, resultText(envCheck)).toBe(0); + expect(envCheck.stdout).not.toContain(braveKey); + if (envCheck.stdout.trim()) expect(envCheck.stdout.trim()).toMatch(PLACEHOLDER_PATTERN); + + const agent = await sandboxShell( + sandbox, + `openclaw agent --agent main --json --session-id e2e-brave-agent-$$ -m 'Use the web search tool to find one result for the query: NVIDIA. Reply with only the title of the top result.'`, + { + artifactName: "phase-4a-agent-web-search", + timeoutMs: 150_000, + redactionValues, + }, + ); + expect(resultText(agent)).not.toMatch( + /SsrFBlockedError|Blocked hostname|ECONNREFUSED|EAI_AGAIN|gateway unavailable|network connection error/i, + ); + expect(agent.exitCode, resultText(agent)).toBe(0); + const assistantText = extractOpenClawAgentText(agent.stdout); + expect(assistantText, resultText(agent)).toMatch(/nvidia|geforce|cuda|gpu/i); + + const curl = await sandboxShell( + sandbox, + `curl -sS --max-time 20 -G 'https://api.search.brave.com/res/v1/web/search' --data-urlencode 'q=NVIDIA' --data-urlencode 'count=1' -H 'X-Subscription-Token: ${placeholder}' -w '\nHTTP_STATUS:%{http_code}\n'`, + { + artifactName: "phase-4b-direct-brave-curl", + timeoutMs: 60_000, + redactionValues, + }, + ); + const status = resultText(curl).match(/HTTP_STATUS:(\d{3})/)?.[1]; + expect(status, resultText(curl)).toBe("200"); + const body = resultText(curl).replace(/\n?HTTP_STATUS:\d{3}\s*$/u, ""); + const braveResponse = JSON.parse(body) as { web?: { results?: unknown[] } }; + expect(braveResponse.web?.results?.length ?? 0, body.slice(0, 500)).toBeGreaterThan(0); + }, +); From eda497449812fab6cb3338c2b91b1d165b89dc0e Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sat, 20 Jun 2026 10:31:08 -0700 Subject: [PATCH 8/8] test(e2e): move Brave setup branches to helpers Signed-off-by: Carlos Villela --- .../e2e-scenario/live/brave-search-helpers.ts | 284 ++++++++++++++++ test/e2e-scenario/live/brave-search.test.ts | 303 ++---------------- 2 files changed, 319 insertions(+), 268 deletions(-) create mode 100644 test/e2e-scenario/live/brave-search-helpers.ts diff --git a/test/e2e-scenario/live/brave-search-helpers.ts b/test/e2e-scenario/live/brave-search-helpers.ts new file mode 100644 index 00000000000..2b1aad85bdc --- /dev/null +++ b/test/e2e-scenario/live/brave-search-helpers.ts @@ -0,0 +1,284 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import type { HostCliClient } from "../fixtures/clients/host.ts"; +import { resultText } from "../fixtures/clients/index.ts"; +import { + type SandboxClient, + trustedSandboxShellScript, + validateSandboxName, +} from "../fixtures/clients/sandbox.ts"; +import { expect } from "../fixtures/e2e-test.ts"; +import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; +import { isTransientProviderValidationFailure } from "./network-policy-transient-provider.ts"; + +const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); +const CLI_ENTRYPOINT = path.join(REPO_ROOT, "bin", "nemoclaw.js"); +export const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-brave-search"; +validateSandboxName(SANDBOX_NAME); +const INSTALL_ATTEMPTS = process.env.CI === "true" || process.env.GITHUB_ACTIONS === "true" ? 3 : 1; +const PLACEHOLDER_PATTERN = /^openshell:resolve:env:([A-Za-z0-9_]+_)?BRAVE_API_KEY$/; + +export function commandEnv(extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { + return { + ...buildAvailabilityProbeEnv(), + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_RECREATE_SANDBOX: "1", + NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, + OPENSHELL_GATEWAY: process.env.OPENSHELL_GATEWAY ?? "nemoclaw", + ...extra, + }; +} + +export async function bestEffort(run: () => Promise): Promise { + try { + await run(); + } catch { + // Cleanup should not mask primary failures. + } +} + +function singleLineShell(script: string): string { + const encoded = Buffer.from(script, "utf8").toString("base64"); + return `tmp=$(mktemp); trap 'rm -f "$tmp"' EXIT; printf %s '${encoded}' | base64 -d > "$tmp"; sh "$tmp"`; +} + +export async function sandboxShell( + sandbox: SandboxClient, + script: string, + options: { artifactName: string; timeoutMs?: number; redactionValues?: string[] }, +): Promise { + return await sandbox.execShell(SANDBOX_NAME, trustedSandboxShellScript(singleLineShell(script)), { + artifactName: options.artifactName, + env: commandEnv(), + timeoutMs: options.timeoutMs ?? 60_000, + redactionValues: options.redactionValues, + }); +} + +export async function cleanupBraveState( + host: HostCliClient, + sandbox: SandboxClient, +): Promise { + await bestEffort(() => + host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { + artifactName: "cleanup-nemoclaw-destroy-brave-search", + env: commandEnv(), + timeoutMs: 120_000, + }), + ); + await bestEffort(() => + sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { + artifactName: "cleanup-openshell-delete-brave-search", + env: commandEnv(), + timeoutMs: 60_000, + }), + ); +} + +function parsePlaceholder(configText: string): string | undefined { + const parsed = JSON.parse(configText) as { + tools?: { web?: { search?: { apiKey?: unknown } } }; + }; + const value = parsed.tools?.web?.search?.apiKey; + return typeof value === "string" && value ? value : undefined; +} + +function firstJsonObject(output: string): unknown { + for (let start = output.indexOf("{"); start >= 0; start = output.indexOf("{", start + 1)) { + let depth = 0; + let inString = false; + let escaped = false; + for (let index = start; index < output.length; index += 1) { + const char = output[index]; + if (inString) { + if (escaped) escaped = false; + else if (char === "\\") escaped = true; + else if (char === '"') inString = false; + continue; + } + if (char === '"') inString = true; + else if (char === "{") depth += 1; + else if (char === "}") { + depth -= 1; + if (depth === 0) { + try { + return JSON.parse(output.slice(start, index + 1)); + } catch { + break; + } + } + } + } + } + return undefined; +} + +function collectAssistantText(value: unknown): string[] { + if (typeof value === "string" && value.trim()) return [value.trim()]; + if (!value || typeof value !== "object") return []; + if (Array.isArray(value)) return value.flatMap(collectAssistantText); + const record = value as Record; + const texts: string[] = []; + for (const key of [ + "result", + "payloads", + "messages", + "choices", + "message", + "delta", + "content", + "text", + ]) { + if (key in record) texts.push(...collectAssistantText(record[key])); + } + return texts; +} + +export function extractOpenClawAgentText(output: string): string { + return collectAssistantText(firstJsonObject(output))[0] ?? ""; +} + +export function assertDockerAvailable( + result: ShellProbeResult, + skip: (note?: string) => never, +): void { + result.exitCode === 0 || process.env.GITHUB_ACTIONS === "true" + ? undefined + : skip(`Docker is required for Brave search E2E: ${resultText(result)}`); + result.exitCode === 0 || + process.env.GITHUB_ACTIONS !== "true" || + (() => { + throw new Error(`Docker is required for Brave search E2E: ${resultText(result)}`); + })(); +} + +export async function onboardBrave( + host: HostCliClient, + braveKey: string, + inferenceKey: string, +): Promise { + let onboard: ShellProbeResult | undefined; + const redactionValues = [braveKey, inferenceKey]; + for (let attempt = 1; attempt <= INSTALL_ATTEMPTS; attempt += 1) { + onboard = await host.command( + "node", + [ + CLI_ENTRYPOINT, + "onboard", + "--fresh", + "--non-interactive", + "--yes-i-accept-third-party-software", + ], + { + artifactName: + attempt === 1 + ? "phase-1-onboard-brave-search" + : `phase-1-onboard-brave-search-attempt-${attempt}`, + cwd: REPO_ROOT, + env: commandEnv({ + BRAVE_API_KEY: braveKey, + NVIDIA_INFERENCE_API_KEY: inferenceKey, + NVIDIA_API_KEY: inferenceKey, + }), + redactionValues, + timeoutMs: 20 * 60_000, + }, + ); + const retry = + onboard.exitCode !== 0 && + isTransientProviderValidationFailure(onboard) && + attempt < INSTALL_ATTEMPTS; + onboard.exitCode === 0 && (attempt = INSTALL_ATTEMPTS + 1); + retry && (await new Promise((resolve) => setTimeout(resolve, 10_000 * attempt))); + !retry && onboard.exitCode !== 0 && (attempt = INSTALL_ATTEMPTS + 1); + } + if (!onboard) throw new Error("onboard command did not run"); + return onboard; +} + +export async function uploadSecretForLeakCheck( + sandbox: SandboxClient, + cleanup: { add(name: string, run: () => Promise | void): void }, + braveKey: string, + redactionValues: string[], +): Promise { + const secretDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-brave-secret-")); + const secretFile = path.join(secretDir, "brave-key"); + fs.writeFileSync(secretFile, braveKey, { mode: 0o600 }); + const remoteSecretFile = "/tmp/nemoclaw-brave-key-leak-check"; + cleanup.add("remove temporary Brave leak-check secret", async () => { + fs.rmSync(secretDir, { recursive: true, force: true }); + await bestEffort(() => + sandbox.execShell(SANDBOX_NAME, trustedSandboxShellScript(`rm -f ${remoteSecretFile}`), { + artifactName: "cleanup-brave-leak-secret", + env: commandEnv(), + timeoutMs: 30_000, + }), + ); + }); + const uploadSecret = await sandbox.upload(SANDBOX_NAME, secretFile, remoteSecretFile, { + artifactName: "phase-3-upload-brave-leak-secret", + env: commandEnv(), + redactionValues, + timeoutMs: 30_000, + }); + expect(uploadSecret.exitCode, resultText(uploadSecret)).toBe(0); + return remoteSecretFile; +} + +export async function assertRawConfigHasNoSecret( + sandbox: SandboxClient, + remoteSecretFile: string, +): Promise { + const rawLeakCheck = await sandbox.execShell( + SANDBOX_NAME, + trustedSandboxShellScript( + `python3 - <<'PY' +from pathlib import Path +needle = Path('${remoteSecretFile}').read_text(encoding='utf-8') +body = Path('/sandbox/.openclaw/openclaw.json').read_text(encoding='utf-8') +raise SystemExit(1 if needle in body else 0) +PY`, + ), + { + artifactName: "phase-3-openclaw-config-raw-secret-leak-check", + env: commandEnv(), + timeoutMs: 30_000, + }, + ); + expect(rawLeakCheck.exitCode, "raw BRAVE_API_KEY must not appear anywhere in openclaw.json").toBe( + 0, + ); +} + +export function assertBraveConfig(configText: string): string { + const parsedConfig = JSON.parse(configText) as { + tools?: { web?: { search?: { enabled?: unknown; provider?: unknown; apiKey?: unknown } } }; + }; + const searchConfig = parsedConfig.tools?.web?.search; + expect(searchConfig?.enabled, configText).toBe(true); + expect(searchConfig?.provider, configText).toBe("brave"); + const placeholder = parsePlaceholder(configText); + expect(placeholder, configText).toMatch(PLACEHOLDER_PATTERN); + return placeholder ?? ""; +} + +export function assertOptionalBraveEnv(value: string, braveKey: string): void { + expect(value).not.toContain(braveKey); + value.trim() && expect(value.trim()).toMatch(PLACEHOLDER_PATTERN); +} + +export function assertBraveResponse(body: string): void { + const status = body.match(/HTTP_STATUS:(\d{3})/)?.[1]; + expect(status, body).toBe("200"); + const json = body.replace(/\n?HTTP_STATUS:\d{3}\s*$/u, ""); + const braveResponse = JSON.parse(json) as { web?: { results?: unknown[] } }; + expect(braveResponse.web?.results?.length ?? 0, json.slice(0, 500)).toBeGreaterThan(0); +} diff --git a/test/e2e-scenario/live/brave-search.test.ts b/test/e2e-scenario/live/brave-search.test.ts index cfe433d4a41..b0c1cce917a 100644 --- a/test/e2e-scenario/live/brave-search.test.ts +++ b/test/e2e-scenario/live/brave-search.test.ts @@ -1,156 +1,28 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -/** - * Live Vitest replacement for test/e2e/test-brave-search-e2e.sh. - * - * Preserves the legacy #2687 acceptance boundary: non-interactive onboard with - * a real BRAVE_API_KEY, brave policy/config wiring, secret non-leak checks, - * a real agent web-search turn, and a direct in-sandbox Brave API curl using - * the OpenShell credential placeholder. - */ - -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; +/** Live Vitest replacement for test/e2e/test-brave-search-e2e.sh. */ import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; import { resultText } from "../fixtures/clients/index.ts"; -import { - type SandboxClient, - trustedSandboxShellScript, - validateSandboxName, -} from "../fixtures/clients/sandbox.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; import { shouldRunLiveE2EScenarios } from "../fixtures/live-project-gate.ts"; -import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; -import { isTransientProviderValidationFailure } from "./network-policy-transient-provider.ts"; +import { + assertBraveConfig, + assertBraveResponse, + assertDockerAvailable, + assertOptionalBraveEnv, + assertRawConfigHasNoSecret, + cleanupBraveState, + commandEnv, + extractOpenClawAgentText, + onboardBrave, + SANDBOX_NAME, + sandboxShell, + uploadSecretForLeakCheck, +} from "./brave-search-helpers.ts"; -const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); -const CLI_ENTRYPOINT = path.join(REPO_ROOT, "bin", "nemoclaw.js"); -const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-brave-search"; -validateSandboxName(SANDBOX_NAME); -const INSTALL_ATTEMPTS = process.env.CI === "true" || process.env.GITHUB_ACTIONS === "true" ? 3 : 1; const LIVE_TIMEOUT_MS = 35 * 60_000; -const PLACEHOLDER_PATTERN = /^openshell:resolve:env:([A-Za-z0-9_]+_)?BRAVE_API_KEY$/; - -function commandEnv(extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { - return { - ...buildAvailabilityProbeEnv(), - NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", - NEMOCLAW_NON_INTERACTIVE: "1", - NEMOCLAW_RECREATE_SANDBOX: "1", - NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, - OPENSHELL_GATEWAY: process.env.OPENSHELL_GATEWAY ?? "nemoclaw", - ...extra, - }; -} - -async function bestEffort(run: () => Promise): Promise { - try { - await run(); - } catch { - // Cleanup should not mask primary failures. - } -} - -function singleLineShell(script: string): string { - const encoded = Buffer.from(script, "utf8").toString("base64"); - return `tmp=$(mktemp); trap 'rm -f "$tmp"' EXIT; printf %s '${encoded}' | base64 -d > "$tmp"; sh "$tmp"`; -} - -async function sandboxShell( - sandbox: SandboxClient, - script: string, - options: { artifactName: string; timeoutMs?: number; redactionValues?: string[] }, -): Promise { - return await sandbox.execShell(SANDBOX_NAME, trustedSandboxShellScript(singleLineShell(script)), { - artifactName: options.artifactName, - env: commandEnv(), - timeoutMs: options.timeoutMs ?? 60_000, - redactionValues: options.redactionValues, - }); -} - -async function cleanupBraveSandbox(sandbox: SandboxClient): Promise { - await bestEffort(() => - sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { - artifactName: "cleanup-openshell-delete-brave-search", - env: commandEnv(), - timeoutMs: 60_000, - }), - ); -} - -function parsePlaceholder(configText: string): string | undefined { - const parsed = JSON.parse(configText) as { - tools?: { web?: { search?: { apiKey?: unknown } } }; - }; - const value = parsed.tools?.web?.search?.apiKey; - return typeof value === "string" && value ? value : undefined; -} - -function firstJsonObject(output: string): unknown { - for (let start = output.indexOf("{"); start >= 0; start = output.indexOf("{", start + 1)) { - let depth = 0; - let inString = false; - let escaped = false; - for (let index = start; index < output.length; index += 1) { - const char = output[index]; - if (inString) { - if (escaped) escaped = false; - else if (char === "\\") escaped = true; - else if (char === '"') inString = false; - continue; - } - if (char === '"') inString = true; - else if (char === "{") depth += 1; - else if (char === "}") { - depth -= 1; - if (depth === 0) { - try { - return JSON.parse(output.slice(start, index + 1)); - } catch { - break; - } - } - } - } - } - return undefined; -} - -function collectAssistantText(value: unknown): string[] { - if (typeof value === "string" && value.trim()) return [value.trim()]; - if (!value || typeof value !== "object") return []; - if (Array.isArray(value)) return value.flatMap(collectAssistantText); - const record = value as Record; - const texts: string[] = []; - for (const key of [ - "result", - "payloads", - "messages", - "choices", - "message", - "delta", - "content", - "text", - ]) { - if (key in record) texts.push(...collectAssistantText(record[key])); - } - return texts; -} - -/** - * Source boundary: `openclaw agent --json` may emit launcher progress before - * and after the final JSON envelope. This mirrors the retained legacy shell - * parser's tolerant envelope handling until OpenClaw exposes a stable - * JSON-only stdout contract for live E2E consumers. - */ -function extractOpenClawAgentText(output: string): string { - const parsed = firstJsonObject(output); - return collectAssistantText(parsed)[0] ?? ""; -} test.skipIf(!shouldRunLiveE2EScenarios())( "Brave search preset wires policy/config, hides the real key, and performs real searches (#2687)", @@ -182,68 +54,15 @@ test.skipIf(!shouldRunLiveE2EScenarios())( env: buildAvailabilityProbeEnv(), timeoutMs: 30_000, }); - if (dockerInfo.exitCode !== 0) { - if (process.env.GITHUB_ACTIONS === "true") { - throw new Error(`Docker is required for Brave search E2E: ${resultText(dockerInfo)}`); - } - skip(`Docker is required for Brave search E2E: ${resultText(dockerInfo)}`); - } + assertDockerAvailable(dockerInfo, skip); - cleanup.add(`destroy brave search sandbox ${SANDBOX_NAME}`, async () => { - await bestEffort(() => - host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { - artifactName: "cleanup-nemoclaw-destroy-brave-search", - env: commandEnv(), - timeoutMs: 120_000, - }), - ); - await cleanupBraveSandbox(sandbox); - }); - - await bestEffort(() => - host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { - artifactName: "pre-cleanup-nemoclaw-destroy-brave-search", - env: commandEnv(), - timeoutMs: 120_000, - }), + cleanup.add(`destroy brave search sandbox ${SANDBOX_NAME}`, () => + cleanupBraveState(host, sandbox), ); - await cleanupBraveSandbox(sandbox); + await cleanupBraveState(host, sandbox); - let onboard: ShellProbeResult | undefined; - for (let attempt = 1; attempt <= INSTALL_ATTEMPTS; attempt += 1) { - onboard = await host.command( - "node", - [ - CLI_ENTRYPOINT, - "onboard", - "--fresh", - "--non-interactive", - "--yes-i-accept-third-party-software", - ], - { - artifactName: - attempt === 1 - ? "phase-1-onboard-brave-search" - : `phase-1-onboard-brave-search-attempt-${attempt}`, - cwd: REPO_ROOT, - env: commandEnv({ - BRAVE_API_KEY: braveKey, - NVIDIA_INFERENCE_API_KEY: inferenceKey, - NVIDIA_API_KEY: inferenceKey, - }), - redactionValues, - timeoutMs: 20 * 60_000, - }, - ); - if (onboard.exitCode === 0) break; - if (isTransientProviderValidationFailure(onboard) && attempt < INSTALL_ATTEMPTS) { - await new Promise((resolve) => setTimeout(resolve, 10_000 * attempt)); - continue; - } - break; - } - expect(onboard, "onboard command must run").toBeDefined(); - expect(onboard?.exitCode, resultText(onboard as ShellProbeResult)).toBe(0); + const onboard = await onboardBrave(host, braveKey, inferenceKey); + expect(onboard.exitCode, resultText(onboard)).toBe(0); const policy = await sandbox.openshell(["policy", "get", "--full", SANDBOX_NAME], { artifactName: "phase-2-brave-policy", @@ -260,55 +79,15 @@ test.skipIf(!shouldRunLiveE2EScenarios())( timeoutMs: 60_000, }); expect(config.exitCode, resultText(config)).toBe(0); - const secretDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-brave-secret-")); - const secretFile = path.join(secretDir, "brave-key"); - fs.writeFileSync(secretFile, braveKey, { mode: 0o600 }); - const remoteSecretFile = "/tmp/nemoclaw-brave-key-leak-check"; - cleanup.add("remove temporary Brave leak-check secret", async () => { - fs.rmSync(secretDir, { recursive: true, force: true }); - await bestEffort(() => - sandbox.execShell(SANDBOX_NAME, trustedSandboxShellScript(`rm -f ${remoteSecretFile}`), { - artifactName: "cleanup-brave-leak-secret", - env: commandEnv(), - timeoutMs: 30_000, - }), - ); - }); - const uploadSecret = await sandbox.upload(SANDBOX_NAME, secretFile, remoteSecretFile, { - artifactName: "phase-3-upload-brave-leak-secret", - env: commandEnv(), + + const remoteSecretFile = await uploadSecretForLeakCheck( + sandbox, + cleanup, + braveKey, redactionValues, - timeoutMs: 30_000, - }); - expect(uploadSecret.exitCode, resultText(uploadSecret)).toBe(0); - const rawLeakCheck = await sandbox.execShell( - SANDBOX_NAME, - trustedSandboxShellScript( - `python3 - <<'PY' -from pathlib import Path -needle = Path('${remoteSecretFile}').read_text(encoding='utf-8') -body = Path('/sandbox/.openclaw/openclaw.json').read_text(encoding='utf-8') -raise SystemExit(1 if needle in body else 0) -PY`, - ), - { - artifactName: "phase-3-openclaw-config-raw-secret-leak-check", - env: commandEnv(), - timeoutMs: 30_000, - }, ); - expect( - rawLeakCheck.exitCode, - "raw BRAVE_API_KEY must not appear anywhere in openclaw.json", - ).toBe(0); - const parsedConfig = JSON.parse(config.stdout) as { - tools?: { web?: { search?: { enabled?: unknown; provider?: unknown; apiKey?: unknown } } }; - }; - const searchConfig = parsedConfig.tools?.web?.search; - expect(searchConfig?.enabled, config.stdout).toBe(true); - expect(searchConfig?.provider, config.stdout).toBe("brave"); - const placeholder = parsePlaceholder(config.stdout); - expect(placeholder, config.stdout).toMatch(PLACEHOLDER_PATTERN); + await assertRawConfigHasNoSecret(sandbox, remoteSecretFile); + const placeholder = assertBraveConfig(config.stdout); const envCheck = await sandbox.exec( SANDBOX_NAME, @@ -321,38 +100,26 @@ PY`, }, ); expect(envCheck.exitCode, resultText(envCheck)).toBe(0); - expect(envCheck.stdout).not.toContain(braveKey); - if (envCheck.stdout.trim()) expect(envCheck.stdout.trim()).toMatch(PLACEHOLDER_PATTERN); + assertOptionalBraveEnv(envCheck.stdout, braveKey); const agent = await sandboxShell( sandbox, `openclaw agent --agent main --json --session-id e2e-brave-agent-$$ -m 'Use the web search tool to find one result for the query: NVIDIA. Reply with only the title of the top result.'`, - { - artifactName: "phase-4a-agent-web-search", - timeoutMs: 150_000, - redactionValues, - }, + { artifactName: "phase-4a-agent-web-search", timeoutMs: 150_000, redactionValues }, ); expect(resultText(agent)).not.toMatch( /SsrFBlockedError|Blocked hostname|ECONNREFUSED|EAI_AGAIN|gateway unavailable|network connection error/i, ); expect(agent.exitCode, resultText(agent)).toBe(0); - const assistantText = extractOpenClawAgentText(agent.stdout); - expect(assistantText, resultText(agent)).toMatch(/nvidia|geforce|cuda|gpu/i); + expect(extractOpenClawAgentText(agent.stdout), resultText(agent)).toMatch( + /nvidia|geforce|cuda|gpu/i, + ); const curl = await sandboxShell( sandbox, `curl -sS --max-time 20 -G 'https://api.search.brave.com/res/v1/web/search' --data-urlencode 'q=NVIDIA' --data-urlencode 'count=1' -H 'X-Subscription-Token: ${placeholder}' -w '\nHTTP_STATUS:%{http_code}\n'`, - { - artifactName: "phase-4b-direct-brave-curl", - timeoutMs: 60_000, - redactionValues, - }, + { artifactName: "phase-4b-direct-brave-curl", timeoutMs: 60_000, redactionValues }, ); - const status = resultText(curl).match(/HTTP_STATUS:(\d{3})/)?.[1]; - expect(status, resultText(curl)).toBe("200"); - const body = resultText(curl).replace(/\n?HTTP_STATUS:\d{3}\s*$/u, ""); - const braveResponse = JSON.parse(body) as { web?: { results?: unknown[] } }; - expect(braveResponse.web?.results?.length ?? 0, body.slice(0, 500)).toBeGreaterThan(0); + assertBraveResponse(resultText(curl)); }, );