diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index 2393329a5b3..683a972d59f 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -742,6 +742,71 @@ jobs: if-no-files-found: ignore retention-days: 14 + brave-search-vitest: + needs: generate-matrix + if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',brave-search-vitest,') || contains(format(',{0},', inputs.scenarios), ',brave-search,') }} + runs-on: ubuntu-latest + timeout-minutes: 45 + env: + FREE_STANDING_VITEST_JOB: "1" + FREE_STANDING_SCENARIO_ID: "brave-search" + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/brave-search + NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js + NEMOCLAW_RUN_E2E_SCENARIOS: "1" + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + NEMOCLAW_SANDBOX_NAME: "e2e-brave-search" + OPENSHELL_GATEWAY: "nemoclaw" + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Set up Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 + with: + node-version: 22 + cache: npm + + - name: Install root dependencies + run: npm ci --ignore-scripts + + - name: Build CLI + run: npm run build:cli + + - name: Install OpenShell CLI + run: bash scripts/install-openshell.sh + + - name: Run Brave search live Vitest test + env: + BRAVE_API_KEY: ${{ secrets.BRAVE_API_KEY }} + NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }} + run: | + set -euo pipefail + export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH" + if command -v openshell >/dev/null 2>&1; then + OPENSHELL_BIN="$(command -v openshell)" + elif [ -x "$HOME/.local/bin/openshell" ]; then + OPENSHELL_BIN="$HOME/.local/bin/openshell" + else + echo "::error::OpenShell CLI not found after install" + ls -la /usr/local/bin/openshell "$HOME/.local/bin/openshell" 2>&1 || true + exit 1 + fi + export OPENSHELL_BIN + "$OPENSHELL_BIN" --version + npx vitest run --project e2e-scenarios-live test/e2e-scenario/live/brave-search.test.ts --silent=false --reporter=default + + - name: Upload Brave search artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: e2e-vitest-scenarios-brave-search + path: e2e-artifacts/vitest/brave-search/ + include-hidden-files: false + if-no-files-found: ignore + retention-days: 14 + ollama-auth-proxy-vitest: needs: generate-matrix if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',ollama-auth-proxy-vitest,') || contains(format(',{0},', inputs.scenarios), ',ollama-auth-proxy,') }} @@ -3691,8 +3756,11 @@ jobs: openclaw-skill-cli-vitest, inference-routing-vitest, cloud-inference-vitest, + brave-search-vitest, ollama-auth-proxy-vitest, + cron-preflight-inference-local-vitest, + credential-sanitization-vitest, credential-migration-vitest, sessions-agents-cli-vitest, diff --git a/test/e2e-scenario/live/brave-search-helpers.ts b/test/e2e-scenario/live/brave-search-helpers.ts new file mode 100644 index 00000000000..2b1aad85bdc --- /dev/null +++ b/test/e2e-scenario/live/brave-search-helpers.ts @@ -0,0 +1,284 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import type { HostCliClient } from "../fixtures/clients/host.ts"; +import { resultText } from "../fixtures/clients/index.ts"; +import { + type SandboxClient, + trustedSandboxShellScript, + validateSandboxName, +} from "../fixtures/clients/sandbox.ts"; +import { expect } from "../fixtures/e2e-test.ts"; +import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; +import { isTransientProviderValidationFailure } from "./network-policy-transient-provider.ts"; + +const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); +const CLI_ENTRYPOINT = path.join(REPO_ROOT, "bin", "nemoclaw.js"); +export const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-brave-search"; +validateSandboxName(SANDBOX_NAME); +const INSTALL_ATTEMPTS = process.env.CI === "true" || process.env.GITHUB_ACTIONS === "true" ? 3 : 1; +const PLACEHOLDER_PATTERN = /^openshell:resolve:env:([A-Za-z0-9_]+_)?BRAVE_API_KEY$/; + +export function commandEnv(extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { + return { + ...buildAvailabilityProbeEnv(), + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_RECREATE_SANDBOX: "1", + NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, + OPENSHELL_GATEWAY: process.env.OPENSHELL_GATEWAY ?? "nemoclaw", + ...extra, + }; +} + +export async function bestEffort(run: () => Promise): Promise { + try { + await run(); + } catch { + // Cleanup should not mask primary failures. + } +} + +function singleLineShell(script: string): string { + const encoded = Buffer.from(script, "utf8").toString("base64"); + return `tmp=$(mktemp); trap 'rm -f "$tmp"' EXIT; printf %s '${encoded}' | base64 -d > "$tmp"; sh "$tmp"`; +} + +export async function sandboxShell( + sandbox: SandboxClient, + script: string, + options: { artifactName: string; timeoutMs?: number; redactionValues?: string[] }, +): Promise { + return await sandbox.execShell(SANDBOX_NAME, trustedSandboxShellScript(singleLineShell(script)), { + artifactName: options.artifactName, + env: commandEnv(), + timeoutMs: options.timeoutMs ?? 60_000, + redactionValues: options.redactionValues, + }); +} + +export async function cleanupBraveState( + host: HostCliClient, + sandbox: SandboxClient, +): Promise { + await bestEffort(() => + host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { + artifactName: "cleanup-nemoclaw-destroy-brave-search", + env: commandEnv(), + timeoutMs: 120_000, + }), + ); + await bestEffort(() => + sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { + artifactName: "cleanup-openshell-delete-brave-search", + env: commandEnv(), + timeoutMs: 60_000, + }), + ); +} + +function parsePlaceholder(configText: string): string | undefined { + const parsed = JSON.parse(configText) as { + tools?: { web?: { search?: { apiKey?: unknown } } }; + }; + const value = parsed.tools?.web?.search?.apiKey; + return typeof value === "string" && value ? value : undefined; +} + +function firstJsonObject(output: string): unknown { + for (let start = output.indexOf("{"); start >= 0; start = output.indexOf("{", start + 1)) { + let depth = 0; + let inString = false; + let escaped = false; + for (let index = start; index < output.length; index += 1) { + const char = output[index]; + if (inString) { + if (escaped) escaped = false; + else if (char === "\\") escaped = true; + else if (char === '"') inString = false; + continue; + } + if (char === '"') inString = true; + else if (char === "{") depth += 1; + else if (char === "}") { + depth -= 1; + if (depth === 0) { + try { + return JSON.parse(output.slice(start, index + 1)); + } catch { + break; + } + } + } + } + } + return undefined; +} + +function collectAssistantText(value: unknown): string[] { + if (typeof value === "string" && value.trim()) return [value.trim()]; + if (!value || typeof value !== "object") return []; + if (Array.isArray(value)) return value.flatMap(collectAssistantText); + const record = value as Record; + const texts: string[] = []; + for (const key of [ + "result", + "payloads", + "messages", + "choices", + "message", + "delta", + "content", + "text", + ]) { + if (key in record) texts.push(...collectAssistantText(record[key])); + } + return texts; +} + +export function extractOpenClawAgentText(output: string): string { + return collectAssistantText(firstJsonObject(output))[0] ?? ""; +} + +export function assertDockerAvailable( + result: ShellProbeResult, + skip: (note?: string) => never, +): void { + result.exitCode === 0 || process.env.GITHUB_ACTIONS === "true" + ? undefined + : skip(`Docker is required for Brave search E2E: ${resultText(result)}`); + result.exitCode === 0 || + process.env.GITHUB_ACTIONS !== "true" || + (() => { + throw new Error(`Docker is required for Brave search E2E: ${resultText(result)}`); + })(); +} + +export async function onboardBrave( + host: HostCliClient, + braveKey: string, + inferenceKey: string, +): Promise { + let onboard: ShellProbeResult | undefined; + const redactionValues = [braveKey, inferenceKey]; + for (let attempt = 1; attempt <= INSTALL_ATTEMPTS; attempt += 1) { + onboard = await host.command( + "node", + [ + CLI_ENTRYPOINT, + "onboard", + "--fresh", + "--non-interactive", + "--yes-i-accept-third-party-software", + ], + { + artifactName: + attempt === 1 + ? "phase-1-onboard-brave-search" + : `phase-1-onboard-brave-search-attempt-${attempt}`, + cwd: REPO_ROOT, + env: commandEnv({ + BRAVE_API_KEY: braveKey, + NVIDIA_INFERENCE_API_KEY: inferenceKey, + NVIDIA_API_KEY: inferenceKey, + }), + redactionValues, + timeoutMs: 20 * 60_000, + }, + ); + const retry = + onboard.exitCode !== 0 && + isTransientProviderValidationFailure(onboard) && + attempt < INSTALL_ATTEMPTS; + onboard.exitCode === 0 && (attempt = INSTALL_ATTEMPTS + 1); + retry && (await new Promise((resolve) => setTimeout(resolve, 10_000 * attempt))); + !retry && onboard.exitCode !== 0 && (attempt = INSTALL_ATTEMPTS + 1); + } + if (!onboard) throw new Error("onboard command did not run"); + return onboard; +} + +export async function uploadSecretForLeakCheck( + sandbox: SandboxClient, + cleanup: { add(name: string, run: () => Promise | void): void }, + braveKey: string, + redactionValues: string[], +): Promise { + const secretDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-brave-secret-")); + const secretFile = path.join(secretDir, "brave-key"); + fs.writeFileSync(secretFile, braveKey, { mode: 0o600 }); + const remoteSecretFile = "/tmp/nemoclaw-brave-key-leak-check"; + cleanup.add("remove temporary Brave leak-check secret", async () => { + fs.rmSync(secretDir, { recursive: true, force: true }); + await bestEffort(() => + sandbox.execShell(SANDBOX_NAME, trustedSandboxShellScript(`rm -f ${remoteSecretFile}`), { + artifactName: "cleanup-brave-leak-secret", + env: commandEnv(), + timeoutMs: 30_000, + }), + ); + }); + const uploadSecret = await sandbox.upload(SANDBOX_NAME, secretFile, remoteSecretFile, { + artifactName: "phase-3-upload-brave-leak-secret", + env: commandEnv(), + redactionValues, + timeoutMs: 30_000, + }); + expect(uploadSecret.exitCode, resultText(uploadSecret)).toBe(0); + return remoteSecretFile; +} + +export async function assertRawConfigHasNoSecret( + sandbox: SandboxClient, + remoteSecretFile: string, +): Promise { + const rawLeakCheck = await sandbox.execShell( + SANDBOX_NAME, + trustedSandboxShellScript( + `python3 - <<'PY' +from pathlib import Path +needle = Path('${remoteSecretFile}').read_text(encoding='utf-8') +body = Path('/sandbox/.openclaw/openclaw.json').read_text(encoding='utf-8') +raise SystemExit(1 if needle in body else 0) +PY`, + ), + { + artifactName: "phase-3-openclaw-config-raw-secret-leak-check", + env: commandEnv(), + timeoutMs: 30_000, + }, + ); + expect(rawLeakCheck.exitCode, "raw BRAVE_API_KEY must not appear anywhere in openclaw.json").toBe( + 0, + ); +} + +export function assertBraveConfig(configText: string): string { + const parsedConfig = JSON.parse(configText) as { + tools?: { web?: { search?: { enabled?: unknown; provider?: unknown; apiKey?: unknown } } }; + }; + const searchConfig = parsedConfig.tools?.web?.search; + expect(searchConfig?.enabled, configText).toBe(true); + expect(searchConfig?.provider, configText).toBe("brave"); + const placeholder = parsePlaceholder(configText); + expect(placeholder, configText).toMatch(PLACEHOLDER_PATTERN); + return placeholder ?? ""; +} + +export function assertOptionalBraveEnv(value: string, braveKey: string): void { + expect(value).not.toContain(braveKey); + value.trim() && expect(value.trim()).toMatch(PLACEHOLDER_PATTERN); +} + +export function assertBraveResponse(body: string): void { + const status = body.match(/HTTP_STATUS:(\d{3})/)?.[1]; + expect(status, body).toBe("200"); + const json = body.replace(/\n?HTTP_STATUS:\d{3}\s*$/u, ""); + const braveResponse = JSON.parse(json) as { web?: { results?: unknown[] } }; + expect(braveResponse.web?.results?.length ?? 0, json.slice(0, 500)).toBeGreaterThan(0); +} diff --git a/test/e2e-scenario/live/brave-search.test.ts b/test/e2e-scenario/live/brave-search.test.ts new file mode 100644 index 00000000000..b0c1cce917a --- /dev/null +++ b/test/e2e-scenario/live/brave-search.test.ts @@ -0,0 +1,125 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** Live Vitest replacement for test/e2e/test-brave-search-e2e.sh. */ + +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { resultText } from "../fixtures/clients/index.ts"; +import { expect, test } from "../fixtures/e2e-test.ts"; +import { shouldRunLiveE2EScenarios } from "../fixtures/live-project-gate.ts"; +import { + assertBraveConfig, + assertBraveResponse, + assertDockerAvailable, + assertOptionalBraveEnv, + assertRawConfigHasNoSecret, + cleanupBraveState, + commandEnv, + extractOpenClawAgentText, + onboardBrave, + SANDBOX_NAME, + sandboxShell, + uploadSecretForLeakCheck, +} from "./brave-search-helpers.ts"; + +const LIVE_TIMEOUT_MS = 35 * 60_000; + +test.skipIf(!shouldRunLiveE2EScenarios())( + "Brave search preset wires policy/config, hides the real key, and performs real searches (#2687)", + { timeout: LIVE_TIMEOUT_MS }, + async ({ artifacts, cleanup, host, sandbox, secrets, skip }) => { + const braveKey = secrets.required("BRAVE_API_KEY"); + const inferenceKey = secrets.required("NVIDIA_INFERENCE_API_KEY"); + const redactionValues = [braveKey, inferenceKey]; + + await artifacts.writeJson("scenario.json", { + id: "brave-search", + runner: "vitest", + legacySource: "test/e2e/test-brave-search-e2e.sh", + boundary: + "source CLI onboard + OpenShell policy/config + in-sandbox OpenClaw/Brave API calls", + sandboxName: SANDBOX_NAME, + contracts: [ + "onboard succeeds with BRAVE_API_KEY present", + "the brave network policy preset includes api.search.brave.com", + "OpenClaw web search config is enabled and selects provider=brave", + "the real BRAVE_API_KEY is absent from openclaw.json and sandbox shell env", + "OpenClaw agent can perform a Brave-backed web search", + "curl from inside the sandbox can query Brave using the placeholder token header", + ], + }); + + const dockerInfo = await host.command("docker", ["info"], { + artifactName: "phase-0-docker-info", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }); + assertDockerAvailable(dockerInfo, skip); + + cleanup.add(`destroy brave search sandbox ${SANDBOX_NAME}`, () => + cleanupBraveState(host, sandbox), + ); + await cleanupBraveState(host, sandbox); + + const onboard = await onboardBrave(host, braveKey, inferenceKey); + expect(onboard.exitCode, resultText(onboard)).toBe(0); + + const policy = await sandbox.openshell(["policy", "get", "--full", SANDBOX_NAME], { + artifactName: "phase-2-brave-policy", + env: commandEnv(), + timeoutMs: 60_000, + }); + expect(policy.exitCode, resultText(policy)).toBe(0); + expect(resultText(policy)).toContain("api.search.brave.com"); + + const config = await sandbox.exec(SANDBOX_NAME, ["cat", "/sandbox/.openclaw/openclaw.json"], { + artifactName: "phase-2-openclaw-config", + env: commandEnv(), + redactionValues, + timeoutMs: 60_000, + }); + expect(config.exitCode, resultText(config)).toBe(0); + + const remoteSecretFile = await uploadSecretForLeakCheck( + sandbox, + cleanup, + braveKey, + redactionValues, + ); + await assertRawConfigHasNoSecret(sandbox, remoteSecretFile); + const placeholder = assertBraveConfig(config.stdout); + + const envCheck = await sandbox.exec( + SANDBOX_NAME, + ["sh", "-lc", "printenv BRAVE_API_KEY || true"], + { + artifactName: "phase-3-sandbox-brave-env", + env: commandEnv(), + redactionValues, + timeoutMs: 30_000, + }, + ); + expect(envCheck.exitCode, resultText(envCheck)).toBe(0); + assertOptionalBraveEnv(envCheck.stdout, braveKey); + + const agent = await sandboxShell( + sandbox, + `openclaw agent --agent main --json --session-id e2e-brave-agent-$$ -m 'Use the web search tool to find one result for the query: NVIDIA. Reply with only the title of the top result.'`, + { artifactName: "phase-4a-agent-web-search", timeoutMs: 150_000, redactionValues }, + ); + expect(resultText(agent)).not.toMatch( + /SsrFBlockedError|Blocked hostname|ECONNREFUSED|EAI_AGAIN|gateway unavailable|network connection error/i, + ); + expect(agent.exitCode, resultText(agent)).toBe(0); + expect(extractOpenClawAgentText(agent.stdout), resultText(agent)).toMatch( + /nvidia|geforce|cuda|gpu/i, + ); + + const curl = await sandboxShell( + sandbox, + `curl -sS --max-time 20 -G 'https://api.search.brave.com/res/v1/web/search' --data-urlencode 'q=NVIDIA' --data-urlencode 'count=1' -H 'X-Subscription-Token: ${placeholder}' -w '\nHTTP_STATUS:%{http_code}\n'`, + { artifactName: "phase-4b-direct-brave-curl", timeoutMs: 60_000, redactionValues }, + ); + assertBraveResponse(resultText(curl)); + }, +);