diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index 858c5575946..62e7d27629d 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -639,6 +639,50 @@ jobs: if-no-files-found: ignore retention-days: 14 + hermes-sandbox-secret-boundary-vitest: + needs: generate-matrix + if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',hermes-sandbox-secret-boundary-vitest,') || contains(format(',{0},', inputs.scenarios), ',hermes-sandbox-secret-boundary,') }} + runs-on: ubuntu-latest + timeout-minutes: 60 + env: + FREE_STANDING_VITEST_JOB: "1" + FREE_STANDING_SCENARIO_ID: "hermes-sandbox-secret-boundary" + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/hermes-sandbox-secret-boundary + NEMOCLAW_RUN_E2E_SCENARIOS: "1" + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Set up Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 + with: + node-version: 22 + cache: npm + + - name: Install root dependencies + run: npm ci --ignore-scripts + + - name: Run Hermes sandbox secret-boundary live test + # Migrated from test/e2e/test-hermes-sandbox-secret-boundary.sh. This + # builds the real Hermes images unless prebuilt NEMOCLAW_HERMES_* image + # env vars are supplied, then probes image and startup secret boundaries. + run: | + set -euo pipefail + npx vitest run --project e2e-scenarios-live \ + test/e2e-scenario/live/hermes-sandbox-secret-boundary.test.ts \ + --silent=false --reporter=default + + - name: Upload Hermes sandbox secret-boundary artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: e2e-vitest-scenarios-hermes-sandbox-secret-boundary + path: e2e-artifacts/vitest/hermes-sandbox-secret-boundary/ + include-hidden-files: false + if-no-files-found: ignore + retention-days: 14 + inference-routing-vitest: needs: generate-matrix if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',inference-routing-vitest,') || contains(format(',{0},', inputs.scenarios), ',inference-routing,') }} @@ -5326,6 +5370,7 @@ jobs: hermes-slack-vitest, hermes-discord-vitest, hermes-root-entrypoint-smoke-vitest, + hermes-sandbox-secret-boundary-vitest, network-policy-vitest, common-egress-agent-vitest, shields-config-vitest, diff --git a/test/e2e-scenario/fixtures/docker-probe.ts b/test/e2e-scenario/fixtures/docker-probe.ts index 42d6c64f323..f3356e49ad9 100644 --- a/test/e2e-scenario/fixtures/docker-probe.ts +++ b/test/e2e-scenario/fixtures/docker-probe.ts @@ -29,6 +29,13 @@ export type DockerProbeRunner = ( options: SpawnSyncOptionsWithStringEncoding, ) => SpawnSyncReturns; +type DockerProbeRunOptions = { + artifactName: string; + timeoutMs?: number; + artifactRedactionValues?: string[]; + returnRaw?: boolean; +}; + const DOCKER_ENV_ALLOWLIST = [ "DOCKER_HOST", "DOCKER_CONTEXT", @@ -98,7 +105,7 @@ export class DockerProbe { async run( args: string[], - options: { artifactName: string; timeoutMs?: number } = { artifactName: "docker" }, + options: DockerProbeRunOptions = { artifactName: "docker" }, ): Promise { fs.mkdirSync(this.dockerConfigDir, { recursive: true }); const command = ["docker", ...args]; @@ -109,16 +116,16 @@ export class DockerProbe { maxBuffer: 10 * 1024 * 1024, timeout: options.timeoutMs ?? 30_000, }); - const commandResult = redactDockerProbeResult( - { - command, - exitCode: result.status, - signal: result.signal, - stdout: result.stdout ?? "", - stderr: result.stderr ?? "", - error: result.error instanceof Error ? result.error.message : undefined, - }, - this.redact, + const rawCommandResult = { + command, + exitCode: result.status, + signal: result.signal, + stdout: result.stdout ?? "", + stderr: result.stderr ?? "", + error: result.error instanceof Error ? result.error.message : undefined, + }; + const commandResult = redactDockerProbeResult(rawCommandResult, (text) => + this.redact(text, options.artifactRedactionValues ?? []), ); const artifactBase = `docker/${String(++this.sequence).padStart(3, "0")}-${safeName( options.artifactName, @@ -126,13 +133,15 @@ export class DockerProbe { await this.artifacts.writeText(`${artifactBase}.stdout.txt`, commandResult.stdout); await this.artifacts.writeText(`${artifactBase}.stderr.txt`, commandResult.stderr); await this.artifacts.writeJson(`${artifactBase}.result.json`, commandResult); - return commandResult; + return options.returnRaw === true ? rawCommandResult : commandResult; } - async expect( - args: string[], - options: { artifactName: string; timeoutMs?: number }, - ): Promise { + async expect(args: string[], options: DockerProbeRunOptions): Promise { + if (options.returnRaw === true) { + throw new Error( + "DockerProbe.expect cannot return raw Docker output; use run(..., { returnRaw: true }) only for explicit leak assertions that never log the raw result.", + ); + } const result = await this.run(args, options); if (result.exitCode !== 0) { throw new Error(resultText(result)); diff --git a/test/e2e-scenario/live/hermes-sandbox-secret-boundary.test.ts b/test/e2e-scenario/live/hermes-sandbox-secret-boundary.test.ts new file mode 100644 index 00000000000..171214faa25 --- /dev/null +++ b/test/e2e-scenario/live/hermes-sandbox-secret-boundary.test.ts @@ -0,0 +1,651 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { Buffer } from "node:buffer"; + +import { type DockerCommandResult, DockerProbe, resultText } from "../fixtures/docker-probe.ts"; +import { expect, test } from "../fixtures/e2e-test.ts"; + +// Migrated from test/e2e/test-hermes-sandbox-secret-boundary.sh. This remains +// a real Docker/image-boundary smoke: it builds the Hermes sandbox images when +// prebuilt image env vars are absent, inspects /sandbox/.hermes inside the +// images, and executes /usr/local/bin/nemoclaw-start to prove raw secret-shaped +// config/process env is rejected without echoing values. + +const BUILD_TIMEOUT_MS = 10 * 60_000; +const RUN_TIMEOUT_MS = 60_000; +const RAW_SECRET_SENTINEL = "SENTINEL_RAW_SECRET_VALUE"; +const RAW_REFRESH_TOKEN = "raw-refresh-token"; + +const liveTest = process.env.NEMOCLAW_RUN_E2E_SCENARIOS === "1" ? test : test.skip; + +const IMAGE_INSPECTION_SCRIPT = String.raw` +import re +import sys +from pathlib import Path + +secret_key_re = re.compile(r"(^|_)(TOKEN|KEY|SECRET|PASSWORD|CREDENTIAL|API)(_|$)") +slack_alias_re = re.compile(r"^(xoxb|xapp)-OPENSHELL-RESOLVE-ENV-[A-Z0-9_]+$") +allowed_nonsecret_keys = {"API_SERVER_HOST", "API_SERVER_PORT"} +allowed_literals = {"", "[STRIPPED_BY_MIGRATION]"} +required_remote_toolsets = { + "web", + "browser", + "terminal", + "file", + "code_execution", + "vision", + "image_gen", + "skills", + "todo", + "memory", + "session_search", + "delegation", + "cronjob", + "nemoclaw", + "audio", +} + + +def unquote(value: str) -> str: + value = value.strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in ("'", '"'): + return value[1:-1] + return value + + +def env_violations(path: Path) -> list[str]: + violations: list[str] = [] + for lineno, raw_line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): + stripped = raw_line.strip() + if not stripped or stripped.startswith("#") or "=" not in stripped: + continue + if stripped.startswith("export "): + stripped = stripped[len("export ") :].lstrip() + key, value = stripped.split("=", 1) + key = key.strip() + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", key): + continue + if key in allowed_nonsecret_keys: + continue + if not secret_key_re.search(key): + continue + value = unquote(value) + if ( + value in allowed_literals + or value.startswith("openshell:resolve:env:") + or slack_alias_re.fullmatch(value) + ): + continue + violations.append(f"{key} line {lineno}") + return violations + + +def parse_platform_toolsets(text: str) -> dict[str, list[str]]: + toolsets: dict[str, list[str]] = {} + in_block = False + block_indent = 0 + current: str | None = None + for raw_line in text.splitlines(): + stripped = raw_line.strip() + if not stripped or stripped.startswith("#"): + continue + indent = len(raw_line) - len(raw_line.lstrip(" ")) + if stripped == "platform_toolsets:": + in_block = True + block_indent = indent + continue + if not in_block: + continue + if indent <= block_indent and not stripped.startswith("- "): + break + key_match = re.fullmatch(r"([A-Za-z0-9_-]+):(?:\s*\[\])?", stripped) + if key_match: + current = key_match.group(1) + toolsets[current] = [] + continue + if stripped.startswith("- ") and current: + toolsets[current].append(unquote(stripped[2:])) + return toolsets + + +env_path = Path("/sandbox/.hermes/.env") +config_path = Path("/sandbox/.hermes/config.yaml") +if env_path.is_symlink(): + print(f"{env_path} is a symlink", file=sys.stderr) + sys.exit(1) +if not env_path.is_file(): + print(f"{env_path} missing", file=sys.stderr) + sys.exit(1) +if not config_path.is_file(): + print(f"{config_path} missing", file=sys.stderr) + sys.exit(1) + +violations = env_violations(env_path) +if violations: + print("raw secret-shaped Hermes .env values:", ", ".join(violations), file=sys.stderr) + sys.exit(1) + +toolsets = parse_platform_toolsets(config_path.read_text(encoding="utf-8")) +api_server_toolsets = set(toolsets.get("api_server", [])) +if not api_server_toolsets: + print("platform_toolsets.api_server missing", file=sys.stderr) + sys.exit(1) +missing = sorted(required_remote_toolsets - api_server_toolsets) +if missing: + print(f"platform_toolsets.api_server missing expected Hermes toolsets: {missing}", file=sys.stderr) + sys.exit(1) +if "no_mcp" in api_server_toolsets: + print("platform_toolsets.api_server unexpectedly disables default MCP servers with no_mcp", file=sys.stderr) + sys.exit(1) +`; + +const MANAGED_TOOL_INSPECTION_SCRIPT = String.raw` +import re +import sys +from pathlib import Path + +secret_key_re = re.compile(r"(^|_)(TOKEN|KEY|SECRET|PASSWORD|CREDENTIAL|API)(_|$)") +slack_alias_re = re.compile(r"^(xoxb|xapp)-OPENSHELL-RESOLVE-ENV-[A-Z0-9_]+$") +allowed_nonsecret_keys = {"API_SERVER_HOST", "API_SERVER_PORT"} +allowed_literals = {"", "[STRIPPED_BY_MIGRATION]"} +required_env_lines = { + "NEMOCLAW_HERMES_TOOL_GATEWAY_BROKER=1", + "FIRECRAWL_GATEWAY_URL=http://host.openshell.internal:11436/firecrawl", + "OPENAI_AUDIO_GATEWAY_URL=http://host.openshell.internal:11436/openai-audio", + "BROWSER_USE_GATEWAY_URL=http://host.openshell.internal:11436/browser-use", + "FAL_QUEUE_GATEWAY_URL=http://host.openshell.internal:11436/fal-queue", + "MODAL_GATEWAY_URL=http://host.openshell.internal:11436/modal", +} +required_config_fragments = [ + "backend: firecrawl", + "provider: openai", + "cloud_provider: browser-use", + "image_gen:", + "backend: modal", + "modal_mode: managed", + "tts:", +] + + +def unquote(value: str) -> str: + value = value.strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in ("'", '"'): + return value[1:-1] + return value + + +def env_violations(path: Path) -> list[str]: + violations: list[str] = [] + for lineno, raw_line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): + stripped = raw_line.strip() + if not stripped or stripped.startswith("#") or "=" not in stripped: + continue + if stripped.startswith("export "): + stripped = stripped[len("export ") :].lstrip() + key, value = stripped.split("=", 1) + key = key.strip() + if key in allowed_nonsecret_keys: + continue + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", key): + continue + if not secret_key_re.search(key): + continue + value = unquote(value) + if ( + value in allowed_literals + or value.startswith("openshell:resolve:env:") + or slack_alias_re.fullmatch(value) + ): + continue + violations.append(f"{key} line {lineno}") + return violations + + +env_path = Path("/sandbox/.hermes/.env") +config_path = Path("/sandbox/.hermes/config.yaml") +if not env_path.is_file() or env_path.is_symlink(): + print(f"{env_path} missing, not a file, or unsafe symlink", file=sys.stderr) + sys.exit(1) +if not config_path.is_file(): + print(f"{config_path} missing", file=sys.stderr) + sys.exit(1) + +env_text = env_path.read_text(encoding="utf-8") +config_text = config_path.read_text(encoding="utf-8") +env_lines = set(env_text.splitlines()) +violations = env_violations(env_path) +if violations: + print("raw secret-shaped managed-tool .env values:", ", ".join(violations), file=sys.stderr) + sys.exit(1) + +missing_env = sorted(required_env_lines - env_lines) +missing_config = [fragment for fragment in required_config_fragments if fragment not in config_text] +for forbidden in ( + "TOOL_GATEWAY_USER_TOKEN", + "NEMOCLAW_HERMES_TOOL_GATEWAY_REFRESH_TOKEN=", + "raw-refresh-token", +): + if forbidden in env_text or forbidden in config_text: + print(f"managed-tool sandbox config contains forbidden token surface: {forbidden}", file=sys.stderr) + sys.exit(1) + +if missing_env: + print("managed-tool .env missing expected gateway lines: " + ", ".join(missing_env), file=sys.stderr) + sys.exit(1) +if missing_config: + print("managed-tool config.yaml missing expected fragments: " + ", ".join(missing_config), file=sys.stderr) + sys.exit(1) +`; + +function safeTag(value: string): string { + return value.replace(/[^A-Za-z0-9_.-]+/g, "-").replace(/^-+|-+$/g, "") || "local"; +} + +function shellQuote(value: string): string { + return `'${value.replaceAll("'", "'\\''")}'`; +} + +function safeArtifactPart(value: string): string { + return value.toLowerCase().replaceAll("_", "-"); +} + +function managedPresetsBase64(): string { + return Buffer.from( + JSON.stringify(["nous-web", "nous-audio", "nous-browser", "nous-image", "nous-code"]), + ).toString("base64"); +} + +function throwDockerRequired(result: DockerCommandResult): never { + throw new Error(`Docker is required for Hermes sandbox secret-boundary:\n${resultText(result)}`); +} + +async function requireDocker(probe: DockerProbe, skip: (message: string) => void): Promise { + const result = await probe.run(["info"], { artifactName: "docker-info", timeoutMs: 30_000 }); + result.exitCode === 0 + ? undefined + : process.env.GITHUB_ACTIONS === "true" + ? throwDockerRequired(result) + : skip("Docker daemon is required for Hermes sandbox secret-boundary"); +} + +function inspectPrebuiltImage(probe: DockerProbe, image: string): Promise { + return probe + .expect(["image", "inspect", image], { + artifactName: "inspect-prebuilt-hermes-image", + timeoutMs: 30_000, + }) + .then(() => false); +} + +async function buildBaseImageWhenNeeded( + probe: DockerProbe, + baseImage: string, + baseImageFromEnv: boolean, + artifactName: string, +): Promise { + await (baseImageFromEnv + ? Promise.resolve() + : probe.expect(["build", "-f", "agents/hermes/Dockerfile.base", "-t", baseImage, "."], { + artifactName, + timeoutMs: BUILD_TIMEOUT_MS, + })); +} + +async function buildHermesProductionImage( + probe: DockerProbe, + image: string, + baseImage: string, +): Promise { + await probe.expect( + [ + "build", + "-f", + "agents/hermes/Dockerfile", + "--build-arg", + `BASE_IMAGE=${baseImage}`, + "-t", + image, + ".", + ], + { artifactName: "build-hermes-production-image", timeoutMs: BUILD_TIMEOUT_MS }, + ); + return true; +} + +async function buildHermesImageIfNeeded( + probe: DockerProbe, + image: string, + baseImage: string, + baseImageFromEnv: boolean, +): Promise { + return process.env.NEMOCLAW_HERMES_TEST_IMAGE + ? inspectPrebuiltImage(probe, image) + : (await buildBaseImageWhenNeeded( + probe, + baseImage, + baseImageFromEnv, + "build-hermes-base-image", + ), + buildHermesProductionImage(probe, image, baseImage)); +} + +function inspectPrebuiltManagedImage(probe: DockerProbe, managedImage: string): Promise { + return probe + .expect(["image", "inspect", managedImage], { + artifactName: "inspect-prebuilt-managed-hermes-image", + timeoutMs: 30_000, + }) + .then(() => false); +} + +async function buildManagedBaseImageWhenNeeded( + probe: DockerProbe, + baseImage: string, + baseImageFromEnv: boolean, +): Promise { + const baseExists = await probe.run(["image", "inspect", baseImage], { + artifactName: "inspect-managed-base-image", + timeoutMs: 30_000, + }); + await (baseImageFromEnv || baseExists.exitCode === 0 + ? Promise.resolve() + : probe.expect(["build", "-f", "agents/hermes/Dockerfile.base", "-t", baseImage, "."], { + artifactName: "build-managed-hermes-base-image", + timeoutMs: BUILD_TIMEOUT_MS, + })); +} + +async function buildManagedProductionImage( + probe: DockerProbe, + managedImage: string, + baseImage: string, +): Promise { + await probe.expect( + [ + "build", + "-f", + "agents/hermes/Dockerfile", + "--build-arg", + `BASE_IMAGE=${baseImage}`, + "--build-arg", + "NEMOCLAW_HERMES_TOOL_GATEWAY_BROKER=1", + "--build-arg", + `NEMOCLAW_HERMES_TOOL_GATEWAY_PRESETS_B64=${managedPresetsBase64()}`, + "-t", + managedImage, + ".", + ], + { artifactName: "build-managed-hermes-production-image", timeoutMs: BUILD_TIMEOUT_MS }, + ); + return true; +} + +async function buildManagedImageIfNeeded( + probe: DockerProbe, + managedImage: string, + baseImage: string, + baseImageFromEnv: boolean, +): Promise { + return process.env.NEMOCLAW_HERMES_MANAGED_TEST_IMAGE + ? inspectPrebuiltManagedImage(probe, managedImage) + : (await buildManagedBaseImageWhenNeeded(probe, baseImage, baseImageFromEnv), + buildManagedProductionImage(probe, managedImage, baseImage)); +} + +async function inspectImageBoundary(probe: DockerProbe, image: string): Promise { + const result = await probe.run( + ["run", "--rm", "--entrypoint", "python3", image, "-c", IMAGE_INSPECTION_SCRIPT], + { + artifactName: "inspect-hermes-sandbox-boundary", + timeoutMs: RUN_TIMEOUT_MS, + }, + ); + + expect( + result.exitCode, + `Hermes image should have no raw secret-shaped .env values and preserve remote toolsets\n${resultText(result)}`, + ).toBe(0); +} + +async function inspectManagedToolBoundary(probe: DockerProbe, image: string): Promise { + const result = await probe.run( + ["run", "--rm", "--entrypoint", "python3", image, "-c", MANAGED_TOOL_INSPECTION_SCRIPT], + { + artifactName: "inspect-managed-tool-hermes-boundary", + timeoutMs: RUN_TIMEOUT_MS, + }, + ); + + expect( + result.exitCode, + `Managed-tool Hermes image should keep gateway auth out of sandbox config\n${resultText(result)}`, + ).toBe(0); +} + +function redactExpectedSecret(value: string, text: string): string { + return text.split(value).join("[REDACTED_TEST_SECRET]"); +} + +async function runStartupWithEnvFileEntry( + probe: DockerProbe, + image: string, + assignment: string, + key: string, + value: string, +): Promise { + const script = `set -euo pipefail; printf '%s\n' ${shellQuote( + assignment, + )} >> /sandbox/.hermes/.env; exec /usr/local/bin/nemoclaw-start true`; + return probe.run( + ["run", "--rm", "--user", "sandbox", "--entrypoint", "/bin/bash", image, "-lc", script], + { + artifactName: `startup-rejects-env-file-${safeArtifactPart(key)}`, + artifactRedactionValues: [value], + returnRaw: true, + timeoutMs: RUN_TIMEOUT_MS, + }, + ); +} + +async function runStartupWithRuntimeEnvEntry( + probe: DockerProbe, + image: string, + assignment: string, + key: string, + value: string, +): Promise { + return probe.run( + [ + "run", + "--rm", + "--user", + "sandbox", + "--env", + assignment, + "--entrypoint", + "/usr/local/bin/nemoclaw-start", + image, + "true", + ], + { + artifactName: `startup-rejects-runtime-env-${safeArtifactPart(key)}`, + artifactRedactionValues: [value], + returnRaw: true, + timeoutMs: RUN_TIMEOUT_MS, + }, + ); +} + +async function expectStartupRejectsEnvFileEntry( + probe: DockerProbe, + image: string, + assignment: string, + key: string, + value: string, +): Promise { + const result = await runStartupWithEnvFileEntry(probe, image, assignment, key, value); + const output = `${result.stdout}\n${result.stderr}`; + const safeResultText = redactExpectedSecret(value, resultText(result)); + + expect(result.exitCode, `Hermes startup should reject ${key}\n${safeResultText}`).not.toBe(0); + expect( + output, + `Hermes startup rejection should mention raw secret-shaped values for ${key}`, + ).toContain("raw secret-shaped values"); + expect(output, `Hermes startup rejection should name ${key}`).toContain(key); + expect(output, `Hermes startup rejection must not print ${key}'s raw value`).not.toContain(value); +} + +async function expectStartupRejectsRuntimeEnvEntry( + probe: DockerProbe, + image: string, + assignment: string, + key: string, + value: string, +): Promise { + const result = await runStartupWithRuntimeEnvEntry(probe, image, assignment, key, value); + const output = `${result.stdout}\n${result.stderr}`; + const safeResultText = redactExpectedSecret(value, resultText(result)); + + expect( + result.exitCode, + `Hermes startup should reject runtime env ${key}\n${safeResultText}`, + ).not.toBe(0); + expect( + output, + `Hermes startup rejection should mention process environment for ${key}`, + ).toContain("process environment"); + expect(output, `Hermes startup rejection should name ${key}`).toContain(key); + expect(output, `Hermes startup rejection must not print ${key}'s raw value`).not.toContain(value); +} + +liveTest( + "hermes sandbox secret boundary keeps raw secrets out of images and startup", + async ({ artifacts, cleanup, secrets, skip }) => { + const probe = new DockerProbe(artifacts, (text, extraValues) => + secrets.redact(text, extraValues), + ); + const runId = safeTag(`${process.env.GITHUB_RUN_ID ?? "local"}-${process.pid}-${Date.now()}`); + const baseImageFromEnv = Boolean( + process.env.NEMOCLAW_HERMES_BASE_IMAGE ?? process.env.HERMES_BASE_IMAGE, + ); + const image = + process.env.NEMOCLAW_HERMES_TEST_IMAGE ?? `nemoclaw-hermes-secret-boundary:${runId}`; + const baseImage = + process.env.NEMOCLAW_HERMES_BASE_IMAGE ?? + process.env.HERMES_BASE_IMAGE ?? + `nemoclaw-hermes-secret-boundary-base:${runId}`; + const managedImage = + process.env.NEMOCLAW_HERMES_MANAGED_TEST_IMAGE ?? + `nemoclaw-hermes-secret-boundary-managed:${runId}`; + let removeImage = false; + let removeManagedImage = false; + let removeBaseImage = false; + + await artifacts.writeJson("scenario.json", { + id: "hermes-sandbox-secret-boundary", + runner: "vitest", + boundary: "docker-hermes-image-and-startup", + legacySource: "test/e2e/test-hermes-sandbox-secret-boundary.sh", + image, + baseImage, + managedImage, + prebuiltImage: Boolean(process.env.NEMOCLAW_HERMES_TEST_IMAGE), + prebuiltManagedImage: Boolean(process.env.NEMOCLAW_HERMES_MANAGED_TEST_IMAGE), + contract: [ + "Docker is required and prebuilt image env vars must reference inspectable images", + "Hermes .env in the sandbox image is a real file and has no raw secret-shaped values", + "Hermes config preserves api_server remote platform toolsets and does not use no_mcp", + "managed-tool image keeps gateway auth tokens out of sandbox env/config while preserving gateway URLs/config", + "nemoclaw-start rejects raw secret-shaped .env entries without echoing their values", + "nemoclaw-start rejects raw secret-shaped process env entries without echoing their values", + ], + }); + + cleanup.add("remove Hermes sandbox secret-boundary images", async () => { + const images = [ + removeImage ? image : undefined, + removeManagedImage ? managedImage : undefined, + removeBaseImage ? baseImage : undefined, + ].filter((value): value is string => Boolean(value)); + await (images.length === 0 + ? Promise.resolve() + : probe.run(["rmi", "-f", ...images], { + artifactName: "cleanup-hermes-secret-boundary-images", + timeoutMs: 60_000, + })); + }); + + await requireDocker(probe, skip); + + removeImage = await buildHermesImageIfNeeded(probe, image, baseImage, baseImageFromEnv); + await probe.expect(["image", "inspect", image], { + artifactName: "inspect-hermes-image-after-build", + timeoutMs: 30_000, + }); + removeManagedImage = await buildManagedImageIfNeeded( + probe, + managedImage, + baseImage, + baseImageFromEnv, + ); + removeBaseImage = !baseImageFromEnv && (removeImage || removeManagedImage); + await probe.expect(["image", "inspect", managedImage], { + artifactName: "inspect-managed-hermes-image-after-build", + timeoutMs: 30_000, + }); + + await inspectImageBoundary(probe, image); + await inspectManagedToolBoundary(probe, managedImage); + await expectStartupRejectsEnvFileEntry( + probe, + image, + `DEVTEST_API_TOKEN=${RAW_SECRET_SENTINEL}`, + "DEVTEST_API_TOKEN", + RAW_SECRET_SENTINEL, + ); + await expectStartupRejectsEnvFileEntry( + probe, + image, + `INTERNAL_API=${RAW_SECRET_SENTINEL}`, + "INTERNAL_API", + RAW_SECRET_SENTINEL, + ); + await expectStartupRejectsEnvFileEntry( + probe, + image, + "OPENAI_API_KEY=sk-OPENSHELL-PROXY-REWRITE", + "OPENAI_API_KEY", + "sk-OPENSHELL-PROXY-REWRITE", + ); + await expectStartupRejectsRuntimeEnvEntry( + probe, + image, + `DEVTEST_API_TOKEN=${RAW_SECRET_SENTINEL}`, + "DEVTEST_API_TOKEN", + RAW_SECRET_SENTINEL, + ); + await expectStartupRejectsRuntimeEnvEntry( + probe, + image, + `NEMOCLAW_HERMES_TOOL_GATEWAY_REFRESH_TOKEN=${RAW_REFRESH_TOKEN}`, + "NEMOCLAW_HERMES_TOOL_GATEWAY_REFRESH_TOKEN", + RAW_REFRESH_TOKEN, + ); + + await artifacts.writeJson("scenario-result.json", { + id: "hermes-sandbox-secret-boundary", + image, + managedImage, + assertions: { + imageEnvSecretBoundaryVerified: true, + imageRemoteToolsetsVerified: true, + managedToolGatewayAuthBoundaryVerified: true, + envFileSecretRejectionsVerified: true, + runtimeEnvSecretRejectionsVerified: true, + rejectionOutputRedactionVerified: true, + }, + }); + }, +); diff --git a/test/e2e-scenario/support-tests/docker-probe.test.ts b/test/e2e-scenario/support-tests/docker-probe.test.ts index 87139948368..8dec3a87084 100644 --- a/test/e2e-scenario/support-tests/docker-probe.test.ts +++ b/test/e2e-scenario/support-tests/docker-probe.test.ts @@ -106,6 +106,77 @@ describe("DockerProbe secret hygiene", () => { } }); + it("can return raw Docker output for leak assertions while writing only redacted artifacts", async () => { + const leakedSecret = "SENTINEL_RAW_SECRET_VALUE"; + const artifactsRoot = await fs.mkdtemp(path.join(os.tmpdir(), "docker-probe-raw-output-")); + const artifacts = new ArtifactSink(artifactsRoot); + const secrets = new SecretStore({}, (message) => { + throw new Error(message ?? "unexpected skip"); + }); + const probe = new DockerProbe( + artifacts, + (text, extraValues) => secrets.redact(text, extraValues), + () => ({ + pid: 123, + output: [null, "", `startup rejected DEVTEST_API_TOKEN=${leakedSecret}`], + stdout: "", + stderr: `startup rejected DEVTEST_API_TOKEN=${leakedSecret}`, + status: 1, + signal: null, + }), + ); + + const result = await probe.run(["run", "hermes"], { + artifactName: "startup-rejects-env-file-devtest-api-token", + artifactRedactionValues: [leakedSecret], + returnRaw: true, + }); + + expect(result.stderr).toContain(leakedSecret); + const stdoutArtifact = await readArtifact( + artifactsRoot, + "docker/001-startup-rejects-env-file-devtest-api-token.stdout.txt", + ); + expect(stdoutArtifact).not.toContain(leakedSecret); + for (const relativePath of [ + "docker/001-startup-rejects-env-file-devtest-api-token.stderr.txt", + "docker/001-startup-rejects-env-file-devtest-api-token.result.json", + ]) { + const artifact = await readArtifact(artifactsRoot, relativePath); + expect(artifact).not.toContain(leakedSecret); + expect(artifact).toContain("[REDACTED]"); + } + }); + + it("rejects raw Docker output from expect to keep thrown diagnostics redacted", async () => { + const leakedSecret = "SENTINEL_RAW_SECRET_VALUE"; + const artifactsRoot = await fs.mkdtemp(path.join(os.tmpdir(), "docker-probe-expect-raw-")); + const artifacts = new ArtifactSink(artifactsRoot); + const secrets = new SecretStore({}, (message) => { + throw new Error(message ?? "unexpected skip"); + }); + const probe = new DockerProbe( + artifacts, + (text, extraValues) => secrets.redact(text, extraValues), + () => ({ + pid: 123, + output: [null, "", `startup rejected DEVTEST_API_TOKEN=${leakedSecret}`], + stdout: "", + stderr: `startup rejected DEVTEST_API_TOKEN=${leakedSecret}`, + status: 1, + signal: null, + }), + ); + + await expect( + probe.expect(["run", "hermes"], { + artifactName: "startup-rejects-env-file-devtest-api-token", + artifactRedactionValues: [leakedSecret], + returnRaw: true, + }), + ).rejects.toThrow("DockerProbe.expect cannot return raw Docker output"); + }); + it("redacts diagnostic-style Docker inspect, logs, process, start-log, and gateway-log artifacts", async () => { const secret = "docker-diagnostic-artifact-secret"; const diagnostics = new Map([ diff --git a/test/e2e-scenario/support-tests/hermes-secret-boundary-workflow.test.ts b/test/e2e-scenario/support-tests/hermes-secret-boundary-workflow.test.ts new file mode 100644 index 00000000000..3d89e26448a --- /dev/null +++ b/test/e2e-scenario/support-tests/hermes-secret-boundary-workflow.test.ts @@ -0,0 +1,103 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { expect, it } from "vitest"; +import YAML from "yaml"; + +import { + evaluateE2eVitestWorkflowDispatchSelectors, + readFreeStandingJobsInventory, + validateE2eVitestScenariosWorkflowBoundary, +} from "../../../tools/e2e-scenarios/workflow-boundary.mts"; + +function readWorkflow(): Record { + return YAML.parse( + fs.readFileSync( + path.join(process.cwd(), ".github/workflows/e2e-vitest-scenarios.yaml"), + "utf-8", + ), + ) as Record; +} + +it("routes Hermes sandbox secret-boundary selective dispatch to its free-standing Vitest job", () => { + const inventory = readFreeStandingJobsInventory(); + + expect(inventory.allowedJobs).toContain("hermes-sandbox-secret-boundary-vitest"); + expect(inventory.scenarioToJob.get("hermes-sandbox-secret-boundary")).toBe( + "hermes-sandbox-secret-boundary-vitest", + ); + expect( + evaluateE2eVitestWorkflowDispatchSelectors({ + scenarios: "hermes-sandbox-secret-boundary", + }), + ).toMatchObject({ + valid: true, + liveScenariosRuns: false, + selectedFreeStandingJobs: ["hermes-sandbox-secret-boundary-vitest"], + registryScenarios: [], + }); + expect( + evaluateE2eVitestWorkflowDispatchSelectors({ + jobs: "hermes-sandbox-secret-boundary-vitest", + }), + ).toMatchObject({ + valid: true, + liveScenariosRuns: false, + selectedFreeStandingJobs: ["hermes-sandbox-secret-boundary-vitest"], + registryScenarios: [], + }); +}); + +it("rejects Hermes sandbox secret-boundary workflow secret and Docker-auth drift", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "hermes-secret-boundary-workflow-")); + const workflowPath = path.join(tmp, "workflow.yaml"); + const workflow = readWorkflow() as { + jobs: Record; steps: Array> }>; + }; + const job = workflow.jobs["hermes-sandbox-secret-boundary-vitest"]; + expect(job).toBeDefined(); + + job.env = { + ...job.env, + NVIDIA_INFERENCE_API_KEY: "${{ secrets.NVIDIA_INFERENCE_API_KEY }}", + DOCKERHUB_USERNAME: "${{ secrets.DOCKERHUB_USERNAME }}", + DOCKERHUB_TOKEN: "${{ secrets.DOCKERHUB_TOKEN }}", + }; + job.steps.splice(1, 0, { + name: "Authenticate to Docker Hub", + env: { + DOCKERHUB_USERNAME: "${{ secrets.DOCKERHUB_USERNAME }}", + DOCKERHUB_TOKEN: "${{ secrets.DOCKERHUB_TOKEN }}", + }, + run: "docker login docker.io --username $DOCKERHUB_USERNAME --password $DOCKERHUB_TOKEN", + }); + const runVitest = job.steps.find( + (step) => step.name === "Run Hermes sandbox secret-boundary live test", + ); + expect(runVitest).toBeDefined(); + runVitest!.env = { + NVIDIA_INFERENCE_API_KEY: "${{ secrets.NVIDIA_INFERENCE_API_KEY }}", + }; + fs.writeFileSync(workflowPath, YAML.stringify(workflow)); + + try { + expect(validateE2eVitestScenariosWorkflowBoundary(workflowPath)).toEqual( + expect.arrayContaining([ + "hermes-sandbox-secret-boundary-vitest job env must not include NVIDIA_INFERENCE_API_KEY", + "hermes-sandbox-secret-boundary-vitest job env must not include DOCKERHUB_USERNAME", + "hermes-sandbox-secret-boundary-vitest job env must not include DOCKERHUB_TOKEN", + "hermes-sandbox-secret-boundary-vitest must not authenticate to Docker Hub before branch-controlled test code runs", + "hermes-sandbox-secret-boundary-vitest step 'Authenticate to Docker Hub' env must not include DOCKERHUB_USERNAME", + "hermes-sandbox-secret-boundary-vitest step 'Authenticate to Docker Hub' env must not include DOCKERHUB_TOKEN", + "hermes-sandbox-secret-boundary-vitest step 'Authenticate to Docker Hub' run script must not use docker login or inline secret interpolation", + "hermes-sandbox-secret-boundary-vitest step 'Run Hermes sandbox secret-boundary live test' env must not include NVIDIA_INFERENCE_API_KEY", + ]), + ); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } +}); diff --git a/tools/e2e-scenarios/workflow-boundary.mts b/tools/e2e-scenarios/workflow-boundary.mts index 756db5252ff..4ab480940a6 100644 --- a/tools/e2e-scenarios/workflow-boundary.mts +++ b/tools/e2e-scenarios/workflow-boundary.mts @@ -4141,6 +4141,135 @@ function validateHermesRootEntrypointSmokeVitestJob( } } +function validateHermesSandboxSecretBoundaryVitestJob( + errors: string[], + jobs: WorkflowRecord, +): void { + const jobName = "hermes-sandbox-secret-boundary-vitest"; + const scenarioName = "hermes-sandbox-secret-boundary"; + const job = asRecord(jobs[jobName]); + if (Object.keys(job).length === 0) { + errors.push("workflow missing hermes-sandbox-secret-boundary-vitest job"); + return; + } + + if (job["runs-on"] !== "ubuntu-latest") { + errors.push("hermes-sandbox-secret-boundary-vitest job must run on ubuntu-latest"); + } + validateFreeStandingJobSelector(errors, jobs, jobName, scenarioName); + if (job["timeout-minutes"] !== 60) { + errors.push("hermes-sandbox-secret-boundary-vitest job must keep the 60 minute timeout"); + } + + const jobEnv = asRecord(job.env); + if (jobEnv.NEMOCLAW_RUN_E2E_SCENARIOS !== "1") { + errors.push("hermes-sandbox-secret-boundary-vitest job must set NEMOCLAW_RUN_E2E_SCENARIOS=1"); + } + if ( + jobEnv.E2E_ARTIFACT_DIR !== + "${{ github.workspace }}/e2e-artifacts/vitest/hermes-sandbox-secret-boundary" + ) { + errors.push( + "hermes-sandbox-secret-boundary-vitest job must write artifacts under e2e-artifacts/vitest/hermes-sandbox-secret-boundary", + ); + } + for (const secret of ["NVIDIA_INFERENCE_API_KEY", "DOCKERHUB_USERNAME", "DOCKERHUB_TOKEN"]) { + requireEnvDoesNotExposeSecret( + errors, + "hermes-sandbox-secret-boundary-vitest job", + jobEnv, + secret, + ); + } + + const steps = asSteps(job.steps); + requireNoDispatchInputInterpolation(errors, steps); + for (const step of steps) { + const stepName = `hermes-sandbox-secret-boundary-vitest step '${ + step.name ?? step.uses ?? "" + }'`; + const stepEnv = asRecord(step.env); + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "NVIDIA_INFERENCE_API_KEY"); + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "DOCKERHUB_USERNAME"); + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "DOCKERHUB_TOKEN"); + requireNoDockerHubAuthInRun(errors, stepName, stringValue(step.run)); + } + + if (namedStep(steps, "Authenticate to Docker Hub")) { + errors.push( + "hermes-sandbox-secret-boundary-vitest must not authenticate to Docker Hub before branch-controlled test code runs", + ); + } + + const checkout = steps.find((step) => stringValue(step.uses).startsWith("actions/checkout@")); + if (!checkout) errors.push("hermes-sandbox-secret-boundary-vitest job missing checkout step"); + requireFullShaAction(errors, checkout, "hermes-sandbox-secret-boundary-vitest checkout"); + if (asRecord(checkout?.with)["persist-credentials"] !== false) { + errors.push( + "hermes-sandbox-secret-boundary-vitest checkout step must set persist-credentials=false", + ); + } + + const setupNode = namedStep(steps, "Set up Node"); + if (!setupNode) + errors.push("hermes-sandbox-secret-boundary-vitest job missing step: Set up Node"); + requireFullShaAction(errors, setupNode, "hermes-sandbox-secret-boundary-vitest setup-node"); + + const installRootDependencies = requireJobStep( + errors, + jobName, + steps, + "Install root dependencies", + ); + requireRunContains(errors, installRootDependencies, "npm ci --ignore-scripts"); + + const runVitest = requireJobStep( + errors, + jobName, + steps, + "Run Hermes sandbox secret-boundary live test", + ); + requireRunContains(errors, runVitest, "npx vitest run --project e2e-scenarios-live"); + requireRunContains( + errors, + runVitest, + "test/e2e-scenario/live/hermes-sandbox-secret-boundary.test.ts", + ); + requireRunDoesNotContain(errors, runVitest, "${{ inputs."); + + const upload = requireJobStep( + errors, + jobName, + steps, + "Upload Hermes sandbox secret-boundary artifacts", + ); + requireFullShaAction(errors, upload, "hermes-sandbox-secret-boundary-vitest upload-artifact"); + const uploadWith = asRecord(upload?.with); + if (uploadWith.name !== "e2e-vitest-scenarios-hermes-sandbox-secret-boundary") { + errors.push("hermes-sandbox-secret-boundary-vitest artifact upload name must be stable"); + } + const uploadPath = stringValue(uploadWith.path); + requireUploadPathContains( + errors, + uploadPath, + "e2e-artifacts/vitest/hermes-sandbox-secret-boundary/", + ); + if (uploadWith["include-hidden-files"] !== false) { + errors.push( + "hermes-sandbox-secret-boundary-vitest artifact upload must set include-hidden-files: false", + ); + } + if (uploadWith["if-no-files-found"] !== "ignore") { + errors.push( + "hermes-sandbox-secret-boundary-vitest artifact upload must ignore missing fixture artifacts", + ); + } + if (uploadWith["retention-days"] !== 14) { + errors.push("hermes-sandbox-secret-boundary-vitest artifact upload retention-days must be 14"); + } +} + + function validateDiagnosticsVitestJob( errors: string[], jobs: WorkflowRecord, @@ -7564,6 +7693,7 @@ export function validateE2eVitestScenariosWorkflowBoundary( validateHermesE2EVitestJob(errors, jobs); validateFreeStandingJobSelector(errors, jobs, "hermes-discord-vitest", "hermes-discord"); validateHermesRootEntrypointSmokeVitestJob(errors, jobs); + validateHermesSandboxSecretBoundaryVitestJob(errors, jobs); validateNetworkPolicyVitestJob(errors, jobs); validateCommonEgressAgentVitestJob(errors, jobs); validateShieldsConfigVitestJob(errors, jobs);